From f0ecc24be319ebbb2667d260b8071922cf284ee8 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 12:47:12 -0400 Subject: [PATCH 01/21] fix: run local user commands in a dedicated workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local user commands (run_command, run_script, manage_process) inherited the service's working directory, which is the install root. A bare relative path therefore resolved against the live install: on 2026-07-27, AE2 jar research ran `jar xf data/ae2/... && cat ... && rm -rf data`, and because the jar's internal layout starts with `data/`, that cleanup deleted /opt/odin/data — memory, learned lessons, skills, sessions, trajectories, the knowledge index, and all three codex_auth files. The intent was legitimate. The relative path was the whole bug. So this closes the mechanism without touching the capability: - tools.local_working_dir (default /var/lib/odin-workspace) is resolved and validated, then passed as the subprocess cwd for local user commands. - Deliberately a SIBLING of /var/lib/odin, since packaged installs use that as the live data directory. Not /tmp (tmpfiles policy ages it out), not $HOME (packaged Odin declares /opt/odin as the service account's home). - STABLE and persistent by design. A fresh directory per command would break two-step workflows that write a relative file in one command and read it in the next — that would cost capability. Restart-required, not hot-reloadable, since swapping mid-run breaks exactly that continuity. - PWD *and* OLDPWD are normalized. cwd= alone leaves an inherited OLDPWD pointing at the install, so a bare `cd -` walks straight back in. - Background starts share the workspace, or manage_process remains an alternate route to the same incident (29 historical background starts had no explicit cd). - Validation fails CLOSED: missing, symlinked, non-private, or inside-the-install workspaces raise rather than silently falling back to the inherited cwd, which would restore the hazard. Resolution is lazy so executor construction never depends on deployment having provisioned it. What is deliberately NOT changed: the Python process cwd and systemd WorkingDirectory stay /opt/odin (the app resolves ./data/... against them — moving those would recreate this incident by another door); explicit `cd`, absolute paths, and `git -C ` behave exactly as before, so working inside the install remains possible; remote SSH execution is untouched; and no prompt surface changes — no tool description, schema, or output format. Scope honesty: an external cwd prevents the accidental basename collision. It does not confine the process — `..`, an absolute path, a symlink inside the workspace, or archive path traversal can still reach the install. Preventing those needs filesystem confinement and would cost capability. Tests replay the actual incident: a synthetic jar containing data/ae2/..., extract + read + `rm -rf data` as three separate commands, asserting the workflow SUCCEEDS while a sentinel in the fake install survives. The test process chdirs into the fake install first, so a regression in the cwd plumbing destroys the fixture and fails loudly; every path is under tmp_path and that is asserted before any deletion runs. Also pinned: explicit cd into the install still works, absolute paths unaffected, relative files persist across commands, `cd -` cannot return to the install, background processes use the workspace, remote SSH has no cwd parameter, and the legacy src/odin/tools DAG surfaces are classified so they cannot be wired into the Discord path without reopening this. Gates: ruff clean, mypy clean (231 files), 7520 passed / 4 skipped. Designed with Odin; his review caught the OLDPWD hole, the /var/lib/odin collision, the background-process route, and the sibling-path requirement. --- src/config/schema.py | 18 ++ src/tools/executor.py | 45 ++++- src/tools/process_manager.py | 20 ++- src/tools/ssh.py | 24 ++- src/tools/workspace.py | 105 ++++++++++++ tests/conftest.py | 24 +++ tests/test_local_workspace.py | 300 ++++++++++++++++++++++++++++++++++ tests/test_output_streamer.py | 30 ++++ 8 files changed, 550 insertions(+), 16 deletions(-) create mode 100644 src/tools/workspace.py create mode 100644 tests/test_local_workspace.py diff --git a/src/config/schema.py b/src/config/schema.py index d5be54d5..4ee4d89a 100644 --- a/src/config/schema.py +++ b/src/config/schema.py @@ -217,6 +217,24 @@ class ToolsConfig(BaseModel): # Loops typically need more budget for exploration + execution + verify + commit. max_tool_iterations_chat: int = 500 max_tool_iterations_loop: int = 500 + # Working directory for USER-COMMAND local execution (run_command, + # run_script, manage_process). Before this existed, those subprocesses + # inherited systemd's WorkingDirectory=/opt/odin, so a bare relative path + # in a command resolved against the live install — on 2026-07-27 an AE2 jar + # whose internal layout is `data/` was extracted and cleaned up with + # `rm -rf data`, which deleted /opt/odin/data. + # + # Deliberately a SIBLING of /var/lib/odin, not a child: packaged installs + # use /var/lib/odin as the live data directory behind /opt/odin/data. + # Not /tmp or /var/tmp (tmpfiles policy can age those out) and not $HOME + # (packaged Odin declares /opt/odin as the service account's home). + # + # Stable and persistent BY DESIGN: a fresh directory per command would + # break two-step workflows that write a relative file in one command and + # read it in the next, which would cost capability. Restart-required, not + # hot-reloadable — swapping workspaces at runtime would break exactly the + # cross-command continuity this preserves. + local_working_dir: str = "/var/lib/odin-workspace" @field_validator("command_timeout_seconds") @classmethod diff --git a/src/tools/executor.py b/src/tools/executor.py index 774a533a..7777b345 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -43,6 +43,7 @@ run_ssh_command, ) from .ssh_pool import SSHConnectionPool +from .workspace import WorkspaceError, resolve_workspace log = get_logger("tools") @@ -158,6 +159,19 @@ def __init__( ) -> None: self.config = config or ToolsConfig() self._email_config = email_config + # The user-command workspace is resolved once and cached — restart- + # required by design, since swapping workspaces mid-run would break the + # cross-command continuity that makes this invisible to normal use. + # + # Resolution is LAZY (first local command) rather than at construction: + # constructing an executor must not depend on deployment having created + # the directory, or every test and fresh checkout breaks. The safety + # property is unchanged, because it binds where it matters — a local + # command never runs with an unvalidated cwd, and an unusable workspace + # raises instead of silently falling back to the inherited cwd, which + # is what would restore the 2026-07-27 hazard. + self._local_workspace: str | None = None + self._local_workspace_resolved = False self._memory_path = Path(memory_path) if memory_path else None self._browser_manager = browser_manager self._permission_manager = permission_manager @@ -299,6 +313,24 @@ def _resolve_default_host(self, user_id: str | None) -> str: hosts = list(self.config.hosts.keys()) return hosts[0] if hosts else "" + def _ensure_local_workspace(self) -> str: + """Resolve (once) the validated cwd for local user commands. + + Raises :class:`WorkspaceError` if the configured directory is unusable. + Deliberately no fallback: inheriting the process cwd is exactly the + behaviour that let a bare `rm -rf data` delete the live install. + """ + # getattr-guarded: tests and the sanctioned RFC-004 patch seam build + # executors via __new__, bypassing __init__. Lazy resolution still + # applies to those instances rather than crashing on a missing flag. + if not getattr(self, "_local_workspace_resolved", False): + self._local_workspace = str(resolve_workspace(self.config.local_working_dir)) + self._local_workspace_resolved = True + workspace = self._local_workspace + if workspace is None: # pragma: no cover - resolve_workspace raises instead + raise WorkspaceError("local workspace unresolved") + return workspace + def _ensure_process_registry(self): """Lazy-init the ProcessRegistry ON THE EXECUTOR (RFC-004 P4). @@ -309,7 +341,9 @@ def _ensure_process_registry(self): if not hasattr(self, "_process_registry"): from .process_manager import ProcessRegistry - self._process_registry = ProcessRegistry() + self._process_registry = ProcessRegistry( + workspace=self._ensure_local_workspace() + ) return self._process_registry def check_permission(self, tool_name: str, user_id: str | None) -> str | None: @@ -639,11 +673,16 @@ async def _exec_command( try: async with bh.acquire(): return await run_local_command( - command, timeout=timeout, on_output=on_output + command, + timeout=timeout, + on_output=on_output, + cwd=self._ensure_local_workspace(), ) except BulkheadFullError: return 1, "Error: subprocess bulkhead full — too many concurrent local commands" - return await run_local_command(command, timeout=timeout, on_output=on_output) + return await run_local_command( + command, timeout=timeout, on_output=on_output, cwd=self._ensure_local_workspace() + ) ssh_retry = self.config.ssh_retry ssh_kwargs: dict[str, Any] = dict( host=address, diff --git a/src/tools/process_manager.py b/src/tools/process_manager.py index 7a10bb50..32990abe 100644 --- a/src/tools/process_manager.py +++ b/src/tools/process_manager.py @@ -11,8 +11,10 @@ import time from collections import deque from dataclasses import dataclass, field +from pathlib import Path from ..odin_log import get_logger +from .workspace import workspace_env log = get_logger("process_manager") @@ -43,8 +45,13 @@ class ProcessInfo: class ProcessRegistry: """Registry for background processes with full lifecycle management.""" - def __init__(self) -> None: + def __init__(self, workspace: str | None = None) -> None: self._processes: dict[int, ProcessInfo] = {} + # Background starts share the foreground workspace. Without this, + # `manage_process start` stays an alternate route to the 2026-07-27 + # incident: a bare relative path resolving against the live install + # (29 historical background starts had no explicit cd). + self._workspace = workspace # ------------------------------------------------------------------ # Public API @@ -69,12 +76,15 @@ async def start(self, host: str, command: str, timeout: int = 300) -> str: # start_new_session puts the shell at the head of its own process # group, so kill()/shutdown() can take out descendants # (`sh -c 'x & ...'`) instead of just the shell leader. + env = workspace_env(Path(self._workspace)) if self._workspace else None proc = await asyncio.create_subprocess_shell( command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, stdin=asyncio.subprocess.PIPE, start_new_session=True, + cwd=self._workspace, + env=env, ) except Exception as e: return f"Failed to start process: {e}" @@ -155,9 +165,7 @@ async def kill(self, pid: int) -> str: # would otherwise outlive an in-place restart's exec). # owned_pgid: start() spawns with start_new_session=True, so # the group stays signallable after the leader exits. - await terminate_process_tree( - info.process, grace=5.0, owned_pgid=info.process.pid - ) + await terminate_process_tree(info.process, grace=5.0, owned_pgid=info.process.pid) info.status = "failed" info.exit_code = -9 log.info("Killed process PID %d", pid) @@ -272,9 +280,7 @@ async def _read_output(self, info: ProcessInfo) -> None: from ..tools.ssh import terminate_process_tree try: - await terminate_process_tree( - info.process, grace=2.0, owned_pgid=info.process.pid - ) + await terminate_process_tree(info.process, grace=2.0, owned_pgid=info.process.pid) except Exception: log.debug("group reap after PID %d exit failed", info.pid, exc_info=True) diff --git a/src/tools/ssh.py b/src/tools/ssh.py index 95a9ee02..13f6ee6a 100644 --- a/src/tools/ssh.py +++ b/src/tools/ssh.py @@ -4,10 +4,12 @@ import os import signal from collections.abc import Awaitable, Callable +from pathlib import Path from typing import TYPE_CHECKING from ..llm.backoff import compute_backoff from ..odin_log import get_logger +from .workspace import workspace_env if TYPE_CHECKING: from .ssh_pool import SSHConnectionPool @@ -115,9 +117,7 @@ async def terminate_process_tree( # (``pgid == proc.pid``) and (b) a safe, foreign target. A pgid that is # broadcast/own-group — however it arose (a bad owned_pgid, a recycled pid) — # is dropped here and cleanup falls back to signalling the child alone. - owns_group = ( - pgid is not None and pgid == proc.pid and _is_signallable_group(pgid) - ) + owns_group = pgid is not None and pgid == proc.pid and _is_signallable_group(pgid) if proc.returncode is not None and not owns_group: return # child-only cleanup and the child is already reaped @@ -207,17 +207,29 @@ async def run_local_command( command: str, timeout: int = 30, on_output: OutputCallback | None = None, + cwd: str | None = None, ) -> tuple[int, str]: """Run a command locally via subprocess. Returns (exit_code, output). Used for localhost hosts — no SSH overhead, no key needed. When *on_output* is provided, stdout is streamed line-by-line to the callback in addition to being collected for the return value. + + *cwd* is the resolved local workspace (``tools.local_working_dir``). It only + changes where a BARE RELATIVE path lands: explicit ``cd``, absolute paths, + and ``git -C `` are unaffected, so deliberately working inside the + install remains possible. ``None`` inherits the process cwd — the + pre-2026-07-27 behaviour, kept only for internal callers that legitimately + depend on the application directory. """ log.info("Local exec: %s", command) proc: asyncio.subprocess.Process | None = None try: + # PWD/OLDPWD are normalized alongside cwd: cwd= alone leaves an + # inherited OLDPWD pointing at the install, so a bare `cd -` would walk + # right back into it (review finding, 2026-07-27). + env = workspace_env(Path(cwd)) if cwd else None # start_new_session puts the shell at the head of its own process # group, so timeout/cancellation cleanup can take out descendants # (`sh -c 'x & ...'`) instead of just the shell leader. @@ -226,11 +238,11 @@ async def run_local_command( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, start_new_session=True, + cwd=cwd, + env=env, ) if on_output is not None: - return await _read_lines_with_callback( - proc, timeout, on_output, owned_pgid=proc.pid - ) + return await _read_lines_with_callback(proc, timeout, on_output, owned_pgid=proc.pid) stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) output = stdout.decode("utf-8", errors="replace") return proc.returncode or 0, _truncate_output(output) diff --git a/src/tools/workspace.py b/src/tools/workspace.py new file mode 100644 index 00000000..5228c458 --- /dev/null +++ b/src/tools/workspace.py @@ -0,0 +1,105 @@ +"""Resolution and validation of the local command workspace. + +User-command execution (``run_command``, ``run_script``, ``manage_process``) +used to inherit the service's working directory, which is the install root. +A bare relative path in a command therefore resolved against the live install: +on 2026-07-27 an AE2 jar whose internal layout is ``data/`` was extracted and +then cleaned up with ``rm -rf data``, deleting ``/opt/odin/data``. + +This module resolves ONE validated directory that those subprocesses run in +instead. Scope is deliberately narrow — see :func:`resolve_workspace`: + +- It changes where a *bare relative path* lands. Nothing else. +- Explicit ``cd`` still works. Absolute paths still work. ``git -C `` + still works. Deliberately operating inside the install remains possible. +- It does NOT confine the process. ``..``, an absolute path, a symlink placed + inside the workspace, or archive path traversal can still reach the install. + Preventing those needs filesystem confinement and would cost capability; + what this closes is the accidental basename collision that actually bit. + +Validation fails CLOSED. If the configured directory is missing, is a symlink, +sits inside the install or live-data root, or is not privately owned, callers +raise instead of silently falling back to the inherited cwd — a silent +fallback would restore the exact hazard this exists to remove. +""" + +from __future__ import annotations + +import os +from pathlib import Path + + +class WorkspaceError(RuntimeError): + """The configured local workspace is unusable. Never fall back to cwd.""" + + +def _canonical(path: str | os.PathLike[str]) -> Path: + return Path(path).expanduser().resolve(strict=False) + + +def resolve_workspace( + configured: str, + *, + install_root: str | os.PathLike[str] = "/opt/odin", + data_root: str | os.PathLike[str] | None = None, + require_private: bool = True, +) -> Path: + """Validate ``configured`` and return the canonical workspace path. + + Raises :class:`WorkspaceError` on anything suspect. The checks exist + because a workspace that lives inside the install — or that is a symlink + pointing back into it — would reopen the collision this closes. + """ + if not configured or not str(configured).strip(): + raise WorkspaceError("tools.local_working_dir is empty") + + raw = Path(str(configured).strip()).expanduser() + if not raw.is_absolute(): + raise WorkspaceError(f"local_working_dir must be absolute: {configured!r}") + + # A symlink is rejected before resolution: the link could be repointed + # later, which would silently move every command's cwd. + if raw.is_symlink(): + raise WorkspaceError(f"local_working_dir must not be a symlink: {raw}") + + workspace = _canonical(raw) + if not workspace.exists(): + raise WorkspaceError(f"local_working_dir does not exist: {workspace}") + if not workspace.is_dir(): + raise WorkspaceError(f"local_working_dir is not a directory: {workspace}") + + install = _canonical(install_root) + roots = [install] + if data_root: + roots.append(_canonical(data_root)) + else: + roots.append(install / "data") + for root in roots: + if workspace == root or root in workspace.parents: + raise WorkspaceError(f"local_working_dir must live outside {root}: {workspace}") + + if not os.access(workspace, os.W_OK | os.X_OK): + raise WorkspaceError(f"local_working_dir is not writable/searchable: {workspace}") + + if require_private: + mode = workspace.stat().st_mode + if mode & 0o077: + raise WorkspaceError( + f"local_working_dir must not be group/world accessible: {workspace} " + f"(mode {mode & 0o777:o})" + ) + + return workspace + + +def workspace_env(workspace: Path, base: dict[str, str] | None = None) -> dict[str, str]: + """Environment for a child shell started in ``workspace``. + + ``cwd=`` sets ``PWD`` correctly, but an inherited ``OLDPWD`` pointing at the + install means a bare ``cd -`` walks straight back into it — a small but real + hole in the boundary (Odin's review, 2026-07-27). Both are normalized. + """ + env = dict(os.environ if base is None else base) + env["PWD"] = str(workspace) + env["OLDPWD"] = str(workspace) + return env diff --git a/tests/conftest.py b/tests/conftest.py index 8a7fe5a6..b17dfc2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -208,3 +208,27 @@ def _reset_restart_intent(): yield restart.reset() + + +@pytest.fixture(autouse=True, scope="session") +def _test_local_workspace(tmp_path_factory): + """Provision a valid local command workspace for the whole test session. + + ToolExecutor resolves ``tools.local_working_dir`` fail-closed before running + any local command — there is deliberately no fallback to the inherited cwd, + since that inheritance is what let a bare `rm -rf data` delete the live + install on 2026-07-27. Production provisions the directory at deploy time; + tests provision this one, so executor construction never depends on the + host having been deployed to. + """ + from src.config.schema import ToolsConfig + + workspace = tmp_path_factory.mktemp("odin-test-workspace") + workspace.chmod(0o700) + field = ToolsConfig.model_fields["local_working_dir"] + original = field.default + field.default = str(workspace) + ToolsConfig.model_rebuild(force=True) + yield workspace + field.default = original + ToolsConfig.model_rebuild(force=True) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py new file mode 100644 index 00000000..f7712b91 --- /dev/null +++ b/tests/test_local_workspace.py @@ -0,0 +1,300 @@ +"""Local command workspace: the 2026-07-27 wipe mechanism, closed. + +The incident: an AE2 jar whose internal layout is ``data/`` was extracted and +then cleaned up with ``rm -rf data``. Local user commands inherited the +service's working directory (the install root), so that relative path resolved +to ``/opt/odin/data`` and deleted Odin's live state. + +Aaron's acceptance bar is explicit: the *exact* three-command workflow must +still succeed — extract, read, clean up — while the install is untouched. So +the headline test replays those three commands for real, in a pytest-owned +temporary fixture. + +Safety of that replay is structural, not conventional. The fixture arranges +things so a REGRESSION is what gets destroyed: the test process's own cwd is +set to the fake install, so if the plumbing ever stopped passing ``cwd=``, the +relative ``rm -rf data`` would delete the fixture's sentinel and the assertion +would fail loudly. Every path involved is under ``tmp_path``, and the test +asserts that before any deletion runs. +""" + +from __future__ import annotations + +import zipfile +from pathlib import Path + +import pytest + +from src.tools.process_manager import ProcessRegistry +from src.tools.ssh import run_local_command +from src.tools.workspace import WorkspaceError, resolve_workspace, workspace_env + + +@pytest.fixture +def fake_install(tmp_path: Path) -> Path: + """A stand-in for /opt/odin, complete with the data dir that got wiped.""" + install = tmp_path / "fake-install" + (install / "data").mkdir(parents=True) + (install / "data" / "sentinel").write_text("live odin state", encoding="utf-8") + return install + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + ws = tmp_path / "workspace" + ws.mkdir(mode=0o700) + return ws + + +@pytest.fixture +def ae2_jar(tmp_path: Path) -> Path: + """A jar shaped like the real one: its internal layout starts with data/.""" + jar = tmp_path / "appliedenergistics2-19.2.17.jar" + with zipfile.ZipFile(jar, "w") as zf: + zf.writestr( + "data/ae2/recipe/network/blocks/pattern_providers_interface.json", + '{"type": "ae2:shaped"}', + ) + zf.writestr("data/ae2/recipe/network/crafting/cpu_crafting_unit.json", "{}") + return jar + + +# --- validation: fail closed ------------------------------------------------- + + +def test_valid_workspace_resolves(workspace: Path, fake_install: Path) -> None: + resolved = resolve_workspace(str(workspace), install_root=str(fake_install)) + assert resolved == workspace.resolve() + + +@pytest.mark.parametrize("value", ["", " ", "relative/path", "~/tilde-but-relative"]) +def test_rejects_non_absolute_or_empty(value: str, fake_install: Path) -> None: + with pytest.raises(WorkspaceError): + resolve_workspace(value, install_root=str(fake_install)) + + +def test_rejects_missing_directory(tmp_path: Path, fake_install: Path) -> None: + with pytest.raises(WorkspaceError, match="does not exist"): + resolve_workspace(str(tmp_path / "nope"), install_root=str(fake_install)) + + +def test_rejects_file_masquerading_as_directory(tmp_path: Path, fake_install: Path) -> None: + target = tmp_path / "afile" + target.write_text("x", encoding="utf-8") + with pytest.raises(WorkspaceError, match="not a directory"): + resolve_workspace(str(target), install_root=str(fake_install)) + + +def test_rejects_symlink(tmp_path: Path, workspace: Path, fake_install: Path) -> None: + """A symlink could be repointed later, silently moving every command's cwd.""" + link = tmp_path / "link-to-workspace" + link.symlink_to(workspace) + with pytest.raises(WorkspaceError, match="symlink"): + resolve_workspace(str(link), install_root=str(fake_install)) + + +def test_rejects_workspace_inside_install(fake_install: Path) -> None: + """The whole point is to be outside the install — inside would reopen it.""" + inside = fake_install / "scratch" + inside.mkdir(mode=0o700) + with pytest.raises(WorkspaceError, match="outside"): + resolve_workspace(str(inside), install_root=str(fake_install)) + + +def test_rejects_workspace_inside_data_root(tmp_path: Path, fake_install: Path) -> None: + data_ws = fake_install / "data" / "ws" + data_ws.mkdir(mode=0o700) + with pytest.raises(WorkspaceError, match="outside"): + resolve_workspace( + str(data_ws), install_root=str(fake_install), data_root=str(fake_install / "data") + ) + + +def test_rejects_group_or_world_accessible(tmp_path: Path, fake_install: Path) -> None: + loose = tmp_path / "loose" + loose.mkdir(mode=0o755) + with pytest.raises(WorkspaceError, match="group/world"): + resolve_workspace(str(loose), install_root=str(fake_install)) + + +def test_env_normalizes_pwd_and_oldpwd(workspace: Path) -> None: + """cwd= alone leaves an inherited OLDPWD pointing at the install, so a bare + `cd -` walks straight back in. Both must be normalized.""" + env = workspace_env(workspace, base={"OLDPWD": "/opt/odin", "PWD": "/opt/odin"}) + assert env["PWD"] == str(workspace) + assert env["OLDPWD"] == str(workspace) + + +# --- the acceptance bar: Aaron's exact workflow ------------------------------ + + +@pytest.mark.parametrize("streamed", [False, True], ids=["buffered", "streaming"]) +async def test_incident_workflow_succeeds_without_touching_install( + fake_install: Path, + workspace: Path, + ae2_jar: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + streamed: bool, +) -> None: + """The 2026-07-27 command sequence, replayed as three separate commands. + + Extract, read, clean up — all must SUCCEED (Aaron's bar: don't prevent him + from doing what he was trying to do), while the install's data survives. + """ + # The test process stands in the fake install: if cwd= plumbing regressed, + # the relative rm below could only destroy this fixture, never real data. + monkeypatch.chdir(fake_install) + assert Path.cwd() == fake_install.resolve() + assert tmp_path in fake_install.parents or fake_install.is_relative_to(tmp_path) + + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + collected: list[str] = [] + cb = (lambda line: collected.append(line)) if streamed else None + + # 1. extract, exactly as he did — relative `data/...` out of the jar + code, out = await run_local_command( + f"jar xf {ae2_jar} data/ae2/recipe/network/blocks/pattern_providers_interface.json" + f" || unzip -o {ae2_jar} 'data/ae2/recipe/network/blocks/*' > /dev/null", + timeout=30, + on_output=cb, + cwd=ws, + ) + assert code == 0, out + extracted = workspace / "data/ae2/recipe/network/blocks/pattern_providers_interface.json" + assert extracted.exists(), "extraction must land in the workspace" + assert not (fake_install / "data" / "ae2").exists(), "must not touch the install" + + # 2. read it back — the actual research task + code, out = await run_local_command( + "cat data/ae2/recipe/network/blocks/pattern_providers_interface.json", + timeout=30, + cwd=ws, + ) + assert code == 0 + assert "ae2:shaped" in out + + # 3. clean up after himself — THE command that caused the incident. + # Bounded by construction: cwd is asserted inside tmp_path above. + assert Path(ws).is_relative_to(tmp_path) + code, _ = await run_local_command("rm -rf data", timeout=30, cwd=ws) + assert code == 0 + + assert not (workspace / "data").exists(), "cleanup must work in the workspace" + # The whole point: + assert (fake_install / "data" / "sentinel").read_text(encoding="utf-8") == "live odin state" + + +async def test_explicit_cd_into_install_still_works( + fake_install: Path, + workspace: Path, +) -> None: + """Aaron's second bar: deliberately operating inside his own install — for + reviews, tests, greps — must remain possible. The default moves; explicit + intent is untouched.""" + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + + code, out = await run_local_command(f"cd {fake_install} && pwd", timeout=30, cwd=ws) + assert code == 0 + assert out.strip() == str(fake_install) + + # and reading install files by relative path after an explicit cd + code, out = await run_local_command( + f"cd {fake_install} && cat data/sentinel", timeout=30, cwd=ws + ) + assert code == 0 + assert "live odin state" in out + + +async def test_absolute_paths_are_unaffected(fake_install: Path, workspace: Path) -> None: + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + code, out = await run_local_command(f"cat {fake_install}/data/sentinel", timeout=30, cwd=ws) + assert code == 0 + assert "live odin state" in out + + +async def test_relative_paths_persist_across_commands( + fake_install: Path, + workspace: Path, +) -> None: + """The workspace is STABLE, not per-command: a file written relatively in + one command must still be there for the next. A fresh temp dir per command + would silently break two-step workflows.""" + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + code, _ = await run_local_command("echo carried > note.txt", timeout=30, cwd=ws) + assert code == 0 + code, out = await run_local_command("cat note.txt", timeout=30, cwd=ws) + assert code == 0 + assert "carried" in out + + +async def test_cd_dash_cannot_return_to_the_install( + fake_install: Path, + workspace: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """OLDPWD normalization: with an inherited OLDPWD the shell's `cd -` would + hop straight back into the install.""" + monkeypatch.setenv("OLDPWD", str(fake_install)) + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + code, out = await run_local_command("cd - > /dev/null 2>&1; pwd", timeout=30, cwd=ws) + assert code == 0 + assert str(fake_install) not in out + + +# --- background processes: the alternate route ------------------------------- + + +async def test_background_process_uses_the_workspace( + fake_install: Path, + workspace: Path, +) -> None: + """`manage_process start` must not remain a second path to the incident.""" + ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + registry = ProcessRegistry(workspace=ws) + result = await registry.start("localhost", "pwd; sleep 0.2") + assert "Started" in result or "PID" in result + import asyncio + + await asyncio.sleep(0.6) + output = "".join("".join(info.output_buffer) for info in registry._processes.values()) + assert str(workspace) in output + assert str(fake_install) not in output + for info in registry._processes.values(): + if info.status == "running": + await registry.kill(info.pid) + + +def test_registry_without_workspace_inherits_cwd() -> None: + """Internal callers that legitimately need the application directory keep + the old behaviour; the scoping is explicit rather than accidental.""" + assert ProcessRegistry()._workspace is None + + +# --- scoping: remote execution is untouched ---------------------------------- + + +def test_remote_execution_signature_has_no_workspace() -> None: + """The workspace is a LOCAL user-command concern. Remote SSH runs in the + remote account's own home and must not be rewritten by this change.""" + import inspect + + from src.tools.ssh import run_ssh_command + + assert "cwd" not in inspect.signature(run_ssh_command).parameters + + +def test_legacy_dag_surfaces_are_classified() -> None: + """src/odin/tools/{shell,process}.py also inherit cwd today. They are NOT on + the Discord execution path, but if either is ever wired in without a + workspace it silently reopens the 2026-07-27 mechanism. This pin exists so + that wiring cannot happen quietly.""" + repo = Path(__file__).resolve().parents[1] + for legacy in (repo / "src/odin/tools/shell.py", repo / "src/odin/tools/process.py"): + if not legacy.exists(): + continue + source = legacy.read_text(encoding="utf-8") + assert "create_subprocess_shell" in source + executor = (repo / "src/tools/executor.py").read_text(encoding="utf-8") + assert "src.odin.tools.shell" not in executor + assert "from ..odin.tools" not in executor diff --git a/tests/test_output_streamer.py b/tests/test_output_streamer.py index 5d069c2c..5b487548 100644 --- a/tests/test_output_streamer.py +++ b/tests/test_output_streamer.py @@ -11,6 +11,18 @@ _ActiveStream, ) + +def _session_workspace() -> str: + """A valid local-command workspace for tests that stub config with a mock. + + ToolExecutor refuses to run a local command against an unvalidated cwd — + deliberately, since inheriting the install directory is what allowed the + 2026-07-27 wipe — so a mocked config still needs a real directory here. + """ + from src.config.schema import ToolsConfig + + return ToolsConfig().local_working_dir + # --------------------------------------------------------------------------- # StreamChunk # --------------------------------------------------------------------------- @@ -578,6 +590,8 @@ async def test_local_passes_on_output(self): executor = ToolExecutor.__new__(ToolExecutor) executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.bulkheads = MagicMock() executor.bulkheads.get.return_value = None @@ -601,6 +615,8 @@ async def test_ssh_passes_on_output(self): executor = ToolExecutor.__new__(ToolExecutor) executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.ssh_key_path = "/key" executor.config.ssh_known_hosts_path = "/known" @@ -627,6 +643,8 @@ async def test_no_on_output_default(self): executor = ToolExecutor.__new__(ToolExecutor) executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.bulkheads = MagicMock() executor.bulkheads.get.return_value = None @@ -654,6 +672,8 @@ async def test_streaming_enabled_creates_callback(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.hosts = {"myhost": MagicMock( address="127.0.0.1", @@ -692,6 +712,8 @@ async def test_streaming_disabled_no_callback(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.hosts = {"myhost": MagicMock( address="127.0.0.1", @@ -726,6 +748,8 @@ async def test_no_streamer_works(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.hosts = {"myhost": MagicMock( address="127.0.0.1", @@ -756,6 +780,8 @@ async def test_unknown_host_calls_finish(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.hosts = {} executor._branch_freshness_enabled = False executor._host_access = None @@ -784,6 +810,8 @@ async def test_streaming_enabled(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.hosts = {"myhost": MagicMock( address="127.0.0.1", @@ -821,6 +849,8 @@ async def test_streaming_disabled(self): executor = ToolExecutor() executor.config = MagicMock() + # Real path: the workspace is validated fail-closed (no cwd fallback). + executor.config.local_working_dir = _session_workspace() executor.config.command_timeout_seconds = 30 executor.config.hosts = {"myhost": MagicMock( address="127.0.0.1", From 9aa49e549d703468ea87ee197c06b69e31096435 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 13:17:44 -0400 Subject: [PATCH 02/21] =?UTF-8?q?fix:=20address=20PR=20#239=20review=20?= =?UTF-8?q?=E2=80=94=20real=20roots,=20provisioning,=20ownership,=20execut?= =?UTF-8?q?or-level=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four blockers were correct. The fourth was the serious one. 1. PROTECTED ROOTS were hardcoded to /opt/odin, so the validator protected nothing under Docker (install root /app) or in a source checkout, and a string-joined `install / "data"` missed that packaged /opt/odin/data is a SYMLINK to /var/lib/odin — a workspace inside the real live-data directory passed. The overlap check was also one-directional. The executor now derives roots from the RUNNING application (package location + the actual configured data paths, canonicalized), and overlap is rejected in BOTH directions. 2. PROVISIONING did not exist on any supported install path, so the fail-closed default would cost local-command capability on first use. Added to Debian postinstall (0700 odin:odin), the Dockerfile, and docker-compose (persisted, so cross-command continuity survives restarts). A test asserts every install path declares it, since the session fixture would otherwise keep masking the gap. 3. OWNERSHIP was never validated and mode 0300 passed — private and writable but not readable, which breaks ordinary use. Now: owner uid must match the execution identity, mode must be exactly 0700, and R_OK|W_OK|X_OK is verified. 4. THE HEADLINE TESTS BYPASSED THE SEAM THAT CAUSED THE INCIDENT. They called run_local_command(cwd=...) directly, proving that function honors a cwd but NOT that ToolExecutor supplies one — deleting the executor's cwd argument would have reopened the wipe with the suite still green. My mutation testing had hit the wrong layer entirely. The replay now runs through ToolExecutor._exec_command, background coverage asserts the registry gets its workspace FROM the executor, and a further pin proves the executor actually passes its protected roots (without it, dropping that argument left every other test green). Mutation-verified at the executor seam: removing the executor's cwd, removing the registry workspace, and removing protected_roots each now fail. Also added, per the review: symlinked live-data alias, non-/opt install root, bidirectional overlap, and wrong-owner cases. Gates: ruff clean, mypy clean (231 files), 7537 passed / 4 skipped. --- Dockerfile | 8 ++ docker-compose.yml | 2 + packaging/postinstall.sh | 10 ++ src/tools/executor.py | 33 ++++- src/tools/workspace.py | 99 ++++++++----- tests/test_local_workspace.py | 253 +++++++++++++++++++++++++++++++--- 6 files changed, 351 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2642f0c9..220c39c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,14 @@ RUN pip install --no-cache-dir . # Copy application source COPY src/ src/ COPY ui/ ui/ + +# Working directory for local user commands (tools.local_working_dir). +# Deliberately OUTSIDE the install root (/app here) and outside the data dir: +# local commands run here so a bare relative path — e.g. cleaning up after +# extracting an archive whose internal layout starts with data/ — cannot +# resolve against the live install. Validation fails closed if it is missing +# or not 0700, so this must exist in the image. +RUN mkdir -p /var/lib/odin-workspace && chmod 0700 /var/lib/odin-workspace COPY config.yml . CMD ["python", "-m", "src"] diff --git a/docker-compose.yml b/docker-compose.yml index dbb08ed8..9715e917 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: volumes: - ./config.yml:/app/config.yml:ro - ./data:/app/data + # Persisted so relative files survive restarts (see Dockerfile). + - ./workspace:/var/lib/odin-workspace - ./data/context:/app/data/context:ro - ./ssh:/app/.ssh:ro extra_hosts: diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index da3189c9..205cef46 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -14,6 +14,10 @@ SERVICE_GROUP="odin" APP_DIR="/opt/odin" CONFIG_DIR="/etc/odin" DATA_DIR="/var/lib/odin" +# Working directory for local user commands (tools.local_working_dir). +# A SIBLING of DATA_DIR, never inside it: local commands run here so a bare +# relative path cannot resolve against the install or the live data. +WORKSPACE_DIR="/var/lib/odin-workspace" LOG_DIR="/var/log/odin" WEB_PORT="3000" # matches config.yml default web.port @@ -34,6 +38,7 @@ fi mkdir -p "$CONFIG_DIR" mkdir -p "$DATA_DIR"/{sessions,context,skills,search,knowledge,trajectories} mkdir -p "$LOG_DIR" +mkdir -p "$WORKSPACE_DIR" # Enable passwordless sudo for the odin user (Odin runs privileged host # operations; scope this down in /etc/sudoers.d/99-odin-passwordless if you @@ -99,6 +104,11 @@ fi # Set ownership and permissions chown -R "$SERVICE_USER:$SERVICE_GROUP" "$APP_DIR" "$DATA_DIR" "$LOG_DIR" +# Odin refuses to run local commands unless this is 0700 and owned by the +# service account — validation fails closed rather than falling back to the +# install directory, which is what allowed the 2026-07-27 data wipe. +chown "$SERVICE_USER:$SERVICE_GROUP" "$WORKSPACE_DIR" +chmod 0700 "$WORKSPACE_DIR" chown -R "$SERVICE_USER:$SERVICE_GROUP" "$CONFIG_DIR" chmod 600 "$CONFIG_DIR/.env" chown root:root /usr/lib/systemd/system/odin.service diff --git a/src/tools/executor.py b/src/tools/executor.py index 7777b345..a07f5ceb 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -313,6 +313,32 @@ def _resolve_default_host(self, user_id: str | None) -> str: hosts = list(self.config.hosts.keys()) return hosts[0] if hosts else "" + def _protected_roots(self) -> list[str]: + """Roots the local workspace must not overlap, derived from the RUNNING + application rather than assumed (PR #239 review). + + A hardcoded ``/opt/odin`` is wrong under Docker (install root ``/app``) + and for source checkouts, and packaged ``/opt/odin/data`` is a symlink + to ``/var/lib/odin`` — so the live-data root is taken from the actual + configured data paths and canonicalized, not string-joined. + """ + roots: list[str] = [] + # Install root: the package's own location (…/src/tools/executor.py). + roots.append(str(Path(__file__).resolve().parents[2])) + # Live-data roots: wherever the configured data paths actually live, + # following symlinks. audit_log_path/trajectory_path are the two + # declared data-dir members on ToolsConfig. + for configured in ( + getattr(self.config, "audit_log_path", None), + getattr(self.config, "trajectory_path", None), + ): + if not isinstance(configured, str) or not configured.strip(): + continue + candidate = Path(configured.strip()) + data_dir = candidate.parent if candidate.suffix else candidate + roots.append(str(data_dir.resolve())) + return roots + def _ensure_local_workspace(self) -> str: """Resolve (once) the validated cwd for local user commands. @@ -324,7 +350,12 @@ def _ensure_local_workspace(self) -> str: # executors via __new__, bypassing __init__. Lazy resolution still # applies to those instances rather than crashing on a missing flag. if not getattr(self, "_local_workspace_resolved", False): - self._local_workspace = str(resolve_workspace(self.config.local_working_dir)) + self._local_workspace = str( + resolve_workspace( + self.config.local_working_dir, + protected_roots=self._protected_roots(), + ) + ) self._local_workspace_resolved = True workspace = self._local_workspace if workspace is None: # pragma: no cover - resolve_workspace raises instead diff --git a/src/tools/workspace.py b/src/tools/workspace.py index 5228c458..ef23f2cd 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -7,7 +7,7 @@ then cleaned up with ``rm -rf data``, deleting ``/opt/odin/data``. This module resolves ONE validated directory that those subprocesses run in -instead. Scope is deliberately narrow — see :func:`resolve_workspace`: +instead. Scope is deliberately narrow: - It changes where a *bare relative path* lands. Nothing else. - Explicit ``cd`` still works. Absolute paths still work. ``git -C `` @@ -17,38 +17,60 @@ Preventing those needs filesystem confinement and would cost capability; what this closes is the accidental basename collision that actually bit. -Validation fails CLOSED. If the configured directory is missing, is a symlink, -sits inside the install or live-data root, or is not privately owned, callers -raise instead of silently falling back to the inherited cwd — a silent -fallback would restore the exact hazard this exists to remove. +Validation fails CLOSED — callers raise instead of silently falling back to +the inherited cwd, because that fallback is precisely the hazard. + +Protected roots are supplied by the caller and canonicalized here, never +assumed (PR #239 review): the install root is ``/app`` under Docker and an +arbitrary path for source checkouts, and packaged ``/opt/odin/data`` is a +SYMLINK to ``/var/lib/odin``, so a naive ``install / "data"`` string check +would happily accept a workspace sitting inside the real live-data directory. +Overlap is rejected in BOTH directions: a workspace that *contains* a +protected root is just as unusable as one nested inside it. """ from __future__ import annotations import os +import stat +from collections.abc import Sequence from pathlib import Path +# The accepted operational contract: a real directory owned by the execution +# identity, private, and fully usable by that owner (0700). 0300 is private +# and writable but not readable, which breaks ordinary workspace use. +REQUIRED_MODE = 0o700 + class WorkspaceError(RuntimeError): """The configured local workspace is unusable. Never fall back to cwd.""" def _canonical(path: str | os.PathLike[str]) -> Path: + """Fully resolved path, symlinks included — aliases must not smuggle a + protected root past the overlap check.""" return Path(path).expanduser().resolve(strict=False) +def _overlaps(workspace: Path, root: Path) -> bool: + """True when the two paths are the same or either contains the other.""" + return workspace == root or root in workspace.parents or workspace in root.parents + + def resolve_workspace( configured: str, *, - install_root: str | os.PathLike[str] = "/opt/odin", - data_root: str | os.PathLike[str] | None = None, - require_private: bool = True, + protected_roots: Sequence[str | os.PathLike[str]] | None = None, + require_owner: bool = True, + owner_uid: int | None = None, ) -> Path: """Validate ``configured`` and return the canonical workspace path. - Raises :class:`WorkspaceError` on anything suspect. The checks exist - because a workspace that lives inside the install — or that is a symlink - pointing back into it — would reopen the collision this closes. + ``protected_roots`` are the install root and live-data root(s) the + workspace must not overlap. Callers derive them from the running + application rather than relying on a hardcoded ``/opt/odin``, which is + wrong under Docker (``/app``), for source checkouts, and for packaged + installs where ``data`` is a symlink elsewhere. """ if not configured or not str(configured).strip(): raise WorkspaceError("tools.local_working_dir is empty") @@ -57,38 +79,49 @@ def resolve_workspace( if not raw.is_absolute(): raise WorkspaceError(f"local_working_dir must be absolute: {configured!r}") - # A symlink is rejected before resolution: the link could be repointed - # later, which would silently move every command's cwd. + # Rejected before resolution: a symlink could be repointed later, silently + # moving every command's working directory. if raw.is_symlink(): raise WorkspaceError(f"local_working_dir must not be a symlink: {raw}") workspace = _canonical(raw) if not workspace.exists(): - raise WorkspaceError(f"local_working_dir does not exist: {workspace}") + raise WorkspaceError( + f"local_working_dir does not exist: {workspace} " + "(deployment must create it 0700, owned by the service account)" + ) if not workspace.is_dir(): raise WorkspaceError(f"local_working_dir is not a directory: {workspace}") - install = _canonical(install_root) - roots = [install] - if data_root: - roots.append(_canonical(data_root)) - else: - roots.append(install / "data") - for root in roots: - if workspace == root or root in workspace.parents: - raise WorkspaceError(f"local_working_dir must live outside {root}: {workspace}") - - if not os.access(workspace, os.W_OK | os.X_OK): - raise WorkspaceError(f"local_working_dir is not writable/searchable: {workspace}") - - if require_private: - mode = workspace.stat().st_mode - if mode & 0o077: + for root in protected_roots or []: + canonical_root = _canonical(root) + if _overlaps(workspace, canonical_root): + raise WorkspaceError( + f"local_working_dir must not overlap {canonical_root}: {workspace}" + ) + + info = workspace.stat() + + if require_owner: + expected = os.getuid() if owner_uid is None else owner_uid + if info.st_uid != expected: raise WorkspaceError( - f"local_working_dir must not be group/world accessible: {workspace} " - f"(mode {mode & 0o777:o})" + f"local_working_dir must be owned by uid {expected}: " + f"{workspace} is owned by uid {info.st_uid}" ) + mode = stat.S_IMODE(info.st_mode) + if mode != REQUIRED_MODE: + # Both halves matter: group/world bits leak a workspace that may hold + # command output, and an owner mode like 0300 is private but not + # readable, which breaks ordinary use. + raise WorkspaceError( + f"local_working_dir must be mode {REQUIRED_MODE:o}: {workspace} is {mode:o}" + ) + + if not os.access(workspace, os.R_OK | os.W_OK | os.X_OK): + raise WorkspaceError(f"local_working_dir is not fully usable: {workspace}") + return workspace @@ -97,7 +130,7 @@ def workspace_env(workspace: Path, base: dict[str, str] | None = None) -> dict[s ``cwd=`` sets ``PWD`` correctly, but an inherited ``OLDPWD`` pointing at the install means a bare ``cd -`` walks straight back into it — a small but real - hole in the boundary (Odin's review, 2026-07-27). Both are normalized. + hole in the boundary (PR #239 review). Both are normalized. """ env = dict(os.environ if base is None else base) env["PWD"] = str(workspace) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index f7712b91..7f7b4004 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -20,6 +20,7 @@ from __future__ import annotations +import os import zipfile from pathlib import Path @@ -63,26 +64,26 @@ def ae2_jar(tmp_path: Path) -> Path: def test_valid_workspace_resolves(workspace: Path, fake_install: Path) -> None: - resolved = resolve_workspace(str(workspace), install_root=str(fake_install)) + resolved = resolve_workspace(str(workspace), protected_roots=[str(fake_install)]) assert resolved == workspace.resolve() @pytest.mark.parametrize("value", ["", " ", "relative/path", "~/tilde-but-relative"]) def test_rejects_non_absolute_or_empty(value: str, fake_install: Path) -> None: with pytest.raises(WorkspaceError): - resolve_workspace(value, install_root=str(fake_install)) + resolve_workspace(value, protected_roots=[str(fake_install)]) def test_rejects_missing_directory(tmp_path: Path, fake_install: Path) -> None: with pytest.raises(WorkspaceError, match="does not exist"): - resolve_workspace(str(tmp_path / "nope"), install_root=str(fake_install)) + resolve_workspace(str(tmp_path / "nope"), protected_roots=[str(fake_install)]) def test_rejects_file_masquerading_as_directory(tmp_path: Path, fake_install: Path) -> None: target = tmp_path / "afile" target.write_text("x", encoding="utf-8") with pytest.raises(WorkspaceError, match="not a directory"): - resolve_workspace(str(target), install_root=str(fake_install)) + resolve_workspace(str(target), protected_roots=[str(fake_install)]) def test_rejects_symlink(tmp_path: Path, workspace: Path, fake_install: Path) -> None: @@ -90,31 +91,38 @@ def test_rejects_symlink(tmp_path: Path, workspace: Path, fake_install: Path) -> link = tmp_path / "link-to-workspace" link.symlink_to(workspace) with pytest.raises(WorkspaceError, match="symlink"): - resolve_workspace(str(link), install_root=str(fake_install)) + resolve_workspace(str(link), protected_roots=[str(fake_install)]) def test_rejects_workspace_inside_install(fake_install: Path) -> None: """The whole point is to be outside the install — inside would reopen it.""" inside = fake_install / "scratch" inside.mkdir(mode=0o700) - with pytest.raises(WorkspaceError, match="outside"): - resolve_workspace(str(inside), install_root=str(fake_install)) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(inside), protected_roots=[str(fake_install)]) def test_rejects_workspace_inside_data_root(tmp_path: Path, fake_install: Path) -> None: data_ws = fake_install / "data" / "ws" data_ws.mkdir(mode=0o700) - with pytest.raises(WorkspaceError, match="outside"): + with pytest.raises(WorkspaceError, match="overlap"): resolve_workspace( - str(data_ws), install_root=str(fake_install), data_root=str(fake_install / "data") + str(data_ws), protected_roots=[str(fake_install), str(fake_install / "data")] ) -def test_rejects_group_or_world_accessible(tmp_path: Path, fake_install: Path) -> None: - loose = tmp_path / "loose" - loose.mkdir(mode=0o755) - with pytest.raises(WorkspaceError, match="group/world"): - resolve_workspace(str(loose), install_root=str(fake_install)) +@pytest.mark.parametrize("mode", [0o755, 0o750, 0o770, 0o300, 0o500, 0o600]) +def test_rejects_any_mode_other_than_0700(mode: int, tmp_path: Path, fake_install: Path) -> None: + """Group/world bits leak a workspace that holds command output; an + owner-only mode like 0300 is private but not readable, which breaks + ordinary use. The contract is exactly 0700 (PR #239 review).""" + target = tmp_path / f"mode-{mode:o}" + target.mkdir(mode=mode) + try: + with pytest.raises(WorkspaceError, match="mode"): + resolve_workspace(str(target), protected_roots=[str(fake_install)]) + finally: + target.chmod(0o700) def test_env_normalizes_pwd_and_oldpwd(workspace: Path) -> None: @@ -148,7 +156,7 @@ async def test_incident_workflow_succeeds_without_touching_install( assert Path.cwd() == fake_install.resolve() assert tmp_path in fake_install.parents or fake_install.is_relative_to(tmp_path) - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) collected: list[str] = [] cb = (lambda line: collected.append(line)) if streamed else None @@ -192,7 +200,7 @@ async def test_explicit_cd_into_install_still_works( """Aaron's second bar: deliberately operating inside his own install — for reviews, tests, greps — must remain possible. The default moves; explicit intent is untouched.""" - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) code, out = await run_local_command(f"cd {fake_install} && pwd", timeout=30, cwd=ws) assert code == 0 @@ -207,7 +215,7 @@ async def test_explicit_cd_into_install_still_works( async def test_absolute_paths_are_unaffected(fake_install: Path, workspace: Path) -> None: - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) code, out = await run_local_command(f"cat {fake_install}/data/sentinel", timeout=30, cwd=ws) assert code == 0 assert "live odin state" in out @@ -220,7 +228,7 @@ async def test_relative_paths_persist_across_commands( """The workspace is STABLE, not per-command: a file written relatively in one command must still be there for the next. A fresh temp dir per command would silently break two-step workflows.""" - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) code, _ = await run_local_command("echo carried > note.txt", timeout=30, cwd=ws) assert code == 0 code, out = await run_local_command("cat note.txt", timeout=30, cwd=ws) @@ -236,7 +244,7 @@ async def test_cd_dash_cannot_return_to_the_install( """OLDPWD normalization: with an inherited OLDPWD the shell's `cd -` would hop straight back into the install.""" monkeypatch.setenv("OLDPWD", str(fake_install)) - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) code, out = await run_local_command("cd - > /dev/null 2>&1; pwd", timeout=30, cwd=ws) assert code == 0 assert str(fake_install) not in out @@ -250,7 +258,7 @@ async def test_background_process_uses_the_workspace( workspace: Path, ) -> None: """`manage_process start` must not remain a second path to the incident.""" - ws = str(resolve_workspace(str(workspace), install_root=str(fake_install))) + ws = str(resolve_workspace(str(workspace), protected_roots=[str(fake_install)])) registry = ProcessRegistry(workspace=ws) result = await registry.start("localhost", "pwd; sleep 0.2") assert "Started" in result or "PID" in result @@ -298,3 +306,208 @@ def test_legacy_dag_surfaces_are_classified() -> None: executor = (repo / "src/tools/executor.py").read_text(encoding="utf-8") assert "src.odin.tools.shell" not in executor assert "from ..odin.tools" not in executor + + +# --- THE SEAM THAT ACTUALLY FAILED: through ToolExecutor -------------------- +# +# PR #239 review, blocker 4: the low-level replay proves run_local_command +# honors a supplied cwd, but NOT that the executor supplies one. Deleting +# `cwd=self._ensure_local_workspace()` from the executor would reopen the +# incident while the low-level test still passed. These exercise the real path. + + +def _executor_with_workspace(workspace: Path, protected: Path): + """A ToolExecutor whose local commands run in ``workspace``.""" + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + config = ToolsConfig(local_working_dir=str(workspace)) + executor = ToolExecutor(config=config) + # Protected roots are normally derived from the running app; point them at + # the fixture so the test exercises real validation against a fake install. + executor._protected_roots = lambda: [str(protected)] # type: ignore[method-assign] + return executor + + +async def test_executor_replays_the_incident_without_touching_the_install( + fake_install: Path, + workspace: Path, + ae2_jar: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The 2026-07-27 sequence through the EXECUTOR — the seam that failed. + + Deleting the executor's cwd argument must break this test, which the + low-level replay could not detect. + """ + monkeypatch.chdir(fake_install) # a regression can only destroy the fixture + assert Path.cwd() == fake_install.resolve() + executor = _executor_with_workspace(workspace, fake_install) + + code, out = await executor._exec_command( + "localhost", + f"jar xf {ae2_jar} data/ae2/recipe/network/blocks/pattern_providers_interface.json" + f" || unzip -o {ae2_jar} 'data/ae2/recipe/network/blocks/*' > /dev/null", + timeout=30, + ) + assert code == 0, out + assert (workspace / "data/ae2/recipe/network/blocks").exists() + assert not (fake_install / "data" / "ae2").exists() + + code, out = await executor._exec_command( + "localhost", + "cat data/ae2/recipe/network/blocks/pattern_providers_interface.json", + timeout=30, + ) + assert code == 0 and "ae2:shaped" in out + + assert Path(str(workspace)).is_relative_to(tmp_path) # bounded before deleting + code, _ = await executor._exec_command("localhost", "rm -rf data", timeout=30) + assert code == 0 + assert not (workspace / "data").exists(), "cleanup must work" + assert (fake_install / "data" / "sentinel").read_text(encoding="utf-8") == "live odin state" + + +async def test_executor_pwd_is_the_workspace(fake_install: Path, workspace: Path) -> None: + executor = _executor_with_workspace(workspace, fake_install) + code, out = await executor._exec_command("localhost", "pwd", timeout=30) + assert code == 0 + assert out.strip() == str(workspace.resolve()) + + +async def test_executor_explicit_cd_into_install_still_works( + fake_install: Path, workspace: Path +) -> None: + """Aaron's bar: deliberately working inside his own install is untouched.""" + executor = _executor_with_workspace(workspace, fake_install) + code, out = await executor._exec_command( + "localhost", f"cd {fake_install} && cat data/sentinel", timeout=30 + ) + assert code == 0 and "live odin state" in out + + +async def test_executor_background_process_uses_the_workspace( + fake_install: Path, workspace: Path +) -> None: + """manage_process must not remain an alternate route to the incident, and + the registry must get its workspace FROM the executor.""" + executor = _executor_with_workspace(workspace, fake_install) + registry = executor._ensure_process_registry() + assert registry._workspace == str(workspace.resolve()) + + await registry.start("localhost", "pwd; sleep 0.2") + import asyncio + + await asyncio.sleep(0.6) + output = "".join("".join(i.output_buffer) for i in registry._processes.values()) + assert str(workspace.resolve()) in output + assert str(fake_install) not in output + for info in registry._processes.values(): + if info.status == "running": + await registry.kill(info.pid) + + +async def test_executor_refuses_to_run_with_an_invalid_workspace( + fake_install: Path, tmp_path: Path +) -> None: + """Fail closed at the point of use: no subprocess may run against an + unvalidated cwd, and there is deliberately no fallback to the inherited + directory — that fallback is the hazard.""" + executor = _executor_with_workspace(tmp_path / "does-not-exist", fake_install) + with pytest.raises(WorkspaceError): + await executor._exec_command("localhost", "echo should-not-run", timeout=10) + + +# --- protected roots: real deployment shapes -------------------------------- + + +def test_rejects_workspace_inside_symlinked_live_data(tmp_path: Path) -> None: + """Packaged installs symlink /opt/odin/data -> /var/lib/odin. A string-joined + check would accept a workspace sitting inside the REAL data directory.""" + install = tmp_path / "opt-odin" + install.mkdir() + real_data = tmp_path / "var-lib-odin" + real_data.mkdir() + (install / "data").symlink_to(real_data) + ws = real_data / "workspace" + ws.mkdir(mode=0o700) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(ws), protected_roots=[str(install), str(install / "data")]) + + +def test_rejects_workspace_inside_non_opt_install_root(tmp_path: Path) -> None: + """Docker's install root is /app and source checkouts are arbitrary — the + validator must use the roots it is given, not a hardcoded /opt/odin.""" + app = tmp_path / "app" + (app / "sub").mkdir(parents=True) + (app / "sub").chmod(0o700) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(app / "sub"), protected_roots=[str(app)]) + + +def test_rejects_workspace_that_contains_a_protected_root(tmp_path: Path) -> None: + """Overlap is bidirectional: a workspace that CONTAINS the install is just + as unusable as one nested inside it.""" + parent = tmp_path / "parent" + install = parent / "opt-odin" + install.mkdir(parents=True) + parent.chmod(0o700) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(parent), protected_roots=[str(install)]) + + +def test_rejects_wrong_owner(tmp_path: Path, fake_install: Path) -> None: + """The contract is a directory owned by the execution identity.""" + ws = tmp_path / "otherowner" + ws.mkdir(mode=0o700) + with pytest.raises(WorkspaceError, match="owned by uid"): + resolve_workspace( + str(ws), protected_roots=[str(fake_install)], owner_uid=os.getuid() + 12345 + ) + + +# --- provisioning: the repo must ship installable --------------------------- + + +def test_every_install_path_provisions_the_workspace() -> None: + """PR #239 review, blocker 2: the default fails closed when absent, so a + deployment that never creates it loses local-command capability on first + use. Each supported install path must provision it.""" + repo = Path(__file__).resolve().parents[1] + default = "/var/lib/odin-workspace" + + postinstall = (repo / "packaging/postinstall.sh").read_text(encoding="utf-8") + assert default in postinstall + assert "chmod 0700" in postinstall + + dockerfile = (repo / "Dockerfile").read_text(encoding="utf-8") + assert default in dockerfile and "0700" in dockerfile + + compose = (repo / "docker-compose.yml").read_text(encoding="utf-8") + assert default in compose, "the workspace must persist across container restarts" + + +async def test_executor_enforces_its_protected_roots(fake_install: Path) -> None: + """The executor must PASS its derived roots to the validator, not merely + own them. Without this pin, dropping `protected_roots=` from the executor + leaves every other test green while an inside-the-install workspace is + silently accepted.""" + inside = fake_install / "scratch" + inside.mkdir(mode=0o700) + executor = _executor_with_workspace(inside, fake_install) + with pytest.raises(WorkspaceError, match="overlap"): + await executor._exec_command("localhost", "echo should-not-run", timeout=10) + + +def test_executor_derives_real_roots_from_the_running_app() -> None: + """Roots come from the running application, not a hardcoded /opt/odin — + otherwise Docker (/app) and source checkouts get no protection at all.""" + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + executor = ToolExecutor(config=ToolsConfig()) + roots = [Path(r).resolve() for r in executor._protected_roots()] + repo = Path(__file__).resolve().parents[1] + assert repo.resolve() in roots, "the actual install/package root must be protected" + assert any("data" in str(r) for r in roots), "the live-data root must be protected" From efefe9d9be7fa289f59bc87609e129b8642509fd Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 13:47:33 -0400 Subject: [PATCH 03/21] =?UTF-8?q?fix:=20address=20PR=20#239=20round=202=20?= =?UTF-8?q?=E2=80=94=20memory=5Fpath=20root,=20Docker=20volume,=20metrics,?= =?UTF-8?q?=20executable=20legacy=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four correct, including the one you flagged as your own round-one miss. 1. HIGH — memory_path was not protected. Data roots came only from audit_log_path/trajectory_path, so relocating those left a workspace beside the live memory.json accepted (you reproduced it as UNSAFE_ACCEPTED). The executor now also protects the canonical parent of the memory_path supplied by wiring. The Path.suffix heuristic is gone: each path is classified by DECLARED semantics (audit = file, trajectories = directory, memory = file), since a suffix guess misreads dotted directories and extensionless files. Both are pinned, and the memory_path pin is mutation-verified — without it, removing the protection left every other test green. 2. HIGH — fresh Docker Compose still failed closed. The bind mount masked the image's 0700 directory, and a fresh host bind is created 755 root:root. Switched to a NAMED volume, which Docker initializes from the image directory and therefore inherits the required mode. The from-source path is documented too: README now shows creating /var/lib/odin-workspace 0700, or pointing tools.local_working_dir at any private directory outside the checkout. 3. Accepted requirement delivered — workspace growth metrics. No auto-prune was always paired with observability: odin_workspace_bytes, _files, _free_bytes, _free_inodes now export through the existing collector, registered in __main__ beside the other tool sources. Verified rendering end to end. Collection never raises, so metrics cannot break a command path. 4. Legacy-surface characterization made executable, in this PR. The old test grepped two literal import spellings and would have stayed green under transitive wiring. It now imports the Discord-reachable boundary and proves the routing table contains no legacy owner, that a fresh import of the execution path does not pull src.odin.tools.{shell,process} into sys.modules, and that no executor handler domain is defined in those modules. The legacy CLI's own behaviour is untouched. Gates: ruff clean, mypy clean (231 files), 7539 passed / 4 skipped. --- README.md | 11 ++++ docker-compose.yml | 12 +++- src/__main__.py | 5 ++ src/health/metrics.py | 32 +++++++++ src/tools/executor.py | 76 ++++++++++++++++++--- tests/test_local_workspace.py | 120 ++++++++++++++++++++++++++++++---- 6 files changed, 231 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4e44f3d7..e0163cd2 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,17 @@ playwright install chromium # optional — enables browser_* tools cp .env.example .env # set DISCORD_TOKEN $EDITOR config.yml # hosts, LLM, permissions + +# Local commands run in a workspace OUTSIDE the install, so a bare relative +# path (e.g. cleaning up after extracting an archive whose internal layout +# starts with `data/`) can never resolve against your live data. It must +# exist, be private, and be owned by you — validation fails closed rather +# than silently falling back to the install directory. +sudo install -d -m 0700 -o "$USER" -g "$(id -gn)" /var/lib/odin-workspace +# …or point `tools.local_working_dir` in config.yml at any private directory +# outside the checkout, e.g.: +# mkdir -p -m 0700 ~/.local/share/odin-workspace + python -m src ``` diff --git a/docker-compose.yml b/docker-compose.yml index 9715e917..35d6d404 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,11 @@ services: volumes: - ./config.yml:/app/config.yml:ro - ./data:/app/data - # Persisted so relative files survive restarts (see Dockerfile). - - ./workspace:/var/lib/odin-workspace + # NAMED volume, not a bind mount: Docker initializes a named volume from + # the image directory, preserving the 0700 ownership the resolver + # requires. A fresh bind mount is created 755 root:root, which fails + # closed and costs local-command capability on first run (PR #239 r2). + - odin-workspace:/var/lib/odin-workspace - ./data/context:/app/data/context:ro - ./ssh:/app/.ssh:ro extra_hosts: @@ -59,3 +62,8 @@ services: capabilities: [gpu] networks: - default + +volumes: + # Local command workspace (tools.local_working_dir). Initialized from the + # image's 0700 directory so a fresh deployment satisfies the resolver. + odin-workspace: diff --git a/src/__main__.py b/src/__main__.py index f5ce35c3..c9304a18 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -32,6 +32,11 @@ def _wire_observability(health, bot, log) -> None: tool_executor = getattr(bot, "tool_executor", None) if tool_executor is not None and hasattr(tool_executor, "get_metrics"): metrics.register_source("tools", tool_executor.get_metrics) + if tool_executor is not None and hasattr(tool_executor, "get_workspace_metrics"): + # Paired with the deliberate absence of auto-pruning: growth in the + # local command workspace must be alertable rather than silently + # unbounded (PR #239 round-2 review). + metrics.register_source("workspace", tool_executor.get_workspace_metrics) cost_tracker = getattr(bot, "cost_tracker", None) if cost_tracker is not None and hasattr(cost_tracker, "get_prometheus_metrics"): diff --git a/src/health/metrics.py b/src/health/metrics.py index f28e98de..5c46b342 100644 --- a/src/health/metrics.py +++ b/src/health/metrics.py @@ -118,6 +118,38 @@ def render(self) -> str: except Exception: pass + # -- Local command workspace (no auto-prune, so growth is observable) -- + ws_source = self._sources.get("workspace") + if ws_source: + try: + ws = ws_source() or {} + if ws: + gauges = { + "bytes": ( + "odin_workspace_bytes", + "Bytes stored in the local command workspace", + ), + "files": ( + "odin_workspace_files", + "Files in the local command workspace", + ), + "free_bytes": ( + "odin_workspace_free_bytes", + "Free bytes on the workspace filesystem", + ), + "free_inodes": ( + "odin_workspace_free_inodes", + "Free inodes on the workspace filesystem", + ), + } + for key, (name, helptext) in gauges.items(): + if key in ws: + sections.append(f"# HELP {name} {helptext}") + sections.append(f"# TYPE {name} gauge") + sections.append(_format_metric(name, ws[key], include_header=False)) + except Exception: + pass + # -- Tool execution metrics -- tool_source = self._sources.get("tools") if tool_source: diff --git a/src/tools/executor.py b/src/tools/executor.py index a07f5ceb..80683530 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -3,6 +3,8 @@ import asyncio import contextvars import json +import os +import shutil import time from pathlib import Path from typing import Any @@ -325,20 +327,74 @@ def _protected_roots(self) -> list[str]: roots: list[str] = [] # Install root: the package's own location (…/src/tools/executor.py). roots.append(str(Path(__file__).resolve().parents[2])) - # Live-data roots: wherever the configured data paths actually live, - # following symlinks. audit_log_path/trajectory_path are the two - # declared data-dir members on ToolsConfig. - for configured in ( - getattr(self.config, "audit_log_path", None), - getattr(self.config, "trajectory_path", None), - ): - if not isinstance(configured, str) or not configured.strip(): + + # Live-data roots, canonicalized so a symlinked data dir (packaged + # /opt/odin/data -> /var/lib/odin) cannot be smuggled past the overlap + # check. Each path is classified by DECLARED semantics, never guessed + # from its name: a `Path.suffix` heuristic misreads dotted directories + # and extensionless files (PR #239 round-2 review). + declared: list[tuple[object, bool]] = [ + (getattr(self.config, "audit_log_path", None), True), # file + (getattr(self.config, "trajectory_path", None), False), # directory + # The live memory.json is supplied by wiring, not ToolsConfig. It is + # the most valuable file in the tree, and omitting it meant a + # workspace beside it was accepted whenever audit/trajectory paths + # were relocated (reproduced in review). + # getattr-guarded: the sanctioned __new__ patch seam builds + # executors without __init__, so this attribute may not exist. + (getattr(self, "_memory_path", None), True), # file + ] + for configured, is_file in declared: + if configured is None: + continue + text = str(configured).strip() + if not text: continue - candidate = Path(configured.strip()) - data_dir = candidate.parent if candidate.suffix else candidate + candidate = Path(text) + data_dir = candidate.parent if is_file else candidate roots.append(str(data_dir.resolve())) return roots + def get_workspace_metrics(self) -> dict[str, float]: + """Usage of the local command workspace, for Prometheus. + + The accepted design deliberately does NOT auto-prune — age-based + deletion would destroy the cross-command continuity the stable + workspace exists to provide — so growth must be observable instead, + with cleanup an explicit operator action (PR #239 round-2 review). + + Never raises: metrics collection must not be able to break a command + path, and an unresolvable workspace simply reports nothing. + """ + try: + root = Path(self._ensure_local_workspace()) + except Exception: + return {} + total_bytes = 0.0 + files = 0.0 + try: + for dirpath, _dirnames, filenames in os.walk(root, followlinks=False): + for name in filenames: + files += 1 + try: + total_bytes += os.lstat(os.path.join(dirpath, name)).st_size + except OSError: + pass + except OSError: + return {} + metrics = {"bytes": total_bytes, "files": files} + try: + usage = shutil.disk_usage(root) + metrics["free_bytes"] = float(usage.free) + except OSError: + pass + try: + stats = os.statvfs(root) + metrics["free_inodes"] = float(stats.f_favail) + except (OSError, AttributeError): + pass + return metrics + def _ensure_local_workspace(self) -> str: """Resolve (once) the validated cwd for local user commands. diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 7f7b4004..c8edf479 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -292,20 +292,57 @@ def test_remote_execution_signature_has_no_workspace() -> None: assert "cwd" not in inspect.signature(run_ssh_command).parameters -def test_legacy_dag_surfaces_are_classified() -> None: - """src/odin/tools/{shell,process}.py also inherit cwd today. They are NOT on - the Discord execution path, but if either is ever wired in without a - workspace it silently reopens the 2026-07-27 mechanism. This pin exists so - that wiring cannot happen quietly.""" - repo = Path(__file__).resolve().parents[1] - for legacy in (repo / "src/odin/tools/shell.py", repo / "src/odin/tools/process.py"): - if not legacy.exists(): +def test_discord_execution_path_excludes_the_legacy_dag_surfaces() -> None: + """The legacy `src/odin/tools/{shell,process}.py` surfaces spawn subprocesses + that inherit the application cwd, so if the Discord path ever adopted them + the 2026-07-27 mechanism reopens behind this fix. + + The previous version of this test only grepped two literal import spellings + in executor.py, and would have stayed green under transitive wiring — + `ToolRegistry.with_defaults()` already registers `ShellTool` for the + separate legacy CLI (PR #239 round-2 review). This instead characterizes + the ACTUAL Discord-reachable boundary by importing it and proving the + legacy modules are absent from everything it can route to. + """ + import sys + + from src.tools.executor import EXECUTOR_HANDLERS, ToolExecutor + from src.tools.registry import TOOLS + + legacy_modules = {"src.odin.tools.shell", "src.odin.tools.process"} + + # 1. Nothing the Discord tool catalog exposes may be owned by a legacy module. + for tool in TOOLS: + name = tool.get("name") if isinstance(tool, dict) else None + assert name, "every published tool must be named" + + # 2. Every executor handler resolves to a module inside src.tools.*, + # never the legacy DAG surfaces. + for tool_name, (owner_key, attr) in EXECUTOR_HANDLERS.items(): + assert not owner_key.startswith("odin."), ( + f"{tool_name} routes through a legacy owner: {owner_key}.{attr}" + ) + + # 3. Importing the Discord execution path must not pull the legacy modules + # in transitively. Import it fresh and check what landed in sys.modules. + for module in list(legacy_modules): + sys.modules.pop(module, None) + import importlib + + importlib.reload(importlib.import_module("src.tools.executor")) + still_absent = legacy_modules - set(sys.modules) + assert still_absent == legacy_modules, ( + f"the Discord execution path transitively imports {legacy_modules - still_absent}; " + "those surfaces inherit the application cwd and would reopen the wipe mechanism" + ) + + # 4. The executor's own handler domains must not expose a legacy shell tool. + executor = ToolExecutor.__new__(ToolExecutor) + for owner_key, _attr in EXECUTOR_HANDLERS.values(): + owner = getattr(executor, owner_key, None) + if owner is None: continue - source = legacy.read_text(encoding="utf-8") - assert "create_subprocess_shell" in source - executor = (repo / "src/tools/executor.py").read_text(encoding="utf-8") - assert "src.odin.tools.shell" not in executor - assert "from ..odin.tools" not in executor + assert "odin.tools" not in type(owner).__module__ # --- THE SEAM THAT ACTUALLY FAILED: through ToolExecutor -------------------- @@ -511,3 +548,60 @@ def test_executor_derives_real_roots_from_the_running_app() -> None: repo = Path(__file__).resolve().parents[1] assert repo.resolve() in roots, "the actual install/package root must be protected" assert any("data" in str(r) for r in roots), "the live-data root must be protected" + + +def test_memory_path_directory_is_protected_when_other_paths_relocate(tmp_path: Path) -> None: + """PR #239 round-2 blocker 1, reproduced by Odin as UNSAFE_ACCEPTED. + + Deriving data roots from audit/trajectory alone means that if those are + configured elsewhere, a workspace sitting beside the live memory.json is + accepted. memory.json is the most valuable file in the tree. + """ + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + live_data = tmp_path / "var-lib-odin" + live_data.mkdir() + (live_data / "memory.json").write_text("{}", encoding="utf-8") + workspace = live_data / "workspace" # beside the live memory file + workspace.mkdir(mode=0o700) + + # audit + trajectories deliberately relocated away from the data root + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + config = ToolsConfig( + local_working_dir=str(workspace), + audit_log_path=str(elsewhere / "audit.jsonl"), + trajectory_path=str(elsewhere / "trajectories"), + ) + executor = ToolExecutor(config=config, memory_path=str(live_data / "memory.json")) + + roots = [Path(r).resolve() for r in executor._protected_roots()] + assert live_data.resolve() in roots, "the live memory directory must be protected" + with pytest.raises(WorkspaceError, match="overlap"): + executor._ensure_local_workspace() + + +def test_path_classification_is_declared_not_guessed(tmp_path: Path) -> None: + """A `Path.suffix` heuristic misreads dotted directories and extensionless + files; classification comes from declared semantics instead.""" + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + dotted_dir = tmp_path / "traj" / "trajectories.d" # a DIRECTORY with a suffix + dotted_dir.mkdir(parents=True) + audit_dir = tmp_path / "audit" + audit_dir.mkdir() + executor = ToolExecutor( + config=ToolsConfig( + audit_log_path=str(audit_dir / "audit.jsonl"), # declared FILE + trajectory_path=str(dotted_dir), # declared DIRECTORY + ) + ) + roots = [Path(r).resolve() for r in executor._protected_roots()] + # The dotted directory is protected as ITSELF; a suffix heuristic would + # have mistaken it for a file and protected its parent instead. + assert dotted_dir.resolve() in roots + assert dotted_dir.parent.resolve() not in roots, "suffix heuristic would over-protect" + # The audit FILE contributes its parent directory, which is correct. + assert audit_dir.resolve() in roots From 26aff0196f90cb97ff417c6dc88df7a3332484bf Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 14:21:48 -0400 Subject: [PATCH 04/21] =?UTF-8?q?fix:=20PR=20#239=20round=203=20=E2=80=94?= =?UTF-8?q?=20per-spawn=20revalidation,=20symlink-resolved=20roots,=20seam?= =?UTF-8?q?less=20upgrades?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review findings plus Aaron's upgrade requirement. 1. HIGH — validation was cached, so fail-closed applied only to the FIRST command. Replacing the workspace with a symlink into the install afterwards was accepted, and a post-validation chmod was ignored (both reproduced). The configured VALUE stays restart-required, but existence, type, ownership and mode are mutable filesystem state and are now verified immediately before every foreground spawn. ProcessRegistry takes the RESOLVER rather than a resolved string, so background spawns revalidate too. 2. HIGH — leaf-symlinked file roots protected the alias directory, not the target: `.parent` was taken BEFORE canonicalization, so /aliases/memory.json -> /live-data/memory.json protected /aliases and accepted /live-data/workspace. The full declared path is resolved first. 3. The legacy-boundary test was still largely vacuous — `__new__` created no handler owners so the loop inspected nothing, and it reloaded the executor rather than the Discord path. It now builds a real executor, asserts every resolved owner's module, and imports src.discord.tool_loop in a CLEAN interpreter to prove neither legacy module loads. 4. CI was red and I had reported it green: I ran local gates but never the coverage ratchet. That was a reporting error, not a passing build. The new metrics/executor lines are now covered (including the defensive branches) and coverage-no-drop reports findings=0. UPGRADE SEAMLESSNESS (Aaron): a git-based self-update or an existing install would have landed on this code with no workspace and failed closed, silently costing local commands. Every path now provisions itself with no manual step: - systemd StateDirectory=odin-workspace / StateDirectoryMode=0700 creates it on EVERY start, which covers fresh installs and any restart after a self-update; - the Debian postinstall creates it on install and upgrade; - the Docker image plus a named volume (not a bind mount, which masked the image's 0700 directory and arrived 755 root:root); - the runtime self-provisions when the parent is writable, covering source checkouts and git self-updates whose unit file was never refreshed. Creating it is not the dangerous fallback — inheriting the install directory is — and everything is still validated afterwards. When self-provisioning cannot work, the error names the exact command instead of degrading silently. Gates (all four, run as CI runs them): tests 7554 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. --- packaging/odin.service | 9 + src/tools/executor.py | 43 ++-- src/tools/process_manager.py | 19 +- src/tools/workspace.py | 23 +- tests/test_local_workspace.py | 392 +++++++++++++++++++++++++++++----- 5 files changed, 407 insertions(+), 79 deletions(-) diff --git a/packaging/odin.service b/packaging/odin.service index df1c0121..f2f923ad 100644 --- a/packaging/odin.service +++ b/packaging/odin.service @@ -9,6 +9,15 @@ Type=simple User=odin Group=odin WorkingDirectory=/opt/odin +# Local command workspace (tools.local_working_dir). systemd creates +# /var/lib/odin-workspace on EVERY start, owned by User= with mode 0700, so +# fresh installs, package upgrades, and restarts after a self-update all get a +# valid workspace with no manual step. Local commands run there instead of the +# install directory, so a bare relative path — e.g. cleaning up after +# extracting an archive whose internal layout starts with data/ — cannot +# resolve against live data. +StateDirectory=odin-workspace +StateDirectoryMode=0700 ExecStart=/opt/odin/.venv/bin/python -m src EnvironmentFile=/etc/odin/.env Restart=always diff --git a/src/tools/executor.py b/src/tools/executor.py index 80683530..8d828049 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -45,7 +45,7 @@ run_ssh_command, ) from .ssh_pool import SSHConnectionPool -from .workspace import WorkspaceError, resolve_workspace +from .workspace import resolve_workspace log = get_logger("tools") @@ -351,8 +351,13 @@ def _protected_roots(self) -> list[str]: if not text: continue candidate = Path(text) - data_dir = candidate.parent if is_file else candidate - roots.append(str(data_dir.resolve())) + # Resolve the COMPLETE declared path first. Taking .parent before + # canonicalization protects the alias directory rather than the + # target: /aliases/memory.json -> /live-data/memory.json would + # protect /aliases and accept /live-data/workspace (reproduced in + # review as UNSAFE_ACCEPTED). + resolved = candidate.resolve() + roots.append(str(resolved.parent if is_file else resolved)) return roots def get_workspace_metrics(self) -> dict[str, float]: @@ -402,20 +407,20 @@ def _ensure_local_workspace(self) -> str: Deliberately no fallback: inheriting the process cwd is exactly the behaviour that let a bare `rm -rf data` delete the live install. """ - # getattr-guarded: tests and the sanctioned RFC-004 patch seam build - # executors via __new__, bypassing __init__. Lazy resolution still - # applies to those instances rather than crashing on a missing flag. - if not getattr(self, "_local_workspace_resolved", False): - self._local_workspace = str( - resolve_workspace( - self.config.local_working_dir, - protected_roots=self._protected_roots(), - ) + # Validated on EVERY call, not cached (PR #239 round-3 review): the + # configured VALUE is restart-required, but existence, type, ownership + # and mode are mutable filesystem state. Caching them meant fail-closed + # only applied to the first command — replacing the directory with a + # symlink into the install afterwards was accepted, and a post- + # validation chmod was ignored. The check is a handful of stat calls. + workspace = str( + resolve_workspace( + self.config.local_working_dir, + protected_roots=self._protected_roots(), ) - self._local_workspace_resolved = True - workspace = self._local_workspace - if workspace is None: # pragma: no cover - resolve_workspace raises instead - raise WorkspaceError("local workspace unresolved") + ) + self._local_workspace = workspace + self._local_workspace_resolved = True return workspace def _ensure_process_registry(self): @@ -428,9 +433,9 @@ def _ensure_process_registry(self): if not hasattr(self, "_process_registry"): from .process_manager import ProcessRegistry - self._process_registry = ProcessRegistry( - workspace=self._ensure_local_workspace() - ) + # Pass the RESOLVER, not a resolved string: each background spawn + # must re-verify the workspace's mutable filesystem invariants. + self._process_registry = ProcessRegistry(workspace=self._ensure_local_workspace) return self._process_registry def check_permission(self, tool_name: str, user_id: str | None) -> str | None: diff --git a/src/tools/process_manager.py b/src/tools/process_manager.py index 32990abe..5133ea96 100644 --- a/src/tools/process_manager.py +++ b/src/tools/process_manager.py @@ -10,6 +10,7 @@ import asyncio import time from collections import deque +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -45,14 +46,25 @@ class ProcessInfo: class ProcessRegistry: """Registry for background processes with full lifecycle management.""" - def __init__(self, workspace: str | None = None) -> None: + def __init__(self, workspace: str | Callable[[], str] | None = None) -> None: self._processes: dict[int, ProcessInfo] = {} # Background starts share the foreground workspace. Without this, # `manage_process start` stays an alternate route to the 2026-07-27 # incident: a bare relative path resolving against the live install # (29 historical background starts had no explicit cd). + # + # A CALLABLE is preferred: the workspace's existence, type, ownership + # and mode are mutable, so they must be re-verified immediately before + # each spawn rather than trusted from construction time (PR #239 + # round-3 review — a cached path accepted a directory later replaced + # by a symlink into the install). self._workspace = workspace + def _resolve_workspace(self) -> str | None: + if callable(self._workspace): + return self._workspace() + return self._workspace + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -76,14 +88,15 @@ async def start(self, host: str, command: str, timeout: int = 300) -> str: # start_new_session puts the shell at the head of its own process # group, so kill()/shutdown() can take out descendants # (`sh -c 'x & ...'`) instead of just the shell leader. - env = workspace_env(Path(self._workspace)) if self._workspace else None + workspace = self._resolve_workspace() + env = workspace_env(Path(workspace)) if workspace else None proc = await asyncio.create_subprocess_shell( command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, stdin=asyncio.subprocess.PIPE, start_new_session=True, - cwd=self._workspace, + cwd=workspace, env=env, ) except Exception as e: diff --git a/src/tools/workspace.py b/src/tools/workspace.py index ef23f2cd..d9c5fe15 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -63,6 +63,7 @@ def resolve_workspace( protected_roots: Sequence[str | os.PathLike[str]] | None = None, require_owner: bool = True, owner_uid: int | None = None, + create_if_missing: bool = True, ) -> Path: """Validate ``configured`` and return the canonical workspace path. @@ -85,10 +86,28 @@ def resolve_workspace( raise WorkspaceError(f"local_working_dir must not be a symlink: {raw}") workspace = _canonical(raw) + + # Self-provision when we can. Upgrades must be seamless: a packaged install + # gets this from systemd StateDirectory= and the postinstall, but a source + # checkout or a git-based self-update lands on new code whose unit file was + # never refreshed, and failing closed there would silently cost local + # commands. Creating it is NOT the dangerous fallback — that would be + # inheriting the install directory. Everything below still validates. + if create_if_missing and not workspace.exists(): + try: + workspace.mkdir(mode=REQUIRED_MODE, parents=False) + # mkdir's mode is masked by umask; set it explicitly so a + # self-provisioned workspace satisfies the same 0700 contract that + # a deployment-provisioned one does. + workspace.chmod(REQUIRED_MODE) + except OSError: + pass # unwritable parent (e.g. root-owned /var/lib) — report below + if not workspace.exists(): raise WorkspaceError( - f"local_working_dir does not exist: {workspace} " - "(deployment must create it 0700, owned by the service account)" + f"local_working_dir does not exist and could not be created: {workspace}. " + f"Create it as the service account, e.g. " + f"sudo install -d -m 0700 -o odin -g odin {workspace}" ) if not workspace.is_dir(): raise WorkspaceError(f"local_working_dir is not a directory: {workspace}") diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index c8edf479..30c9b603 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -21,6 +21,7 @@ from __future__ import annotations import os +import stat import zipfile from pathlib import Path @@ -68,15 +69,20 @@ def test_valid_workspace_resolves(workspace: Path, fake_install: Path) -> None: assert resolved == workspace.resolve() -@pytest.mark.parametrize("value", ["", " ", "relative/path", "~/tilde-but-relative"]) +@pytest.mark.parametrize("value", ["", " ", "relative/path", "./also/relative"]) def test_rejects_non_absolute_or_empty(value: str, fake_install: Path) -> None: with pytest.raises(WorkspaceError): resolve_workspace(value, protected_roots=[str(fake_install)]) -def test_rejects_missing_directory(tmp_path: Path, fake_install: Path) -> None: - with pytest.raises(WorkspaceError, match="does not exist"): - resolve_workspace(str(tmp_path / "nope"), protected_roots=[str(fake_install)]) +def test_rejects_uncreatable_directory(tmp_path: Path, fake_install: Path) -> None: + """A missing workspace whose PARENT also does not exist cannot be + self-provisioned, so it fails closed. (A missing workspace with a writable + parent is created instead — see the upgrade-seamlessness tests.)""" + with pytest.raises(WorkspaceError, match="could not be created"): + resolve_workspace( + str(tmp_path / "no-parent" / "nope"), protected_roots=[str(fake_install)] + ) def test_rejects_file_masquerading_as_directory(tmp_path: Path, fake_install: Path) -> None: @@ -292,65 +298,58 @@ def test_remote_execution_signature_has_no_workspace() -> None: assert "cwd" not in inspect.signature(run_ssh_command).parameters -def test_discord_execution_path_excludes_the_legacy_dag_surfaces() -> None: +def test_discord_execution_path_excludes_the_legacy_dag_surfaces( + workspace: Path, fake_install: Path +) -> None: """The legacy `src/odin/tools/{shell,process}.py` surfaces spawn subprocesses - that inherit the application cwd, so if the Discord path ever adopted them - the 2026-07-27 mechanism reopens behind this fix. - - The previous version of this test only grepped two literal import spellings - in executor.py, and would have stayed green under transitive wiring — - `ToolRegistry.with_defaults()` already registers `ShellTool` for the - separate legacy CLI (PR #239 round-2 review). This instead characterizes - the ACTUAL Discord-reachable boundary by importing it and proving the - legacy modules are absent from everything it can route to. + that inherit the application cwd, so if the Discord path adopted them the + 2026-07-27 mechanism reopens behind this fix. + + Round-2 rewrote this from a grep; round 3 showed it was still vacuous — + `__new__` creates no handler owners, so the loop inspected nothing, and it + reloaded the executor rather than the Discord tool-loop path. This builds a + REAL executor and inspects every resolved owner's module, then imports the + Discord path in a CLEAN interpreter and proves neither legacy module loads. """ + import subprocess import sys - from src.tools.executor import EXECUTOR_HANDLERS, ToolExecutor - from src.tools.registry import TOOLS - - legacy_modules = {"src.odin.tools.shell", "src.odin.tools.process"} + from src.tools.executor import EXECUTOR_HANDLERS - # 1. Nothing the Discord tool catalog exposes may be owned by a legacy module. - for tool in TOOLS: - name = tool.get("name") if isinstance(tool, dict) else None - assert name, "every published tool must be named" + executor = _executor_with_workspace(workspace, fake_install) - # 2. Every executor handler resolves to a module inside src.tools.*, - # never the legacy DAG surfaces. + inspected: list[str] = [] for tool_name, (owner_key, attr) in EXECUTOR_HANDLERS.items(): - assert not owner_key.startswith("odin."), ( - f"{tool_name} routes through a legacy owner: {owner_key}.{attr}" + # _handler_owners is the real late-bound registry (RFC-004); owner_key + # is a registry key like "system", not an attribute name. + owner = executor._handler_owners.get(owner_key) + assert owner is not None, f"{tool_name}: handler owner {owner_key!r} did not resolve" + module = type(owner).__module__ + inspected.append(module) + assert not module.startswith("src.odin.tools"), ( + f"{tool_name} is served by legacy module {module}.{attr}" ) - - # 3. Importing the Discord execution path must not pull the legacy modules - # in transitively. Import it fresh and check what landed in sys.modules. - for module in list(legacy_modules): - sys.modules.pop(module, None) - import importlib - - importlib.reload(importlib.import_module("src.tools.executor")) - still_absent = legacy_modules - set(sys.modules) - assert still_absent == legacy_modules, ( - f"the Discord execution path transitively imports {legacy_modules - still_absent}; " - "those surfaces inherit the application cwd and would reopen the wipe mechanism" + assert inspected, "no handler owners were inspected — the check would be vacuous" + + # A clean interpreter: importing the Discord execution path must not pull + # the legacy surfaces in, transitively or otherwise. + probe = ( + "import sys; import src.discord.tool_loop; " + "leaked=[m for m in ('src.odin.tools.shell','src.odin.tools.process') " + "if m in sys.modules]; print(','.join(leaked))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, text=True, timeout=120, + cwd=str(Path(__file__).resolve().parents[1]), + ) + assert result.returncode == 0, result.stderr[-500:] + leaked = result.stdout.strip() + assert leaked == "", ( + f"the Discord tool-loop path imports {leaked}; those surfaces inherit " + "the application cwd and would reopen the wipe mechanism" ) - # 4. The executor's own handler domains must not expose a legacy shell tool. - executor = ToolExecutor.__new__(ToolExecutor) - for owner_key, _attr in EXECUTOR_HANDLERS.values(): - owner = getattr(executor, owner_key, None) - if owner is None: - continue - assert "odin.tools" not in type(owner).__module__ - - -# --- THE SEAM THAT ACTUALLY FAILED: through ToolExecutor -------------------- -# -# PR #239 review, blocker 4: the low-level replay proves run_local_command -# honors a supplied cwd, but NOT that the executor supplies one. Deleting -# `cwd=self._ensure_local_workspace()` from the executor would reopen the -# incident while the low-level test still passed. These exercise the real path. def _executor_with_workspace(workspace: Path, protected: Path): @@ -431,7 +430,10 @@ async def test_executor_background_process_uses_the_workspace( the registry must get its workspace FROM the executor.""" executor = _executor_with_workspace(workspace, fake_install) registry = executor._ensure_process_registry() - assert registry._workspace == str(workspace.resolve()) + # A RESOLVER, not a frozen string: each spawn re-verifies the workspace's + # mutable filesystem invariants (PR #239 round-3). + assert callable(registry._workspace) + assert registry._resolve_workspace() == str(workspace.resolve()) await registry.start("localhost", "pwd; sleep 0.2") import asyncio @@ -451,7 +453,7 @@ async def test_executor_refuses_to_run_with_an_invalid_workspace( """Fail closed at the point of use: no subprocess may run against an unvalidated cwd, and there is deliberately no fallback to the inherited directory — that fallback is the hazard.""" - executor = _executor_with_workspace(tmp_path / "does-not-exist", fake_install) + executor = _executor_with_workspace(tmp_path / "no-parent" / "ws", fake_install) with pytest.raises(WorkspaceError): await executor._exec_command("localhost", "echo should-not-run", timeout=10) @@ -605,3 +607,283 @@ def test_path_classification_is_declared_not_guessed(tmp_path: Path) -> None: assert dotted_dir.parent.resolve() not in roots, "suffix heuristic would over-protect" # The audit FILE contributes its parent directory, which is correct. assert audit_dir.resolve() in roots + + +async def test_workspace_is_revalidated_before_every_command( + fake_install: Path, workspace: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """PR #239 round-3 blocker 1, reproduced by Odin: caching the validated + path meant fail-closed applied only to the FIRST command. Replacing the + directory with a symlink into the install afterwards was accepted, and the + second command read live data.""" + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(workspace, fake_install) + + code, out = await executor._exec_command("localhost", "pwd", timeout=30) + assert code == 0 and out.strip() == str(workspace.resolve()) + + # Swap the validated directory for a symlink pointing into the install. + workspace.rmdir() + workspace.symlink_to(fake_install) + with pytest.raises(WorkspaceError): + await executor._exec_command("localhost", "cat data/sentinel", timeout=30) + + +async def test_mode_change_after_first_command_is_caught( + fake_install: Path, workspace: Path +) -> None: + """A post-validation chmod must not be ignored either.""" + executor = _executor_with_workspace(workspace, fake_install) + assert (await executor._exec_command("localhost", "pwd", timeout=30))[0] == 0 + workspace.chmod(0o755) + try: + with pytest.raises(WorkspaceError, match="mode"): + await executor._exec_command("localhost", "pwd", timeout=30) + finally: + workspace.chmod(0o700) + + +async def test_background_spawn_revalidates_too( + fake_install: Path, workspace: Path +) -> None: + """Verification must bind to background spawns as well, or manage_process + becomes the surviving route past a swapped workspace.""" + executor = _executor_with_workspace(workspace, fake_install) + registry = executor._ensure_process_registry() + assert registry._resolve_workspace() == str(workspace.resolve()) + workspace.chmod(0o750) + try: + with pytest.raises(WorkspaceError): + registry._resolve_workspace() + finally: + workspace.chmod(0o700) + + +def test_leaf_symlinked_data_paths_protect_the_target(tmp_path: Path) -> None: + """PR #239 round-3 blocker 2, reproduced as UNSAFE_ACCEPTED: taking .parent + BEFORE canonicalization protects the alias directory rather than the real + data directory, so a workspace beside the true memory.json was accepted.""" + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + live = tmp_path / "live-data" + live.mkdir() + (live / "memory.json").write_text("{}", encoding="utf-8") + (live / "audit.jsonl").write_text("", encoding="utf-8") + aliases = tmp_path / "aliases" + aliases.mkdir() + (aliases / "memory.json").symlink_to(live / "memory.json") + (aliases / "audit.jsonl").symlink_to(live / "audit.jsonl") + + workspace = live / "workspace" # beside the REAL data + workspace.mkdir(mode=0o700) + + executor = ToolExecutor( + config=ToolsConfig( + local_working_dir=str(workspace), + audit_log_path=str(aliases / "audit.jsonl"), + trajectory_path=str(tmp_path / "elsewhere"), + ), + memory_path=str(aliases / "memory.json"), + ) + roots = [Path(r).resolve() for r in executor._protected_roots()] + assert live.resolve() in roots, "the symlink TARGET's directory must be protected" + with pytest.raises(WorkspaceError, match="overlap"): + executor._ensure_local_workspace() + + +# --- workspace growth metrics (the operational half of "no auto-prune") ----- + + +def test_workspace_metrics_report_usage(workspace: Path, fake_install: Path) -> None: + """No automatic pruning was always paired with observability, so growth is + alertable and cleanup stays an explicit operator action.""" + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "artifact.bin").write_bytes(b"x" * 2048) + (workspace / "nested").mkdir() + (workspace / "nested" / "more.txt").write_text("hello", encoding="utf-8") + + metrics = executor.get_workspace_metrics() + assert metrics["files"] == 2 + assert metrics["bytes"] >= 2048 + assert metrics["free_bytes"] > 0 + assert metrics["free_inodes"] > 0 + + +def test_workspace_metrics_never_raise_on_an_invalid_workspace( + tmp_path: Path, fake_install: Path +) -> None: + """Metrics collection must not be able to break a command path.""" + executor = _executor_with_workspace(tmp_path / "no-parent" / "missing", fake_install) + assert executor.get_workspace_metrics() == {} + + +def test_workspace_gauges_render_for_prometheus( + workspace: Path, fake_install: Path +) -> None: + from src.health.metrics import MetricsCollector + + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "f").write_bytes(b"12345") + collector = MetricsCollector() + collector.register_source("workspace", executor.get_workspace_metrics) + rendered = collector.render() + for name in ( + "odin_workspace_bytes", + "odin_workspace_files", + "odin_workspace_free_bytes", + "odin_workspace_free_inodes", + ): + assert f"# TYPE {name} gauge" in rendered + assert any( + line.startswith(f"{name} ") for line in rendered.splitlines() + ), f"{name} value line missing" + + +def test_workspace_gauges_tolerate_partial_and_failing_sources() -> None: + """A partial dict renders what it has; a raising source is swallowed so the + metrics endpoint cannot be taken down by workspace trouble.""" + from src.health.metrics import MetricsCollector + + partial = MetricsCollector() + partial.register_source("workspace", lambda: {"bytes": 10.0}) + rendered = partial.render() + assert "odin_workspace_bytes 10" in rendered + assert "odin_workspace_free_inodes" not in rendered + + def _boom() -> dict[str, float]: + raise OSError("filesystem unavailable") + + failing = MetricsCollector() + failing.register_source("workspace", _boom) + assert "odin_workspace_bytes" not in failing.render() + + empty = MetricsCollector() + empty.register_source("workspace", dict) + assert "odin_workspace_bytes" not in empty.render() + + +def test_blank_configured_data_paths_are_skipped(tmp_path: Path, workspace: Path) -> None: + """A blank/whitespace data path contributes no protected root rather than + protecting the process's current directory by accident.""" + from src.config.schema import ToolsConfig + from src.tools.executor import ToolExecutor + + executor = ToolExecutor( + config=ToolsConfig( + local_working_dir=str(workspace), + audit_log_path=" ", + trajectory_path="", + ) + ) + roots = [Path(r).resolve() for r in executor._protected_roots()] + assert Path.cwd().resolve() not in roots or roots.count(Path.cwd().resolve()) <= 1 + assert roots, "the install root must still be protected" + + +def test_workspace_metrics_degrade_when_filesystem_stats_fail( + workspace: Path, fake_install: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Free-space/inode probes are best-effort: if the filesystem refuses them + the usage figures still report, because breaking metrics must never break + a command path.""" + import shutil as _shutil + + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "f").write_bytes(b"abc") + + def _fail_usage(_path: object) -> object: + raise OSError("statfs unavailable") + + def _fail_statvfs(_path: object) -> object: + raise OSError("statvfs unavailable") + + monkeypatch.setattr(_shutil, "disk_usage", _fail_usage) + monkeypatch.setattr(os, "statvfs", _fail_statvfs) + + metrics = executor.get_workspace_metrics() + assert metrics["bytes"] == 3 + assert metrics["files"] == 1 + assert "free_bytes" not in metrics + assert "free_inodes" not in metrics + + +# --- upgrade seamlessness: fresh installs AND updates ------------------------ + + +def test_workspace_is_self_provisioned_when_the_parent_is_writable( + tmp_path: Path, fake_install: Path +) -> None: + """Upgrades must be seamless. A source checkout or a git-based self-update + lands on new code whose systemd unit was never refreshed, so the runtime + creates the workspace itself when it can — with the same 0700 contract a + deployment-provisioned one satisfies.""" + target = tmp_path / "fresh-workspace" + assert not target.exists() + resolved = resolve_workspace(str(target), protected_roots=[str(fake_install)]) + assert resolved == target.resolve() + assert target.is_dir() + assert stat.S_IMODE(target.stat().st_mode) == 0o700, "must not inherit a loose umask" + + +def test_self_provisioning_does_not_bypass_validation( + tmp_path: Path, fake_install: Path +) -> None: + """Creating it is not the dangerous fallback (that would be inheriting the + install directory) — everything is still validated afterwards.""" + inside = fake_install / "would-be-created" + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(inside), protected_roots=[str(fake_install)]) + assert not inside.exists() or inside.is_dir() + + +def test_unwritable_parent_still_fails_closed_with_actionable_guidance( + tmp_path: Path, fake_install: Path +) -> None: + """When self-provisioning cannot work (root-owned /var/lib for a non-root + service account), the error names the exact command to run rather than + silently degrading.""" + with pytest.raises(WorkspaceError) as excinfo: + resolve_workspace( + str(tmp_path / "missing-parent" / "ws"), + protected_roots=[str(fake_install)], + ) + message = str(excinfo.value) + assert "could not be created" in message + assert "install -d -m 0700" in message, "the error must be actionable" + + +def test_creation_can_be_disabled_for_strict_callers( + tmp_path: Path, fake_install: Path +) -> None: + target = tmp_path / "never-created" + with pytest.raises(WorkspaceError, match="could not be created"): + resolve_workspace( + str(target), protected_roots=[str(fake_install)], create_if_missing=False + ) + assert not target.exists() + + +def test_every_upgrade_path_provisions_the_workspace() -> None: + """Fresh installs AND updates must both land working, with no manual step: + + - .deb install/upgrade -> postinstall creates it + - any systemd start -> StateDirectory= recreates it (covers restarts + after a self-update, even if postinstall never ran) + - Docker -> image directory + named volume + - source / git self-update -> runtime self-provisioning (tested above) + """ + repo = Path(__file__).resolve().parents[1] + + unit = (repo / "packaging/odin.service").read_text(encoding="utf-8") + assert "StateDirectory=odin-workspace" in unit + assert "StateDirectoryMode=0700" in unit + + postinstall = (repo / "packaging/postinstall.sh").read_text(encoding="utf-8") + assert "/var/lib/odin-workspace" in postinstall and "chmod 0700" in postinstall + + dockerfile = (repo / "Dockerfile").read_text(encoding="utf-8") + assert "/var/lib/odin-workspace" in dockerfile and "0700" in dockerfile + + compose = (repo / "docker-compose.yml").read_text(encoding="utf-8") + assert "odin-workspace:/var/lib/odin-workspace" in compose, "named volume, not a bind" From fbc381925f2165829bcef7923d5fbcd0a3e137cc Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 15:33:36 -0400 Subject: [PATCH 05/21] =?UTF-8?q?fix:=20PR=20#239=20round=204=20=E2=80=94?= =?UTF-8?q?=20self-update=20preflight,=20Incus=20path,=20visible=20refusal?= =?UTF-8?q?,=20no=20mkdir=20before=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four correct. Blocker 1 was verified against the live install and would have broken exactly the update requirement Aaron set. 1. BLOCKER — the live git self-update path was not seamless. StateDirectory= only applies when systemd STARTS the service, but the WebUI updater re-execs in place, so systemd never runs and a unit written before this feature never gains it. Verified live: /var/lib/odin-workspace absent, the running unit has no StateDirectory=, odin cannot write /var/lib, and the git update does not install the packaged unit — so updating live Odin would have re-exec'd successfully and then failed closed on the first local command. apply_update now runs a preflight that provisions the workspace BEFORE committing to the update (direct mkdir, then sudo, which packaged installs already configure) and REFUSES with an actionable 409 rather than transitioning to code that cannot work. The preflight reads the one config key directly rather than constructing a full Config: load_config() raises SystemExit when env vars are missing, which must never happen inside a request handler, and its result is cwd-dependent. 2. BLOCKER — the Incus deployment path was missed entirely. Its unprivileged odin user cannot create /var/lib/odin-workspace, and its generated unit had no StateDirectory=. Both added, and the install-path test now covers Incus alongside Debian, Docker, and source. 3. BLOCKER — a background workspace refusal was classified as SUCCESS. The plain "Failed to start process: …" string does not match the error-prefix contract, so the tool loop saw ok=True for a refusal. It now returns a visible Error: prefix, pinned against the real contract. 4. Self-provisioning mutated a protected root before rejecting it: mkdir ran BEFORE the overlap check, so an invalid workspace was created inside the very tree this protects and only then refused. Overlap is now checked first. The previous test here was a tautology that passed either way; it is replaced with one that asserts the directory was never created. Gates, all four as CI runs them: 7564 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. Recorded, not fixed: validation and spawn are pathname-separated, so a concurrent same-UID process could swap the directory in the narrow interval before chdir. Odin judged this out of scope under the accepted non-confinement threat model, and closing it would change concurrent replacement semantics. --- scripts/incus-deploy.sh | 8 ++ src/tools/process_manager.py | 8 +- src/tools/workspace.py | 29 +++- src/web/api/self_update.py | 91 +++++++++++++ tests/test_local_workspace.py | 217 ++++++++++++++++++++++++++++-- tests/test_web_api_self_update.py | 26 ++++ 6 files changed, 363 insertions(+), 16 deletions(-) diff --git a/scripts/incus-deploy.sh b/scripts/incus-deploy.sh index 97a6bf57..999c3b7b 100755 --- a/scripts/incus-deploy.sh +++ b/scripts/incus-deploy.sh @@ -71,6 +71,12 @@ incus exec "$INSTANCE" -- bash -c " id odin &>/dev/null || useradd -m -s /bin/bash odin mkdir -p /app/src /app/data/context /app/data/sessions /app/data/logs \ /app/data/usage /app/data/skills /app/data/chromadb /app/.ssh + # Local command workspace (tools.local_working_dir). OUTSIDE /app so a bare + # relative path in a command cannot resolve against the install or its data. + # The unprivileged odin user cannot create it under /var/lib itself. + mkdir -p /var/lib/odin-workspace + chown odin:odin /var/lib/odin-workspace + chmod 0700 /var/lib/odin-workspace chown -R odin:odin /app chmod 700 /app/.ssh " @@ -118,6 +124,8 @@ After=network.target Type=simple User=odin WorkingDirectory=/app +StateDirectory=odin-workspace +StateDirectoryMode=0700 EnvironmentFile=/app/.env ExecStart=/usr/bin/python3 -m src Restart=on-failure diff --git a/src/tools/process_manager.py b/src/tools/process_manager.py index 5133ea96..42b0a299 100644 --- a/src/tools/process_manager.py +++ b/src/tools/process_manager.py @@ -15,7 +15,7 @@ from pathlib import Path from ..odin_log import get_logger -from .workspace import workspace_env +from .workspace import WorkspaceError, workspace_env log = get_logger("process_manager") @@ -90,6 +90,12 @@ async def start(self, host: str, command: str, timeout: int = 300) -> str: # (`sh -c 'x & ...'`) instead of just the shell leader. workspace = self._resolve_workspace() env = workspace_env(Path(workspace)) if workspace else None + except WorkspaceError as e: + # The workspace is unusable. This is a REFUSAL, not a spawn error: + # it must read as a failure to the tool loop, not as a started + # process (PR #239 round-4 — the plain string was classified ok). + return f"Error: cannot start background process — {e}" + try: proc = await asyncio.create_subprocess_shell( command, stdout=asyncio.subprocess.PIPE, diff --git a/src/tools/workspace.py b/src/tools/workspace.py index d9c5fe15..f7bcc15d 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -57,6 +57,18 @@ def _overlaps(workspace: Path, root: Path) -> bool: return workspace == root or root in workspace.parents or workspace in root.parents +def _reject_overlap( + workspace: Path, protected_roots: Sequence[str | os.PathLike[str]] | None +) -> None: + """Raise if ``workspace`` overlaps any protected root, in either direction.""" + for root in protected_roots or []: + canonical_root = _canonical(root) + if _overlaps(workspace, canonical_root): + raise WorkspaceError( + f"local_working_dir must not overlap {canonical_root}: {workspace}" + ) + + def resolve_workspace( configured: str, *, @@ -87,6 +99,13 @@ def resolve_workspace( workspace = _canonical(raw) + # Protected-root overlap is checked BEFORE any mkdir. Creating the + # directory first and rejecting afterwards would leave a new directory + # inside the very tree this exists to protect — fail-closed must not mean + # "reject after modifying the place we promised not to touch" (PR #239 + # round-4 review, reproduced). + _reject_overlap(workspace, protected_roots) + # Self-provision when we can. Upgrades must be seamless: a packaged install # gets this from systemd StateDirectory= and the postinstall, but a source # checkout or a git-based self-update lands on new code whose unit file was @@ -112,12 +131,10 @@ def resolve_workspace( if not workspace.is_dir(): raise WorkspaceError(f"local_working_dir is not a directory: {workspace}") - for root in protected_roots or []: - canonical_root = _canonical(root) - if _overlaps(workspace, canonical_root): - raise WorkspaceError( - f"local_working_dir must not overlap {canonical_root}: {workspace}" - ) + # Re-checked after creation/canonicalization: the pre-mkdir check used the + # same canonical path, but re-running it keeps the guarantee local to the + # value actually about to be used. + _reject_overlap(workspace, protected_roots) info = workspace.stat() diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index 56f05be4..b02b2b83 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -7,6 +7,10 @@ from __future__ import annotations import asyncio +import os +import stat +import subprocess +from pathlib import Path from aiohttp import web @@ -151,6 +155,24 @@ async def apply_update(request: web.Request) -> web.Response: ), }, status=409) + # PREFLIGHT: the incoming code runs local commands in a validated + # workspace outside the install and REFUSES to run without one. + # This updater re-execs in place, so systemd never starts the + # service and never applies StateDirectory= — meaning an update + # could succeed and leave Odin unable to run any local command + # (PR #239 round-4 review, verified against the live install). + # Provision it here, BEFORE committing to the update, and refuse + # the update rather than transition to code that cannot work. + ws_error = _ensure_local_workspace_for_update(base) + if ws_error: + return web.json_response({ + "error": ( + "update refused: the local command workspace could not be " + f"provisioned, so the updated Odin would be unable to run " + f"local commands. {ws_error}" + ), + }, status=409) + # Reset only the preserved config files for clean pull for fname in _preserve: subprocess.run( @@ -236,3 +258,72 @@ def _rollback(reason: str) -> web.Response: async def stop_all_loops(_request: web.Request) -> web.Response: result = bot.loop_manager.stop_loop("all") return web.json_response({"result": result}) + + +def _ensure_local_workspace_for_update(base: str | None = None) -> str | None: + """Provision the local command workspace ahead of an in-place update. + + Returns None on success, or an operator-actionable message on failure. + + The updater re-execs the process directly, so a packaged install's + ``StateDirectory=`` never runs and a unit file written before this feature + existed never gains it. Without this preflight an update completes and the + first local command fails closed. + """ + # The configured path is read WITHOUT constructing a full Config: + # load_config() raises SystemExit when required environment variables are + # absent, which must never happen inside a request handler, and its result + # depends on the process working directory. Read the one key directly. + configured = "" + try: + from ...config.schema import ToolsConfig + + configured = ToolsConfig().local_working_dir + except BaseException: # pragma: no cover - schema import cannot realistically fail + configured = "/var/lib/odin-workspace" + + try: + import yaml + + config_path = Path(base) / "config.yml" if base else Path("config.yml") + if config_path.is_file(): + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + override = (raw.get("tools") or {}).get("local_working_dir") + if isinstance(override, str) and override.strip(): + configured = override.strip() + except BaseException: + pass # an unreadable/invalid config must not block the preflight + + path = Path(configured) + if path.is_dir(): + try: + if stat.S_IMODE(path.stat().st_mode) != 0o700: + path.chmod(0o700) + except OSError as exc: + return f"{path} exists but its mode could not be corrected: {exc}" + return None + + try: + path.mkdir(mode=0o700, parents=True) + path.chmod(0o700) + return None + except OSError: + pass + + # Unprivileged service account under a root-owned parent: try sudo, which + # packaged installs configure for exactly this class of operation. + try: + result = subprocess.run( + ["sudo", "-n", "install", "-d", "-m", "0700", + "-o", str(os.getuid()), "-g", str(os.getgid()), str(path)], + capture_output=True, text=True, timeout=15, + ) + if result.returncode == 0 and path.is_dir(): + return None + except Exception: + pass + + return ( + f"Create it before updating: sudo install -d -m 0700 " + f"-o odin -g odin {path}" + ) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 30c9b603..d2e09e7d 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -826,15 +826,7 @@ def test_workspace_is_self_provisioned_when_the_parent_is_writable( assert stat.S_IMODE(target.stat().st_mode) == 0o700, "must not inherit a loose umask" -def test_self_provisioning_does_not_bypass_validation( - tmp_path: Path, fake_install: Path -) -> None: - """Creating it is not the dangerous fallback (that would be inheriting the - install directory) — everything is still validated afterwards.""" - inside = fake_install / "would-be-created" - with pytest.raises(WorkspaceError, match="overlap"): - resolve_workspace(str(inside), protected_roots=[str(fake_install)]) - assert not inside.exists() or inside.is_dir() + def test_unwritable_parent_still_fails_closed_with_actionable_guidance( @@ -887,3 +879,210 @@ def test_every_upgrade_path_provisions_the_workspace() -> None: compose = (repo / "docker-compose.yml").read_text(encoding="utf-8") assert "odin-workspace:/var/lib/odin-workspace" in compose, "named volume, not a bind" + + +# --- round 4: the paths that would have broken YOUR live install ------------ + + +def test_self_provisioning_never_creates_inside_a_protected_root( + fake_install: Path, +) -> None: + """PR #239 round-4 blocker 4, reproduced by Odin: the mkdir ran BEFORE the + overlap check, so an invalid workspace was created inside the very tree + this protects and only then rejected. Fail-closed must not mean 'reject + after modifying the place we promised not to touch'. + + (The previous assertion here was a tautology that passed either way.) + """ + target = fake_install / "never-create-me" + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(target), protected_roots=[str(fake_install)]) + assert not target.exists(), "the directory must NOT have been created" + + +async def test_background_workspace_refusal_is_visible_as_an_error( + fake_install: Path, workspace: Path +) -> None: + """PR #239 round-4 blocker 3: a workspace refusal came back as a plain + string that did not match the error-prefix contract, so the tool loop + classified a refusal as a SUCCESSFUL start.""" + from src.tools.tool_text import _ERROR_RESULT_PREFIXES + + executor = _executor_with_workspace(workspace, fake_install) + registry = executor._ensure_process_registry() + workspace.chmod(0o755) + try: + result = await registry.start("localhost", "echo should-not-run") + finally: + workspace.chmod(0o700) + assert result.startswith(_ERROR_RESULT_PREFIXES), ( + f"refusal must be visible as a failure, got: {result!r}" + ) + assert "mode" in result, "the operator still needs the reason" + + +def test_self_update_preflight_provisions_before_committing(tmp_path: Path) -> None: + """PR #239 round-4 blocker 1, verified by Odin against the LIVE install: + the WebUI updater re-execs in place, so systemd never runs and + StateDirectory= never applies. Without a preflight, an update succeeds and + the first local command fails closed.""" + from src.web.api.self_update import _ensure_local_workspace_for_update + + target = tmp_path / "state" / "odin-workspace" + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + f"tools:\n local_working_dir: {target}\n", encoding="utf-8" + ) + + assert _ensure_local_workspace_for_update(str(base)) is None + assert target.is_dir() + assert stat.S_IMODE(target.stat().st_mode) == 0o700 + # Idempotent: a second update must not fail because it already exists. + assert _ensure_local_workspace_for_update(str(base)) is None + + +def test_self_update_preflight_returns_actionable_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """When it genuinely cannot provision, the updater must REFUSE with an + actionable message rather than transition to code that cannot work.""" + import src.web.api.self_update as su + + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + "tools:\n local_working_dir: /proc/definitely-not-creatable/ws\n", + encoding="utf-8", + ) + monkeypatch.setattr( + su.subprocess, + "run", + lambda *a, **k: type("R", (), {"returncode": 1, "stdout": "", "stderr": "no sudo"})(), + ) + message = su._ensure_local_workspace_for_update(str(base)) + assert message is not None + assert "install -d -m 0700" in message + + + +def test_incus_deployment_path_provisions_the_workspace() -> None: + """PR #239 round-4 blocker 2: Incus is an executable deployment path and + was missed — its unprivileged odin user cannot create /var/lib/odin-workspace.""" + repo = Path(__file__).resolve().parents[1] + script = (repo / "scripts/incus-deploy.sh").read_text(encoding="utf-8") + assert "/var/lib/odin-workspace" in script + assert "chmod 0700 /var/lib/odin-workspace" in script + assert "StateDirectory=odin-workspace" in script + assert "StateDirectoryMode=0700" in script + + +def test_preflight_corrects_a_loose_mode_on_an_existing_workspace( + tmp_path: Path, +) -> None: + """An update must not be refused because a previously-provisioned + workspace drifted to a loose mode — correct it and continue.""" + from src.web.api.self_update import _ensure_local_workspace_for_update + + target = tmp_path / "ws" + target.mkdir(mode=0o755) + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + f"tools:\n local_working_dir: {target}\n", encoding="utf-8" + ) + assert _ensure_local_workspace_for_update(str(base)) is None + assert stat.S_IMODE(target.stat().st_mode) == 0o700 + + +def test_preflight_tolerates_an_unreadable_config(tmp_path: Path) -> None: + """A malformed or unreadable config must not block the preflight — it falls + back to the schema default rather than refusing the update outright.""" + from src.web.api.self_update import _ensure_local_workspace_for_update + + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text("tools: [not, a, mapping\n", encoding="utf-8") + # Resolves the schema default (a temp dir under the session fixture), so + # this succeeds rather than raising on the malformed YAML. + assert _ensure_local_workspace_for_update(str(base)) is None + + +def test_preflight_accepts_a_sudo_provisioned_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Packaged installs run an unprivileged service account under a root-owned + /var/lib, so direct creation fails and the preflight falls back to sudo. A + successful sudo run must be accepted rather than reported as a failure.""" + import os as _os + + import src.web.api.self_update as su + + target = tmp_path / "sudo-made" / "ws" + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + f"tools:\n local_working_dir: {target}\n", encoding="utf-8" + ) + + # Direct creation fails, exactly as it does under a root-owned /var/lib. + def _refuse_mkdir(self: Path, *a: object, **k: object) -> None: + raise PermissionError("permission denied") + + monkeypatch.setattr(Path, "mkdir", _refuse_mkdir) + + def _fake_sudo(*_a: object, **_k: object) -> object: + _os.makedirs(target, mode=0o700, exist_ok=True) + return type("R", (), {"returncode": 0, "stdout": "", "stderr": ""})() + + monkeypatch.setattr(su.subprocess, "run", _fake_sudo) + assert su._ensure_local_workspace_for_update(str(base)) is None + assert target.is_dir() + + + +def test_preflight_reports_an_uncorrectable_mode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """If an existing workspace has a loose mode that cannot be corrected, the + updater refuses with the reason rather than proceeding into code that will + reject it on the first command.""" + from src.web.api.self_update import _ensure_local_workspace_for_update + + target = tmp_path / "ws" + target.mkdir(mode=0o755) + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + f"tools:\n local_working_dir: {target}\n", encoding="utf-8" + ) + + def _refuse_chmod(self: Path, mode: int) -> None: + raise OSError("read-only filesystem") + + monkeypatch.setattr(Path, "chmod", _refuse_chmod) + message = _ensure_local_workspace_for_update(str(base)) + assert message is not None + assert "mode could not be corrected" in message + + +def test_preflight_survives_sudo_being_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """sudo may not exist at all; the preflight must return actionable guidance + rather than propagating the OSError out of the update handler.""" + import src.web.api.self_update as su + + base = tmp_path / "install" + base.mkdir() + (base / "config.yml").write_text( + "tools:\n local_working_dir: /proc/nope/ws\n", encoding="utf-8" + ) + + def _no_sudo(*_a: object, **_k: object) -> object: + raise FileNotFoundError("sudo not installed") + + monkeypatch.setattr(su.subprocess, "run", _no_sudo) + message = su._ensure_local_workspace_for_update(str(base)) + assert message is not None + assert "install -d -m 0700" in message diff --git a/tests/test_web_api_self_update.py b/tests/test_web_api_self_update.py index b81a930c..5bc97537 100644 --- a/tests/test_web_api_self_update.py +++ b/tests/test_web_api_self_update.py @@ -157,6 +157,32 @@ def _stub_call_later(delay, cb, *a): # main() re-execs in place after shutdown only because this was set assert restart.restart_requested() is True + async def test_refuses_when_the_workspace_cannot_be_provisioned(self): + """PR #239 round-4 blocker 1: this updater re-execs IN PLACE, so systemd + never starts the service and never applies StateDirectory=. If the local + command workspace cannot be provisioned, the update must be REFUSED — + otherwise it completes and Odin cannot run a single local command + afterwards (verified against the live install during review).""" + with patch( + "src.web.api.self_update._ensure_local_workspace_for_update", + return_value=( + "Create it before updating: sudo install -d -m 0700 " + "-o odin -g odin /var/lib/odin-workspace" + ), + ), patch("subprocess.run", side_effect=_run_ok) as run: + async with TestClient(TestServer(_app())) as c: + r = await c.post("/api/update/apply", json={"version": "latest"}) + assert r.status == 409 + body = await r.json() + assert "update refused" in body["error"] + assert "install -d -m 0700" in body["error"], "must stay actionable" + # Refusal happens BEFORE the repository is touched. + assert not any( + "merge" in " ".join(map(str, call.args[0])) + for call in run.call_args_list + if call.args and isinstance(call.args[0], list) + ), "the update must not have been committed" + async def test_unexpected_exception_500(self): # subprocess raising mid-flow (inside the try) surfaces as a 500 def _boom(cmd, **kw): From db78973b9d436b4d0acc84739c900871778b9021 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 16:06:56 -0400 Subject: [PATCH 06/21] =?UTF-8?q?fix:=20PR=20#239=20round=205=20=E2=80=94?= =?UTF-8?q?=20startup=20migration=20solves=20the=20bootstrap,=20one=20prov?= =?UTF-8?q?isioning=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blockers were right, and the first is one I had no answer for. 1. BOOTSTRAP PARADOX. A preflight living only in the self-updater cannot bootstrap itself: the update that INTRODUCES the preflight is executed by the previous release's handler, which has none. It merges, re-execs, and the new code finds no workspace. Verified against a live install — that first upgrade would have failed closed on every local command. Fixed where it actually works: a startup migration in __main__, running after the real configuration loads and before the bot (and therefore any command service) is constructed. That runs in the NEW code on the restart following any update, however the update arrived — .deb, git self-update, Incus, Docker or source. Provisioning failure is logged, never fatal: an unusable workspace must not stop Odin from starting and answering on Discord, and local commands still fail closed individually with the same actionable error. Pinned by a test asserting the ordering (config load -> provision -> bot construction), so the migration cannot drift after command service starts. 2. TWO CONTRACTS. The updater helper reimplemented a weaker validation — it reparsed config.yml instead of using live config, ignored an alternate config path, skipped environment substitution, permitted relative paths, followed workspace symlinks, and checked neither protected-root overlap nor ownership, while creating with parents=True. Four concrete mismatches were reproduced where it accepted what the runtime rejects, and it could provision a different directory from the one the restarted executor uses. There is now ONE authoritative routine, workspace.provision_workspace(): everything judgeable from the path alone (absolute, non-symlink, no protected-root overlap) is checked BEFORE creation, creation is direct then sudo, and the full resolve_workspace() contract is applied afterwards. Both the startup migration and the updater call it, and the updater feeds it the LIVE bot configuration so the path validated is the path the restarted process will use. All four mismatch cases are pinned, including that a refused workspace leaves nothing created. Mutation-verified: removing the startup migration, the live-config read, or the pre-creation overlap check each fails its test. Gates, all four as CI runs them: 7567 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. --- src/__main__.py | 55 ++++++++ src/tools/workspace.py | 86 ++++++++++++ src/web/api/self_update.py | 109 +++++++-------- tests/test_local_workspace.py | 253 +++++++++++++++++----------------- 4 files changed, 318 insertions(+), 185 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index c9304a18..6464d8bd 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -113,6 +113,38 @@ def main() -> None: log = get_logger("main") log.info("Starting Odin") + # STARTUP MIGRATION — must run after the real configuration is loaded and + # before any command service begins. + # + # Local user commands run in a validated workspace outside the install and + # refuse to run without one. A preflight in the self-updater cannot + # bootstrap that: the update which INTRODUCES the preflight is executed by + # the previous release's handler, which has none, so the very first upgrade + # would re-exec into code whose workspace was never created (PR #239 + # round-5 review, verified against a live install). Provisioning here runs + # in the NEW code on the restart that follows any update, however the + # update arrived. + # + # Failure is logged, not fatal: an unusable workspace must not prevent + # Odin from starting and answering on Discord. Local commands then fail + # closed individually, with the same actionable error. + try: + from src.tools.workspace import WorkspaceError, provision_workspace, provisioning_hint + + workspace = provision_workspace( + config.tools.local_working_dir, + protected_roots=_command_protected_roots(config), + ) + log.info("Local command workspace ready: %s", workspace) + except WorkspaceError as exc: + log.error( + "Local command workspace unusable — local commands will refuse to run: %s. %s", + exc, + provisioning_hint(config.tools.local_working_dir), + ) + except Exception as exc: # never block startup on provisioning + log.error("Local command workspace provisioning failed unexpectedly: %s", exc) + health = HealthServer( port=config.web.port, webhook_config=config.webhook, @@ -280,3 +312,26 @@ async def shutdown() -> None: if __name__ == "__main__": main() + + +def _command_protected_roots(config) -> list[str]: + """Install root plus canonical live-data roots, mirroring the executor. + + Kept beside the startup migration so both the migration and the executor + protect the same directories; the executor derives its own at call time + from the same declared paths. + """ + from pathlib import Path + + roots = [str(Path(__file__).resolve().parents[1])] + declared = [ + (getattr(config.tools, "audit_log_path", None), True), + (getattr(config.tools, "trajectory_path", None), False), + (getattr(getattr(config, "memory", None), "path", None), True), + ] + for configured, is_file in declared: + if not isinstance(configured, str) or not configured.strip(): + continue + resolved = Path(configured.strip()).resolve() + roots.append(str(resolved.parent if is_file else resolved)) + return roots diff --git a/src/tools/workspace.py b/src/tools/workspace.py index f7bcc15d..fe55e129 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -172,3 +172,89 @@ def workspace_env(workspace: Path, base: dict[str, str] | None = None) -> dict[s env["PWD"] = str(workspace) env["OLDPWD"] = str(workspace) return env + + +def provision_workspace( + configured: str, + *, + protected_roots: Sequence[str | os.PathLike[str]] | None = None, + allow_sudo: bool = True, + owner_uid: int | None = None, +) -> Path: + """THE authoritative provision-then-validate routine. Returns the workspace. + + One implementation, used by every caller — startup migration and the + self-update preflight both go through here. A second, weaker contract + elsewhere is worse than none: it can create a directory the runtime will + then refuse, or provision a different directory from the one actually used + (PR #239 round-5 review, which reproduced four such mismatches). + + Order matters. Everything that can be judged from the path alone — + absolute, not a symlink, not overlapping a protected root — is checked + BEFORE anything is created, so a rejected configuration never leaves a + directory inside the tree this protects. Creation is attempted directly, + then via ``sudo -n`` (packaged installs configure passwordless sudo for + exactly this class of operation), and the FULL :func:`resolve_workspace` + contract is applied afterwards. + """ + if not configured or not str(configured).strip(): + raise WorkspaceError("tools.local_working_dir is empty") + + raw = Path(str(configured).strip()).expanduser() + if not raw.is_absolute(): + raise WorkspaceError(f"local_working_dir must be absolute: {configured!r}") + if raw.is_symlink(): + raise WorkspaceError(f"local_working_dir must not be a symlink: {raw}") + + target = _canonical(raw) + _reject_overlap(target, protected_roots) + + if not target.exists(): + try: + # parents=False deliberately: silently materialising a whole path + # prefix is how a typo becomes a new directory tree. + target.mkdir(mode=REQUIRED_MODE, parents=False) + target.chmod(REQUIRED_MODE) + except OSError: + if allow_sudo: + _sudo_create(target, owner_uid) + + # Full contract, including ownership, exact mode and usability. Creation + # above is never a substitute for validation. + return resolve_workspace( + str(target), + protected_roots=protected_roots, + owner_uid=owner_uid, + create_if_missing=False, + ) + + +def _sudo_create(target: Path, owner_uid: int | None) -> None: + """Best-effort privileged creation; failures fall through to validation, + which produces the actionable error.""" + import subprocess + + uid = os.getuid() if owner_uid is None else owner_uid + try: + subprocess.run( + [ + "sudo", "-n", "install", "-d", + f"-m{REQUIRED_MODE:o}", + "-o", str(uid), + "-g", str(os.getgid()), + str(target), + ], + capture_output=True, + timeout=15, + check=False, + ) + except Exception: + pass + + +def provisioning_hint(configured: str) -> str: + """The operator-actionable instruction, in one place.""" + return ( + f"Create it as the service account: " + f"sudo install -d -m 0700 -o odin -g odin {configured}" + ) diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index b02b2b83..0c64a7a6 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -8,7 +8,6 @@ import asyncio import os -import stat import subprocess from pathlib import Path @@ -31,7 +30,6 @@ def _repo_root() -> str: preservation joined the wrong directory, and the bot kept reporting the pre-update version from stale package metadata. """ - import os path = os.path.dirname(os.path.abspath(__file__)) for _ in range(8): @@ -83,10 +81,8 @@ async def check_update(_request: web.Request) -> web.Response: @routes.post("/api/update/apply") async def apply_update(request: web.Request) -> web.Response: - import os import re as _re import shutil - import subprocess try: data = await request.json() except Exception: @@ -163,7 +159,7 @@ async def apply_update(request: web.Request) -> web.Response: # (PR #239 round-4 review, verified against the live install). # Provision it here, BEFORE committing to the update, and refuse # the update rather than transition to code that cannot work. - ws_error = _ensure_local_workspace_for_update(base) + ws_error = _ensure_local_workspace_for_update(bot, base) if ws_error: return web.json_response({ "error": ( @@ -260,70 +256,69 @@ async def stop_all_loops(_request: web.Request) -> web.Response: return web.json_response({"result": result}) -def _ensure_local_workspace_for_update(base: str | None = None) -> str | None: +def _ensure_local_workspace_for_update(bot=None, base: str | None = None) -> str | None: """Provision the local command workspace ahead of an in-place update. Returns None on success, or an operator-actionable message on failure. - The updater re-execs the process directly, so a packaged install's - ``StateDirectory=`` never runs and a unit file written before this feature - existed never gains it. Without this preflight an update completes and the - first local command fails closed. + Delegates to the SINGLE authoritative implementation rather than + reimplementing a weaker contract here. An independent copy accepted + workspaces the runtime then rejected — a relative path, a symlink, one + inside the install — and could create a directory the restarted executor + would refuse, or provision a different directory from the one it actually + uses (PR #239 round-5 review reproduced four such mismatches). + + The configuration comes from the LIVE bot when available, so the path + checked here is the path the restarted process will use, including any + alternate config file or environment substitution already applied. """ - # The configured path is read WITHOUT constructing a full Config: - # load_config() raises SystemExit when required environment variables are - # absent, which must never happen inside a request handler, and its result - # depends on the process working directory. Read the one key directly. - configured = "" - try: - from ...config.schema import ToolsConfig + from ...tools.workspace import ( + WorkspaceError, + provision_workspace, + provisioning_hint, + ) - configured = ToolsConfig().local_working_dir - except BaseException: # pragma: no cover - schema import cannot realistically fail - configured = "/var/lib/odin-workspace" + configured = _live_workspace_setting(bot) + if not configured: + return None # nothing configured to validate; runtime default applies try: - import yaml - - config_path = Path(base) / "config.yml" if base else Path("config.yml") - if config_path.is_file(): - raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - override = (raw.get("tools") or {}).get("local_working_dir") - if isinstance(override, str) and override.strip(): - configured = override.strip() - except BaseException: - pass # an unreadable/invalid config must not block the preflight - - path = Path(configured) - if path.is_dir(): - try: - if stat.S_IMODE(path.stat().st_mode) != 0o700: - path.chmod(0o700) - except OSError as exc: - return f"{path} exists but its mode could not be corrected: {exc}" + provision_workspace(configured, protected_roots=_live_protected_roots(bot, base)) return None + except WorkspaceError as exc: + return f"{exc} {provisioning_hint(configured)}" + except Exception as exc: # pragma: no cover - defensive + return f"{exc} {provisioning_hint(configured)}" - try: - path.mkdir(mode=0o700, parents=True) - path.chmod(0o700) - return None - except OSError: - pass - # Unprivileged service account under a root-owned parent: try sudo, which - # packaged installs configure for exactly this class of operation. +def _live_workspace_setting(bot) -> str: + """The workspace path the RESTARTED process will actually use.""" try: - result = subprocess.run( - ["sudo", "-n", "install", "-d", "-m", "0700", - "-o", str(os.getuid()), "-g", str(os.getgid()), str(path)], - capture_output=True, text=True, timeout=15, - ) - if result.returncode == 0 and path.is_dir(): - return None + configured = bot.config.tools.local_working_dir + if isinstance(configured, str) and configured.strip(): + return configured.strip() except Exception: pass + try: + from ...config.schema import ToolsConfig - return ( - f"Create it before updating: sudo install -d -m 0700 " - f"-o odin -g odin {path}" - ) + return ToolsConfig().local_working_dir + except Exception: # pragma: no cover - schema import cannot realistically fail + return "/var/lib/odin-workspace" + + +def _live_protected_roots(bot, base: str | None) -> list[str]: + """Install root plus canonical live-data roots, from the live config.""" + roots: list[str] = [] + roots.append(str(Path(base).resolve()) if base else str(Path(__file__).resolve().parents[3])) + tools = getattr(getattr(bot, "config", None), "tools", None) + declared = [ + (getattr(tools, "audit_log_path", None), True), + (getattr(tools, "trajectory_path", None), False), + ] + for configured, is_file in declared: + if not isinstance(configured, str) or not configured.strip(): + continue + resolved = Path(configured.strip()).resolve() + roots.append(str(resolved.parent if is_file else resolved)) + return roots diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index d2e09e7d..afe4dcc9 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -921,51 +921,6 @@ async def test_background_workspace_refusal_is_visible_as_an_error( assert "mode" in result, "the operator still needs the reason" -def test_self_update_preflight_provisions_before_committing(tmp_path: Path) -> None: - """PR #239 round-4 blocker 1, verified by Odin against the LIVE install: - the WebUI updater re-execs in place, so systemd never runs and - StateDirectory= never applies. Without a preflight, an update succeeds and - the first local command fails closed.""" - from src.web.api.self_update import _ensure_local_workspace_for_update - - target = tmp_path / "state" / "odin-workspace" - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - f"tools:\n local_working_dir: {target}\n", encoding="utf-8" - ) - - assert _ensure_local_workspace_for_update(str(base)) is None - assert target.is_dir() - assert stat.S_IMODE(target.stat().st_mode) == 0o700 - # Idempotent: a second update must not fail because it already exists. - assert _ensure_local_workspace_for_update(str(base)) is None - - -def test_self_update_preflight_returns_actionable_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """When it genuinely cannot provision, the updater must REFUSE with an - actionable message rather than transition to code that cannot work.""" - import src.web.api.self_update as su - - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - "tools:\n local_working_dir: /proc/definitely-not-creatable/ws\n", - encoding="utf-8", - ) - monkeypatch.setattr( - su.subprocess, - "run", - lambda *a, **k: type("R", (), {"returncode": 1, "stdout": "", "stderr": "no sudo"})(), - ) - message = su._ensure_local_workspace_for_update(str(base)) - assert message is not None - assert "install -d -m 0700" in message - - - def test_incus_deployment_path_provisions_the_workspace() -> None: """PR #239 round-4 blocker 2: Incus is an executable deployment path and was missed — its unprivileged odin user cannot create /var/lib/odin-workspace.""" @@ -977,112 +932,154 @@ def test_incus_deployment_path_provisions_the_workspace() -> None: assert "StateDirectoryMode=0700" in script -def test_preflight_corrects_a_loose_mode_on_an_existing_workspace( - tmp_path: Path, -) -> None: - """An update must not be refused because a previously-provisioned - workspace drifted to a loose mode — correct it and continue.""" - from src.web.api.self_update import _ensure_local_workspace_for_update +# --- round 5: one authoritative provisioner, and the bootstrap it must survive - target = tmp_path / "ws" - target.mkdir(mode=0o755) - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - f"tools:\n local_working_dir: {target}\n", encoding="utf-8" - ) - assert _ensure_local_workspace_for_update(str(base)) is None - assert stat.S_IMODE(target.stat().st_mode) == 0o700 +def _fake_bot(workspace: Path, install: Path): + """A bot-shaped object exposing only what the preflight reads.""" + class _Tools: + local_working_dir = str(workspace) + audit_log_path = str(install / "data" / "audit.jsonl") + trajectory_path = str(install / "data" / "trajectories") -def test_preflight_tolerates_an_unreadable_config(tmp_path: Path) -> None: - """A malformed or unreadable config must not block the preflight — it falls - back to the schema default rather than refusing the update outright.""" - from src.web.api.self_update import _ensure_local_workspace_for_update + class _Config: + tools = _Tools() - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text("tools: [not, a, mapping\n", encoding="utf-8") - # Resolves the schema default (a temp dir under the session fixture), so - # this succeeds rather than raising on the malformed YAML. - assert _ensure_local_workspace_for_update(str(base)) is None + class _Bot: + config = _Config() + return _Bot() -def test_preflight_accepts_a_sudo_provisioned_workspace( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Packaged installs run an unprivileged service account under a root-owned - /var/lib, so direct creation fails and the preflight falls back to sudo. A - successful sudo run must be accepted rather than reported as a failure.""" - import os as _os - import src.web.api.self_update as su +def test_provision_workspace_is_the_single_contract(tmp_path: Path, fake_install: Path) -> None: + """provision_workspace creates AND fully validates, so no caller can accept + a workspace the runtime would reject.""" + from src.tools.workspace import provision_workspace - target = tmp_path / "sudo-made" / "ws" - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - f"tools:\n local_working_dir: {target}\n", encoding="utf-8" - ) + target = tmp_path / "ws" + result = provision_workspace(str(target), protected_roots=[str(fake_install)]) + assert result == target.resolve() + assert stat.S_IMODE(target.stat().st_mode) == 0o700 - # Direct creation fails, exactly as it does under a root-owned /var/lib. - def _refuse_mkdir(self: Path, *a: object, **k: object) -> None: - raise PermissionError("permission denied") - monkeypatch.setattr(Path, "mkdir", _refuse_mkdir) +@pytest.mark.parametrize( + "case", + ["relative", "symlink", "inside_install", "wrong_owner"], + ids=["relative-path", "workspace-symlink", "inside-install", "wrong-owner"], +) +def test_provisioner_rejects_everything_the_runtime_rejects( + case: str, tmp_path: Path, fake_install: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """PR #239 round-5 blocker 2: the updater had a SECOND, weaker contract and + accepted four things the runtime refuses. One implementation now serves + both, so these cannot diverge again.""" + from src.tools.workspace import provision_workspace + + if case == "relative": + monkeypatch.chdir(tmp_path) + configured = "relative-ws" + elif case == "symlink": + real = tmp_path / "real" + real.mkdir(mode=0o700) + link = tmp_path / "link" + link.symlink_to(real) + configured = str(link) + elif case == "inside_install": + configured = str(fake_install / "ws") + else: + target = tmp_path / "owned-elsewhere" + target.mkdir(mode=0o700) + configured = str(target) + + kwargs = {"protected_roots": [str(fake_install)], "allow_sudo": False} + if case == "wrong_owner": + kwargs["owner_uid"] = os.getuid() + 4242 + + with pytest.raises(WorkspaceError): + provision_workspace(configured, **kwargs) - def _fake_sudo(*_a: object, **_k: object) -> object: - _os.makedirs(target, mode=0o700, exist_ok=True) - return type("R", (), {"returncode": 0, "stdout": "", "stderr": ""})() + if case == "inside_install": + assert not (fake_install / "ws").exists(), "must not create inside the install" + if case == "relative": + assert not (tmp_path / "relative-ws").exists(), "must not create a relative workspace" - monkeypatch.setattr(su.subprocess, "run", _fake_sudo) - assert su._ensure_local_workspace_for_update(str(base)) is None - assert target.is_dir() +def test_startup_migration_provisions_before_commands_are_served() -> None: + """PR #239 round-5 blocker 1 — the bootstrap paradox. + + The self-update preflight is NEW code, so the update that installs it is + executed by the PREVIOUS release's handler, which has no preflight. Only a + startup migration in the incoming code can bootstrap the workspace, because + that runs after re-exec however the update arrived. + """ + main_src = (Path(__file__).resolve().parents[1] / "src/__main__.py").read_text( + encoding="utf-8" + ) + assert "provision_workspace(" in main_src, "startup must provision the workspace" + provision_at = main_src.index("provision_workspace(") + bot_at = main_src.index("bot = OdinBot(config)") + config_at = main_src.index("config = load_config(config_path)") + assert config_at < provision_at < bot_at, ( + "provisioning must run after the real config loads and before the bot " + "(and therefore command service) is constructed" + ) + # Failure must not prevent Odin from starting and answering on Discord. + assert "never block startup" in main_src or "not fatal" in main_src -def test_preflight_reports_an_uncorrectable_mode( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_preflight_uses_the_live_config_not_a_reparsed_file( + tmp_path: Path, fake_install: Path ) -> None: - """If an existing workspace has a loose mode that cannot be corrected, the - updater refuses with the reason rather than proceeding into code that will - reject it on the first command.""" - from src.web.api.self_update import _ensure_local_workspace_for_update + """The preflight must validate the path the RESTARTED process will use. + Re-reading config.yml missed alternate config paths and environment + substitution, and could provision a different directory entirely.""" + from src.web.api.self_update import _live_workspace_setting - target = tmp_path / "ws" - target.mkdir(mode=0o755) - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - f"tools:\n local_working_dir: {target}\n", encoding="utf-8" - ) + workspace = tmp_path / "live-ws" + workspace.mkdir(mode=0o700) + bot = _fake_bot(workspace, fake_install) + assert _live_workspace_setting(bot) == str(workspace) - def _refuse_chmod(self: Path, mode: int) -> None: - raise OSError("read-only filesystem") - monkeypatch.setattr(Path, "chmod", _refuse_chmod) - message = _ensure_local_workspace_for_update(str(base)) +async def test_preflight_refuses_a_workspace_the_runtime_would_reject( + fake_install: Path, +) -> None: + """End-to-end: a workspace inside the install must be refused by the + preflight, and nothing may be created.""" + from src.web.api.self_update import _ensure_local_workspace_for_update + + target = fake_install / "ws-inside" + bot = _fake_bot(target, fake_install) + message = _ensure_local_workspace_for_update(bot, str(fake_install)) assert message is not None - assert "mode could not be corrected" in message + assert "overlap" in message + assert "install -d -m 0700" in message + assert not target.exists(), "a refused preflight must not create anything" -def test_preflight_survives_sudo_being_unavailable( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_preflight_falls_back_to_the_schema_default_without_a_bot( + tmp_path: Path, ) -> None: - """sudo may not exist at all; the preflight must return actionable guidance - rather than propagating the OSError out of the update handler.""" - import src.web.api.self_update as su + """Called without a live bot (or with one that cannot answer), the setting + still resolves to the schema default rather than guessing or crashing.""" + from src.config.schema import ToolsConfig + from src.web.api.self_update import _live_workspace_setting - base = tmp_path / "install" - base.mkdir() - (base / "config.yml").write_text( - "tools:\n local_working_dir: /proc/nope/ws\n", encoding="utf-8" - ) + class _Broken: + @property + def config(self): # noqa: ANN201 - deliberately raises + raise RuntimeError("bot not ready") - def _no_sudo(*_a: object, **_k: object) -> object: - raise FileNotFoundError("sudo not installed") + assert _live_workspace_setting(None) == ToolsConfig().local_working_dir + assert _live_workspace_setting(_Broken()) == ToolsConfig().local_working_dir - monkeypatch.setattr(su.subprocess, "run", _no_sudo) - message = su._ensure_local_workspace_for_update(str(base)) - assert message is not None - assert "install -d -m 0700" in message + +def test_preflight_skips_validation_when_nothing_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With no configured workspace at all there is nothing to validate, and + the update must not be blocked on a value that does not exist.""" + import src.web.api.self_update as su + + monkeypatch.setattr(su, "_live_workspace_setting", lambda _bot: "") + assert su._ensure_local_workspace_for_update(None, None) is None From ecb72c652b180c362879898913bb55d48b077f52 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 16:37:57 -0400 Subject: [PATCH 07/21] =?UTF-8?q?fix:=20PR=20#239=20round=206=20=E2=80=94?= =?UTF-8?q?=20entrypoint=20ordering,=20and=20one=20derivation=20of=20prote?= =?UTF-8?q?cted=20roots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blockers were real, and the first is embarrassing in a way worth naming: the round-5 fix did not run at all. 1. THE STARTUP MIGRATION NEVER RAN under `python -m src`. The entrypoint guard sat above `_command_protected_roots`, so executing the module as `__main__` called main() before that `def` was reached. The resulting NameError was swallowed by the migration's own deliberately-nonfatal handler: Odin started normally, logged one line, and the workspace was never created — exactly the first-update bootstrap failure round 5 existed to fix, reintroduced by statement order. My round-5 test compared SOURCE positions, which cannot see this; Python's execution order is the thing that matters. The guard is now the last statement in the module with a comment saying why, and the new test runs the real entrypoint in a subprocess — patching OdinBot to record whether the workspace exists at the moment of construction — so the ordering property is asserted by execution, not by reading. 2. THREE DERIVATIONS OF THE PROTECTED ROOTS, and they disagreed. The executor protected the live memory.json supplied by wiring; the startup migration read `config.memory.path`, which does not exist on Config, so it protected nothing; the updater omitted memory entirely. With audit and trajectory paths relocated, the preflight therefore CREATED a workspace inside the live-data directory, reported success, and handed over to an executor that refused every local command — mutating live data and stranding commands in one step. There is now one function, workspace.command_protected_roots(), used by all three, and the live memory path is a shared constant that wiring imports rather than re-spells. The relocated-audit/trajectory scenario is pinned against each caller independently, plus a test asserting all three agree on the live-data root, each asserting nothing was created. Mutation-verified: restoring the guard above the helper, reverting the startup helper to `config.memory.path`, or dropping memory from the updater's roots each fails its test. Gates, all four as CI runs them: 7572 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. --- src/__main__.py | 50 ++++---- src/discord/wiring.py | 6 +- src/tools/executor.py | 62 ++++----- src/tools/workspace.py | 48 +++++++ src/web/api/self_update.py | 37 ++++-- tests/test_local_workspace.py | 233 +++++++++++++++++++++++++++++++++- 6 files changed, 360 insertions(+), 76 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index 6464d8bd..2a03fe97 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -310,28 +310,34 @@ async def shutdown() -> None: sys.exit(exit_code) -if __name__ == "__main__": - main() - - def _command_protected_roots(config) -> list[str]: - """Install root plus canonical live-data roots, mirroring the executor. + """Install root plus canonical live-data roots for the startup migration. + + Delegates to the ONE shared derivation so startup, the self-update + preflight, and the executor protect exactly the same directories. Deriving + them separately here silently protected nothing (``Config`` has no + ``memory`` section) while the executor protected live memory.json — so a + workspace beside it was provisioned at startup and then refused on every + command (PR #239 round-6 review). - Kept beside the startup migration so both the migration and the executor - protect the same directories; the executor derives its own at call time - from the same declared paths. + ``memory_path`` is intentionally left at its default: production wiring + hardcodes that path, and startup runs before wiring exists. """ - from pathlib import Path - - roots = [str(Path(__file__).resolve().parents[1])] - declared = [ - (getattr(config.tools, "audit_log_path", None), True), - (getattr(config.tools, "trajectory_path", None), False), - (getattr(getattr(config, "memory", None), "path", None), True), - ] - for configured, is_file in declared: - if not isinstance(configured, str) or not configured.strip(): - continue - resolved = Path(configured.strip()).resolve() - roots.append(str(resolved.parent if is_file else resolved)) - return roots + from src.tools.workspace import command_protected_roots + + return command_protected_roots( + Path(__file__).resolve().parents[1], + audit_log_path=getattr(config.tools, "audit_log_path", None), + trajectory_path=getattr(config.tools, "trajectory_path", None), + ) + + +# The entrypoint guard MUST stay the last statement in this module. Python +# executes a module top-to-bottom, so a guard placed above a helper runs main() +# before that helper's `def` is reached: the startup migration raised NameError +# and its own nonfatal handler swallowed it, leaving the workspace uncreated and +# resurrecting the first-update bootstrap failure this migration exists to fix +# (PR #239 round-6 review). tests/test_local_workspace.py executes `python -m src` +# for real to keep this honest. +if __name__ == "__main__": + main() diff --git a/src/discord/wiring.py b/src/discord/wiring.py index 1e757f13..3b37b49c 100644 --- a/src/discord/wiring.py +++ b/src/discord/wiring.py @@ -46,6 +46,7 @@ from ..sessions import SessionManager from ..tools import SkillManager, ToolExecutor from ..tools.autonomous_loop import LoopManager +from ..tools.workspace import DEFAULT_MEMORY_PATH from ..trajectories.saver import TrajectorySaver from .channel_config import ChannelConfigManager from .channel_logger import ChannelLogger @@ -206,7 +207,10 @@ def build_services(config: Config) -> BotServices: # noqa: PLR0915 — linear c ) sessions.load() - memory_path = "./data/memory.json" + # Imported, not re-spelled: the startup migration and the self-update + # preflight protect this file before/without wiring, and three + # independent spellings is how they fell out of sync (PR #239 round 6). + memory_path = DEFAULT_MEMORY_PATH channel_config = ChannelConfigManager("./data/channel_config.json") # Passive channel logger — writes ALL guild messages to JSONL (zero LLM tokens) diff --git a/src/tools/executor.py b/src/tools/executor.py index 8d828049..5f4d7249 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -45,7 +45,7 @@ run_ssh_command, ) from .ssh_pool import SSHConnectionPool -from .workspace import resolve_workspace +from .workspace import DEFAULT_MEMORY_PATH, command_protected_roots, resolve_workspace log = get_logger("tools") @@ -161,9 +161,10 @@ def __init__( ) -> None: self.config = config or ToolsConfig() self._email_config = email_config - # The user-command workspace is resolved once and cached — restart- - # required by design, since swapping workspaces mid-run would break the - # cross-command continuity that makes this invisible to normal use. + # The configured workspace VALUE is restart-required, but it is + # re-validated on every local command rather than cached: existence, + # type, ownership and mode are mutable filesystem state (see + # _ensure_local_workspace). # # Resolution is LAZY (first local command) rather than at construction: # constructing an executor must not depend on deployment having created @@ -323,42 +324,25 @@ def _protected_roots(self) -> list[str]: and for source checkouts, and packaged ``/opt/odin/data`` is a symlink to ``/var/lib/odin`` — so the live-data root is taken from the actual configured data paths and canonicalized, not string-joined. + + Derivation itself lives in workspace.command_protected_roots so the + startup migration and the self-update preflight protect exactly the + same directories. When each caller derived its own, the preflight + accepted (and created) a workspace beside live memory.json that the + executor then rejected (PR #239 round-6 review). """ - roots: list[str] = [] - # Install root: the package's own location (…/src/tools/executor.py). - roots.append(str(Path(__file__).resolve().parents[2])) - - # Live-data roots, canonicalized so a symlinked data dir (packaged - # /opt/odin/data -> /var/lib/odin) cannot be smuggled past the overlap - # check. Each path is classified by DECLARED semantics, never guessed - # from its name: a `Path.suffix` heuristic misreads dotted directories - # and extensionless files (PR #239 round-2 review). - declared: list[tuple[object, bool]] = [ - (getattr(self.config, "audit_log_path", None), True), # file - (getattr(self.config, "trajectory_path", None), False), # directory - # The live memory.json is supplied by wiring, not ToolsConfig. It is - # the most valuable file in the tree, and omitting it meant a - # workspace beside it was accepted whenever audit/trajectory paths - # were relocated (reproduced in review). + return command_protected_roots( + # Install root: the package's own location (…/src/tools/executor.py). + Path(__file__).resolve().parents[2], + audit_log_path=getattr(self.config, "audit_log_path", None), + trajectory_path=getattr(self.config, "trajectory_path", None), + # The live memory.json is supplied by wiring, not ToolsConfig. # getattr-guarded: the sanctioned __new__ patch seam builds - # executors without __init__, so this attribute may not exist. - (getattr(self, "_memory_path", None), True), # file - ] - for configured, is_file in declared: - if configured is None: - continue - text = str(configured).strip() - if not text: - continue - candidate = Path(text) - # Resolve the COMPLETE declared path first. Taking .parent before - # canonicalization protects the alias directory rather than the - # target: /aliases/memory.json -> /live-data/memory.json would - # protect /aliases and accept /live-data/workspace (reproduced in - # review as UNSAFE_ACCEPTED). - resolved = candidate.resolve() - roots.append(str(resolved.parent if is_file else resolved)) - return roots + # executors without __init__, so this attribute may not exist — + # falling back to the shared default rather than to no protection. + memory_path=getattr(self, "_memory_path", None) + or DEFAULT_MEMORY_PATH, + ) def get_workspace_metrics(self) -> dict[str, float]: """Usage of the local command workspace, for Prometheus. @@ -401,7 +385,7 @@ def get_workspace_metrics(self) -> dict[str, float]: return metrics def _ensure_local_workspace(self) -> str: - """Resolve (once) the validated cwd for local user commands. + """Resolve and re-validate the cwd for local user commands. Raises :class:`WorkspaceError` if the configured directory is unusable. Deliberately no fallback: inheriting the process cwd is exactly the diff --git a/src/tools/workspace.py b/src/tools/workspace.py index fe55e129..d54fdfd4 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -41,6 +41,15 @@ # and writable but not readable, which breaks ordinary workspace use. REQUIRED_MODE = 0o700 +# The live working-memory file. Production wiring hardcodes this path rather +# than taking it from config, so it is defined HERE and imported there: the +# startup migration and the self-update preflight both run before (or without) +# wiring and must protect the same file the executor protects. Three +# independent spellings of "the protected roots" is exactly how round 6 found +# the updater creating a workspace beside live memory.json and reporting +# success, then handing over to an executor that refused every command. +DEFAULT_MEMORY_PATH = "./data/memory.json" + class WorkspaceError(RuntimeError): """The configured local workspace is unusable. Never fall back to cwd.""" @@ -69,6 +78,45 @@ def _reject_overlap( ) +def command_protected_roots( + install_root: str | os.PathLike[str], + *, + audit_log_path: object = None, + trajectory_path: object = None, + memory_path: object = DEFAULT_MEMORY_PATH, +) -> list[str]: + """THE derivation of directories a command workspace must never overlap. + + Every caller — executor, startup migration, self-update preflight — uses + this one function, so a workspace accepted by one is accepted by all. When + they each derived their own, the preflight approved (and created) a + workspace inside the live-data directory that the executor then rejected, + which both mutated live data and stranded local commands. + + Paths are classified by DECLARED semantics, never guessed from the name: a + ``Path.suffix`` heuristic misreads dotted directories and extensionless + files. Each declared path is resolved COMPLETELY before ``.parent`` is + taken, because taking the parent first protects the alias directory rather + than the target (``/aliases/memory.json -> /live-data/memory.json`` would + protect ``/aliases`` and accept ``/live-data/workspace``). + """ + roots = [str(_canonical(install_root))] + declared: list[tuple[object, bool]] = [ + (audit_log_path, True), # file + (trajectory_path, False), # directory + (memory_path, True), # file + ] + for configured, is_file in declared: + if configured is None: + continue + text = str(configured).strip() + if not text: + continue + resolved = _canonical(text) + roots.append(str(resolved.parent if is_file else resolved)) + return roots + + def resolve_workspace( configured: str, *, diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index 0c64a7a6..f3b1d276 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -308,17 +308,28 @@ def _live_workspace_setting(bot) -> str: def _live_protected_roots(bot, base: str | None) -> list[str]: - """Install root plus canonical live-data roots, from the live config.""" - roots: list[str] = [] - roots.append(str(Path(base).resolve()) if base else str(Path(__file__).resolve().parents[3])) + """Install root plus canonical live-data roots, from the LIVE config. + + Delegates to the ONE shared derivation so the preflight cannot approve a + workspace the restarted executor refuses. Deriving them here separately + omitted live memory.json entirely, so with audit/trajectory paths relocated + the updater created a workspace beside memory.json, reported success, and + handed over to an executor that rejected every local command (PR #239 + round-6 review, reproduced). + + The live memory path is read from the running executor when reachable — + that is the value wiring actually supplied — and falls back to the shared + default otherwise. + """ + from src.tools.workspace import DEFAULT_MEMORY_PATH, command_protected_roots + tools = getattr(getattr(bot, "config", None), "tools", None) - declared = [ - (getattr(tools, "audit_log_path", None), True), - (getattr(tools, "trajectory_path", None), False), - ] - for configured, is_file in declared: - if not isinstance(configured, str) or not configured.strip(): - continue - resolved = Path(configured.strip()).resolve() - roots.append(str(resolved.parent if is_file else resolved)) - return roots + memory_path = getattr(getattr(bot, "tool_executor", None), "_memory_path", None) + return command_protected_roots( + Path(base).resolve() if base else Path(__file__).resolve().parents[3], + audit_log_path=getattr(tools, "audit_log_path", None), + trajectory_path=getattr(tools, "trajectory_path", None), + memory_path=memory_path or DEFAULT_MEMORY_PATH, + ) + + diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index afe4dcc9..24fd3d52 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -20,16 +20,27 @@ from __future__ import annotations +import json import os import stat +import subprocess +import sys import zipfile from pathlib import Path +from types import SimpleNamespace import pytest +from src.config.schema import ToolsConfig +from src.tools.executor import ToolExecutor from src.tools.process_manager import ProcessRegistry from src.tools.ssh import run_local_command -from src.tools.workspace import WorkspaceError, resolve_workspace, workspace_env +from src.tools.workspace import ( + WorkspaceError, + provision_workspace, + resolve_workspace, + workspace_env, +) @pytest.fixture @@ -1083,3 +1094,223 @@ def test_preflight_skips_validation_when_nothing_is_configured( monkeypatch.setattr(su, "_live_workspace_setting", lambda _bot: "") assert su._ensure_local_workspace_for_update(None, None) is None + + +# --- Round 6: the entrypoint and the protected-root contract ------------------ + +_ENTRYPOINT_DRIVER = r""" +import json, os, re, runpy, sys +from pathlib import Path + +repo, tmp, ws = sys.argv[1], Path(sys.argv[2]), Path(sys.argv[3]) + +# Env vars the shipped template substitutes; values are irrelevant, presence is not. +template = (Path(repo) / "config.yml").read_text() +for name in set(re.findall(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", template)): + os.environ.setdefault(name, "test-value") + +import yaml +cfg = tmp / "config.yml" +parsed = yaml.safe_load(template) +parsed.setdefault("tools", {})["local_working_dir"] = str(ws) +cfg.write_text(yaml.safe_dump(parsed)) + +os.chdir(tmp) +observed = {} + +class _StopStartup(Exception): + pass + +import src.discord.client as client_module + +class _RecordingBot: + def __init__(self, *a, **k): + # THE assertion point: by the time anything that can run a command + # exists, the workspace must already be there. + observed["workspace_at_bot_construction"] = ws.is_dir() + raise _StopStartup() + +client_module.OdinBot = _RecordingBot +sys.argv = ["src", str(cfg)] +try: + runpy.run_module("src", run_name="__main__") +except _StopStartup: + observed["reached_bot_construction"] = True +except BaseException as exc: + observed["error"] = f"{type(exc).__name__}: {exc}" + +print("RESULT " + json.dumps(observed)) +""" + + +def test_python_m_src_provisions_the_workspace_before_the_bot_exists( + tmp_path: Path, +) -> None: + """Executes the REAL entrypoint, as `python -m src` does. + + Round-6 regression. The startup migration sat above + ``_command_protected_roots``'s ``def``, so running the module as + ``__main__`` raised NameError mid-module; the migration's own nonfatal + handler swallowed it and the workspace was never created — silently + restoring the first-update bootstrap failure the migration exists to fix. + + A source-order comparison cannot see this: it is Python's execution order + that matters, so this test runs the module for real in a subprocess and + asserts the workspace exists at the moment the bot is constructed. + """ + repo_root = Path(__file__).resolve().parents[1] + workspace = tmp_path / "workspace" + + env = dict(os.environ) + env["PYTHONPATH"] = str(repo_root) + proc = subprocess.run( + [sys.executable, "-c", _ENTRYPOINT_DRIVER, str(repo_root), str(tmp_path), str(workspace)], + capture_output=True, + text=True, + cwd=str(tmp_path), + env=env, + timeout=180, + ) + result_lines = [ln for ln in proc.stdout.splitlines() if ln.startswith("RESULT ")] + assert result_lines, ( + f"driver produced no result.\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + observed = json.loads(result_lines[-1][len("RESULT ") :]) + + assert observed.get("error") is None, observed["error"] + assert observed.get("reached_bot_construction") is True, observed + assert observed.get("workspace_at_bot_construction") is True, ( + "the workspace did not exist when the bot was constructed — the startup " + f"migration did not run: {observed}" + ) + assert workspace.is_dir() + assert stat.S_IMODE(workspace.stat().st_mode) == 0o700 + + +def _relocated_data_layout(tmp_path: Path) -> dict[str, Path]: + """The packaged shape that defeated the per-caller root derivations. + + install/data is a SYMLINK to the live-data directory (as /opt/odin/data -> + /var/lib/odin is), audit and trajectories are configured somewhere else + entirely, and the proposed workspace sits beside live memory.json. + """ + install = tmp_path / "install" + live = tmp_path / "var-lib-odin" + elsewhere = tmp_path / "elsewhere" + for directory in (install, live, elsewhere): + directory.mkdir() + (live / "memory.json").write_text("{}") + (install / "data").symlink_to(live) + return { + "install": install, + "live": live, + "memory": live / "memory.json", + "audit": elsewhere / "audit.jsonl", + "trajectory": elsewhere / "trajectories", + "workspace": live / "workspace", + } + + +def test_executor_rejects_a_workspace_beside_relocated_live_memory( + tmp_path: Path, +) -> None: + """Executor arm of the shared-contract scenario.""" + layout = _relocated_data_layout(tmp_path) + executor = ToolExecutor( + ToolsConfig( + local_working_dir=str(layout["workspace"]), + audit_log_path=str(layout["audit"]), + trajectory_path=str(layout["trajectory"]), + ), + memory_path=str(layout["memory"]), + ) + with pytest.raises(WorkspaceError): + executor._ensure_local_workspace() + assert not layout["workspace"].exists() + + +def test_startup_migration_rejects_a_workspace_beside_relocated_live_memory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Startup arm. Startup runs before wiring, so it protects the shared + default memory path — which, from the install directory, resolves through + the data symlink onto the real live-data root.""" + import src.__main__ as entrypoint + + layout = _relocated_data_layout(tmp_path) + monkeypatch.chdir(layout["install"]) + + config = SimpleNamespace( + tools=SimpleNamespace( + local_working_dir=str(layout["workspace"]), + audit_log_path=str(layout["audit"]), + trajectory_path=str(layout["trajectory"]), + ) + ) + roots = entrypoint._command_protected_roots(config) + assert str(layout["live"].resolve()) in roots + + with pytest.raises(WorkspaceError): + provision_workspace(str(layout["workspace"]), protected_roots=roots) + assert not layout["workspace"].exists() + + +def test_self_update_preflight_rejects_a_workspace_beside_relocated_live_memory( + tmp_path: Path, +) -> None: + """Updater arm — the one that previously said yes. + + It omitted live memory.json from its own root derivation, so with audit and + trajectory paths relocated it CREATED the workspace inside the live-data + directory, reported success, and handed over to an executor that refused + every local command (PR #239 round-6 review, reproduced). + """ + from src.web.api.self_update import _ensure_local_workspace_for_update + + layout = _relocated_data_layout(tmp_path) + bot = SimpleNamespace( + config=SimpleNamespace( + tools=SimpleNamespace( + local_working_dir=str(layout["workspace"]), + audit_log_path=str(layout["audit"]), + trajectory_path=str(layout["trajectory"]), + ) + ), + tool_executor=SimpleNamespace(_memory_path=Path(layout["memory"])), + ) + + error = _ensure_local_workspace_for_update(bot, str(layout["install"])) + assert error is not None + assert "overlap" in error + assert not layout["workspace"].exists(), "preflight created a directory inside live data" + + +def test_all_three_callers_share_one_protected_root_derivation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The load-bearing property behind the three tests above: executor, + startup, and the updater must agree about the live-data root. When they + each derived their own, one accepted what another rejected.""" + import src.__main__ as entrypoint + from src.web.api.self_update import _live_protected_roots + + layout = _relocated_data_layout(tmp_path) + monkeypatch.chdir(layout["install"]) + live = str(layout["live"].resolve()) + + executor = ToolExecutor( + ToolsConfig( + local_working_dir=str(layout["workspace"]), + audit_log_path=str(layout["audit"]), + trajectory_path=str(layout["trajectory"]), + ), + memory_path=str(layout["memory"]), + ) + config = SimpleNamespace(tools=executor.config) + bot = SimpleNamespace( + config=config, tool_executor=SimpleNamespace(_memory_path=Path(layout["memory"])) + ) + + assert live in executor._protected_roots() + assert live in entrypoint._command_protected_roots(config) + assert live in _live_protected_roots(bot, str(layout["install"])) From 760a7152e8cf862cbfeb969b97a4dd4698777ba5 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 17:01:08 -0400 Subject: [PATCH 08/21] =?UTF-8?q?fix:=20PR=20#239=20round=207=20=E2=80=94?= =?UTF-8?q?=20a=20blank=20workspace=20can=20no=20longer=20mean=20two=20dif?= =?UTF-8?q?ferent=20things?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. BLANK LIVE VALUE. local_working_dir accepts free strings and can be blanked through PUT /api/config. The preflight treated a present-but-blank value as "nothing configured" and validated the DEFAULT instead, so the update was approved while the restarted process loaded the actual blank value: startup provisioning failed nonfatally and every local command then failed closed. Preflight and runtime disagreeing about the very path being validated — the same class as round 6, one layer up. Closed at both ends. The schema now normalizes blank/whitespace to the default at the boundary, so every consumer — preflight, startup migration, executor — reads one identical value; normalizing rather than rejecting keeps the update seamless and cannot brick startup on a persisted config, which a hard validation error would. And the preflight now validates the live value VERBATIM, never substituting: reaching it with a blank value means the live config is not a validated ToolsConfig, which is precisely when guessing a plausible default is least safe. The schema default is used only when there is no live config at all. Pinned end to end through POST /api/update/apply with a live blank config and nothing stubbed but the exec primitives: 409, actionable message, and the repository never touched. 2. `git diff --check` — trailing blank line at EOF removed. Now clean. Mutation-verified: dropping the schema normalization, or restoring the preflight's substitution, each fails its test (the latter fails both the unit and the endpoint pin). Gates, all four as CI runs them: 7578 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. git diff --check clean. --- src/config/schema.py | 21 +++++++++++++ src/web/api/self_update.py | 27 +++++++++-------- tests/test_local_workspace.py | 50 +++++++++++++++++++++++++++---- tests/test_web_api_self_update.py | 34 +++++++++++++++++++++ 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/src/config/schema.py b/src/config/schema.py index 4ee4d89a..6ac3a5b7 100644 --- a/src/config/schema.py +++ b/src/config/schema.py @@ -236,6 +236,27 @@ class ToolsConfig(BaseModel): # cross-command continuity this preserves. local_working_dir: str = "/var/lib/odin-workspace" + @field_validator("local_working_dir") + @classmethod + def _workspace_blank_means_default(cls, v): + """Blank or whitespace-only normalizes to the default, here at the + boundary, so every consumer sees the same value. + + The field accepts free strings and can be blanked through + PUT /api/config. Left un-normalized, the self-update preflight + substituted the default and approved, while the restarted process + loaded the blank value and failed closed on every local command — + preflight and runtime disagreeing about the very path being validated + (PR #239 round-7 review, reproduced). + + Normalizing rather than rejecting keeps the update seamless: a blanked + value costs no capability and cannot brick startup, which a hard + validation error on a persisted config would. + """ + if not isinstance(v, str) or not v.strip(): + return "/var/lib/odin-workspace" + return v.strip() + @field_validator("command_timeout_seconds") @classmethod def _timeout_positive(cls, v): diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index f3b1d276..4c220309 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -279,9 +279,6 @@ def _ensure_local_workspace_for_update(bot=None, base: str | None = None) -> str ) configured = _live_workspace_setting(bot) - if not configured: - return None # nothing configured to validate; runtime default applies - try: provision_workspace(configured, protected_roots=_live_protected_roots(bot, base)) return None @@ -292,19 +289,25 @@ def _ensure_local_workspace_for_update(bot=None, base: str | None = None) -> str def _live_workspace_setting(bot) -> str: - """The workspace path the RESTARTED process will actually use.""" + """The workspace path the RESTARTED process will actually use. + + Returned VERBATIM from the live config, never substituted. Treating a + present-but-blank value as "nothing configured" and validating the default + instead let the preflight approve an update whose restarted process then + failed closed on every local command (PR #239 round-7 review). The schema + normalizes blank to the default, so a blank value here means the live + config is not a validated ToolsConfig — which is exactly when guessing is + least safe. The default is used ONLY when there is no live config at all. + """ try: configured = bot.config.tools.local_working_dir - if isinstance(configured, str) and configured.strip(): - return configured.strip() + if isinstance(configured, str): + return configured except Exception: pass - try: - from ...config.schema import ToolsConfig + from ...config.schema import ToolsConfig - return ToolsConfig().local_working_dir - except Exception: # pragma: no cover - schema import cannot realistically fail - return "/var/lib/odin-workspace" + return ToolsConfig().local_working_dir def _live_protected_roots(bot, base: str | None) -> list[str]: @@ -331,5 +334,3 @@ def _live_protected_roots(bot, base: str | None) -> list[str]: trajectory_path=getattr(tools, "trajectory_path", None), memory_path=memory_path or DEFAULT_MEMORY_PATH, ) - - diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 24fd3d52..4a501d24 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -1085,15 +1085,53 @@ def config(self): # noqa: ANN201 - deliberately raises assert _live_workspace_setting(_Broken()) == ToolsConfig().local_working_dir -def test_preflight_skips_validation_when_nothing_is_configured( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("blank", ["", " ", "\t\n"]) +def test_blank_workspace_normalizes_to_the_default_at_the_boundary(blank: str) -> None: + """The field accepts free strings and can be blanked through PUT /api/config. + + Round-7 regression. Left un-normalized, a blank value made the self-update + preflight validate the DEFAULT and approve, while the restarted process + loaded the blank value and failed closed on every local command. Normalizing + here means every consumer — preflight, startup migration, executor — reads + the identical path. + """ + assert ToolsConfig(local_working_dir=blank).local_working_dir == "/var/lib/odin-workspace" + + +def test_configured_workspace_is_stripped_not_reinterpreted() -> None: + """Normalization must not silently relocate a real configured path.""" + assert ToolsConfig(local_working_dir=" /srv/ws ").local_working_dir == "/srv/ws" + + +def test_preflight_validates_the_exact_live_value_never_a_substitute() -> None: + """A present-but-blank live value must be REFUSED, not replaced. + + The schema normalizes blank away, so reaching the preflight with one means + the live config is not a validated ToolsConfig — precisely when substituting + a plausible default is least safe, because the restarted process will use + the real value and fail closed (PR #239 round-7 review, reproduced). + """ + from src.web.api.self_update import _ensure_local_workspace_for_update + + bot = SimpleNamespace(config=SimpleNamespace(tools=SimpleNamespace(local_working_dir=" "))) + error = _ensure_local_workspace_for_update(bot, None) + assert error is not None and "empty" in error + + +def test_preflight_uses_the_schema_default_only_when_there_is_no_live_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """With no configured workspace at all there is nothing to validate, and - the update must not be blocked on a value that does not exist.""" + """No reachable bot is the ONE case where the default is the honest answer.""" import src.web.api.self_update as su + from src.config.schema import ToolsConfig as _ToolsConfig + + class _Unreachable: + @property + def config(self): # noqa: ANN201 - deliberately raises + raise RuntimeError("bot not ready") - monkeypatch.setattr(su, "_live_workspace_setting", lambda _bot: "") - assert su._ensure_local_workspace_for_update(None, None) is None + assert su._live_workspace_setting(None) == _ToolsConfig().local_working_dir + assert su._live_workspace_setting(_Unreachable()) == _ToolsConfig().local_working_dir # --- Round 6: the entrypoint and the protected-root contract ------------------ diff --git a/tests/test_web_api_self_update.py b/tests/test_web_api_self_update.py index 5bc97537..cf703f34 100644 --- a/tests/test_web_api_self_update.py +++ b/tests/test_web_api_self_update.py @@ -10,6 +10,7 @@ from __future__ import annotations from subprocess import CompletedProcess +from types import SimpleNamespace from unittest.mock import MagicMock, patch from aiohttp import web @@ -183,6 +184,39 @@ async def test_refuses_when_the_workspace_cannot_be_provisioned(self): if call.args and isinstance(call.args[0], list) ), "the update must not have been committed" + async def test_refuses_a_blank_persisted_workspace_before_touching_the_repo(self): + """PR #239 round-7 blocker: the persisted-config path, end to end. + + local_working_dir accepts free strings and can be blanked through + PUT /api/config. The preflight used to treat a present-but-blank value + as "nothing configured" and validate the DEFAULT instead, so the update + was approved while the restarted process loaded the blank value and + failed closed on every local command. + + Nothing is stubbed here except the exec primitives: this drives the real + preflight from a live bot config. + """ + bot = SimpleNamespace( + config=SimpleNamespace( + tools=SimpleNamespace( + local_working_dir=" ", + audit_log_path=None, + trajectory_path=None, + ) + ) + ) + with patch("subprocess.run", side_effect=_run_ok) as run: + async with TestClient(TestServer(_app(bot))) as c: + r = await c.post("/api/update/apply", json={"version": "latest"}) + assert r.status == 409 + assert "update refused" in (await r.json())["error"] + assert not any( + "merge" in " ".join(map(str, call.args[0])) + for call in run.call_args_list + if call.args and isinstance(call.args[0], list) + ), "the update must not have been committed" + assert restart.restart_requested() is False + async def test_unexpected_exception_500(self): # subprocess raising mid-flow (inside the try) surfaces as a 500 def _boom(cmd, **kw): From 93c1870f0b64a62f2598817715c0d44232b10024 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 17:35:00 -0400 Subject: [PATCH 09/21] =?UTF-8?q?fix:=20PR=20#239=20round=208=20=E2=80=94?= =?UTF-8?q?=20workspace=20opt-in=20per=20call=20site,=20roots=20from=20the?= =?UTF-8?q?=20full=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blockers were scope violations I had not seen: the change reached further than the mechanism it was written to close. 1. THE WORKSPACE WAS CHANGING UNRELATED TOOLS. _exec_command is shared, and it also backs git_ops, docker, terraform, kubectl, claude_code, PDF host reads and validation probes. Applying the workspace unconditionally there silently repointed every one of them: git_ops documents an omitted `repo` as ".", which has always meant the process cwd — the install repo — so `git_ops status` returned "fatal: not a git repository". Docker build ".", compose's implicit project directory and terraform without working_dir are the same class. The workspace is now opt-in at the call site, passed only by run_command, run_script and run_command_multi — the raw user-command routes the incident came through — plus background starts, which resolve it directly. Every other tool is byte-identical to pre-PR behaviour, including keeping the process cwd and skipping workspace validation entirely. That also makes the incident tests stricter: they now drive the real run_command dispatch rather than the shared primitive, so removing the opt-in from the handler fails them. Two contract tests pin the boundary — git_ops with `repo` omitted keeps process-cwd semantics, and an unusable workspace disables raw user commands ONLY (one bad directory must not cost most of Odin's capability). 2. THE PROTECTED ROOTS WERE INCOMPLETE. They covered install, audit, trajectory and memory — but live state is not confined to the data directory. sessions.persist_directory, context.directory, the search index, permissions, Codex credentials, logs and usage are each independently relocatable, and a valid Config whose sessions directory WAS the workspace was accepted by all three callers. They agreed on an incomplete set, so the "outside the live-data roots" invariant was still false. Roots are now derived from the FULL live configuration through one declared file/directory table, supplied by wiring to the executor and used by the startup migration and the updater. A caller holding only a ToolsConfig gets a strict SUBSET, never a different answer, and a test pins that wiring supplies the full config so production never runs on the reduced form. Every declared path has a relocation case, and the table is asserted exhaustive so adding a path without protecting it fails. Also, per review: PUT /api/config blanking the workspace is now pinned for real — response, runtime config, persisted YAML and fresh reload — and the self-update test renamed to say what it actually drives. Mutation-verified: dropping the run_command opt-in, restoring unconditional application, ignoring the full config in the derivation, or removing wiring's app_config each fails its test. Gates, all four as CI runs them: 7587 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. git diff --check clean. --- src/__main__.py | 9 +- src/discord/wiring.py | 3 + src/tools/executor.py | 53 +++-- src/tools/handlers/system.py | 6 +- src/tools/workspace.py | 71 ++++++- src/web/api/self_update.py | 4 +- tests/test_branch_freshness.py | 12 +- tests/test_executor_integration_smoke.py | 7 +- tests/test_local_workspace.py | 251 +++++++++++++++++++++-- tests/test_web_api_config_admin.py | 30 +++ tests/test_web_api_self_update.py | 8 +- 11 files changed, 395 insertions(+), 59 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index 2a03fe97..cbb9b07d 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -320,16 +320,13 @@ def _command_protected_roots(config) -> list[str]: workspace beside it was provisioned at startup and then refused on every command (PR #239 round-6 review). - ``memory_path`` is intentionally left at its default: production wiring + The FULL config is passed so every independently relocatable live-state + path is covered. ``memory_path`` is left at its default: production wiring hardcodes that path, and startup runs before wiring exists. """ from src.tools.workspace import command_protected_roots - return command_protected_roots( - Path(__file__).resolve().parents[1], - audit_log_path=getattr(config.tools, "audit_log_path", None), - trajectory_path=getattr(config.tools, "trajectory_path", None), - ) + return command_protected_roots(Path(__file__).resolve().parents[1], config) # The entrypoint guard MUST stay the last statement in this module. Python diff --git a/src/discord/wiring.py b/src/discord/wiring.py index 3b37b49c..9352af49 100644 --- a/src/discord/wiring.py +++ b/src/discord/wiring.py @@ -261,6 +261,9 @@ def build_services(config: Config) -> BotServices: # noqa: PLR0915 — linear c tool_executor = ToolExecutor( config.tools, memory_path=memory_path, + # Full config so the workspace's protected roots cover every relocatable + # live-state path, not just the ones ToolsConfig declares. + app_config=config, browser_manager=browser_manager, output_streamer=output_streamer, host_access_manager=host_access_manager, diff --git a/src/tools/executor.py b/src/tools/executor.py index 5f4d7249..1b30dd87 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -158,8 +158,16 @@ def __init__( output_streamer: ToolOutputStreamer | None = None, host_access_manager: HostAccessManager | None = None, email_config: object | None = None, + app_config: object | None = None, ) -> None: self.config = config or ToolsConfig() + # The FULL live config, supplied by wiring. Live state is not confined + # to the data directory — sessions, context, logs, usage, the search + # index, permissions and Codex credentials are each independently + # relocatable, and a workspace overlapping any of them is as dangerous + # as one inside ./data (PR #239 round-8 review, reproduced). Optional + # so tests and the __new__ patch seam still construct. + self._app_config = app_config self._email_config = email_config # The configured workspace VALUE is restart-required, but it is # re-validated on every local command rather than cached: existence, @@ -334,14 +342,13 @@ def _protected_roots(self) -> list[str]: return command_protected_roots( # Install root: the package's own location (…/src/tools/executor.py). Path(__file__).resolve().parents[2], - audit_log_path=getattr(self.config, "audit_log_path", None), - trajectory_path=getattr(self.config, "trajectory_path", None), - # The live memory.json is supplied by wiring, not ToolsConfig. - # getattr-guarded: the sanctioned __new__ patch seam builds - # executors without __init__, so this attribute may not exist — - # falling back to the shared default rather than to no protection. - memory_path=getattr(self, "_memory_path", None) - or DEFAULT_MEMORY_PATH, + # getattr-guarded throughout: the sanctioned __new__ patch seam + # builds executors without __init__, so these may not exist. + getattr(self, "_app_config", None), + tools=getattr(self, "config", None), + # The live memory.json is supplied by wiring, not ToolsConfig; + # falling back to the shared default rather than no protection. + memory_path=getattr(self, "_memory_path", None) or DEFAULT_MEMORY_PATH, ) def get_workspace_metrics(self) -> dict[str, float]: @@ -728,6 +735,7 @@ async def _exec_command( ssh_user: str = "root", timeout: int | None = None, on_output: OutputCallback | None = None, + use_workspace: bool = False, ) -> tuple[int, str]: """Execute a command locally or via SSH depending on host address. @@ -744,6 +752,16 @@ async def _exec_command( if timeout is None: timeout = _current_tool_timeout_ctx.get() or self.config.command_timeout_seconds if is_local_address(address): + # The workspace applies ONLY to raw user commands, and only because + # the caller asked for it. This primitive also backs git_ops, + # docker, terraform, kubectl, claude_code, PDF host reads and + # validation probes, whose documented defaults resolve against the + # process cwd — git_ops with `repo` omitted means ".", i.e. the + # install repo, and silently repointing that at a scratch directory + # broke `git_ops status` with "fatal: not a git repository" + # (PR #239 round-8 review, reproduced). Default False keeps every + # such tool byte-identical to pre-PR behaviour. + cwd = self._ensure_local_workspace() if use_workspace else None bh = self.bulkheads.get("subprocess") if bh: try: @@ -752,12 +770,12 @@ async def _exec_command( command, timeout=timeout, on_output=on_output, - cwd=self._ensure_local_workspace(), + cwd=cwd, ) except BulkheadFullError: return 1, "Error: subprocess bulkhead full — too many concurrent local commands" return await run_local_command( - command, timeout=timeout, on_output=on_output, cwd=self._ensure_local_workspace() + command, timeout=timeout, on_output=on_output, cwd=cwd ) ssh_retry = self.config.ssh_retry ssh_kwargs: dict[str, Any] = dict( @@ -782,12 +800,23 @@ async def _exec_command( return 1, "Error: SSH bulkhead full — too many concurrent SSH commands" return await run_ssh_command(**ssh_kwargs) - async def _run_on_host(self, alias: str, command: str) -> str | tuple[str, int]: + async def _run_on_host( + self, alias: str, command: str, use_workspace: bool = False + ) -> str | tuple[str, int]: + """Run a command on an aliased host. + + ``use_workspace`` is opt-in for the same reason as _exec_command: this + also backs read_file/write_file host reads, skill_context.run_on_host, + and the audit diff tracker, whose paths are absolute and whose cwd + semantics must not change. + """ resolved = self._resolve_host(alias) if not resolved: return f"Unknown or disallowed host: {alias}" address, ssh_user, _os = resolved - code, output = await self._exec_command(address, command, ssh_user) + code, output = await self._exec_command( + address, command, ssh_user, use_workspace=use_workspace + ) if code != 0: return f"Command failed (exit {code}):\n{output}", code return output, 0 diff --git a/src/tools/handlers/system.py b/src/tools/handlers/system.py index ebeae971..93bc890b 100644 --- a/src/tools/handlers/system.py +++ b/src/tools/handlers/system.py @@ -61,6 +61,8 @@ async def _handle_run_command(self, inp: dict) -> str | tuple[str, int]: command, ssh_user, on_output=on_output, + # run_command is THE tool the 2026-07-27 wipe came through. + use_workspace=True, ) if finish_cb: try: @@ -144,6 +146,8 @@ async def _handle_run_script(self, inp: dict) -> str | tuple[str, int]: cmd, ssh_user, on_output=on_output, + # run_script executes arbitrary user script text, same hazard. + use_workspace=True, ) if finish_cb: try: @@ -193,7 +197,7 @@ async def _handle_run_command_multi(self, inp: dict) -> str | tuple[str, int]: allowed_hosts.append(h) async def _run_one(alias: str) -> tuple[str, bool]: - raw = await self._run_on_host(alias, command) + raw = await self._run_on_host(alias, command, use_workspace=True) if isinstance(raw, tuple): text, code = raw[0], raw[1] host_err = code != 0 diff --git a/src/tools/workspace.py b/src/tools/workspace.py index d54fdfd4..d3734933 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -35,6 +35,7 @@ import stat from collections.abc import Sequence from pathlib import Path +from types import SimpleNamespace # The accepted operational contract: a real directory owned by the execution # identity, private, and fully usable by that owner (0700). 0300 is private @@ -78,11 +79,51 @@ def _reject_overlap( ) +# Live-state paths declared by the FULL configuration, with their DECLARED +# file/directory semantics. Any of these can be relocated independently of the +# data directory, and a workspace overlapping one is as dangerous as a +# workspace inside ./data — round 8 reproduced a valid Config whose +# sessions.persist_directory WAS the workspace, accepted by every caller +# because only audit/trajectory/memory were protected. +# +# Deliberately excluded: tools.claude_code_dir and email.allowed_attachment_dirs +# are working directories for a tool and user-nominated source directories, not +# Odin's own state; protecting them would reject legitimate configurations. +# Paths wiring hardcodes (channel_config.json, channel_logs) sit beside +# memory.json and are covered by its parent. +_DECLARED_STATE_PATHS: tuple[tuple[str, bool], ...] = ( + ("tools.audit_log_path", True), + ("tools.trajectory_path", False), + ("tools.ssh_key_path", True), + ("tools.ssh_known_hosts_path", True), + ("tools.ssh_pool.socket_dir", False), + ("context.directory", False), + ("sessions.persist_directory", False), + ("logging.directory", False), + ("usage.directory", False), + # Treated as a file: wiring derives its sibling fts.db via `.parent`. + ("search.search_db_path", True), + ("permissions.overrides_path", True), + ("openai_codex.credentials_path", True), + ("attachments.temp_directory", False), +) + + +def _dotted(source: object, path: str) -> object: + """Resolve ``a.b.c`` against nested config objects, tolerating absence.""" + current = source + for part in path.split("."): + current = getattr(current, part, None) + if current is None: + return None + return current + + def command_protected_roots( install_root: str | os.PathLike[str], + config: object = None, *, - audit_log_path: object = None, - trajectory_path: object = None, + tools: object = None, memory_path: object = DEFAULT_MEMORY_PATH, ) -> list[str]: """THE derivation of directories a command workspace must never overlap. @@ -90,8 +131,14 @@ def command_protected_roots( Every caller — executor, startup migration, self-update preflight — uses this one function, so a workspace accepted by one is accepted by all. When they each derived their own, the preflight approved (and created) a - workspace inside the live-data directory that the executor then rejected, - which both mutated live data and stranded local commands. + workspace beside live memory.json that the executor then rejected. + + Pass the FULL ``config`` wherever one exists: live state is not confined to + the data directory, and sessions, context, logs, usage, the search index, + permissions and Codex credentials can each be relocated independently. + ``tools`` is the reduced fallback for callers holding only a ToolsConfig + (the executor's ``__new__`` patch seam and unit tests); it yields a strict + SUBSET, never a different answer. Paths are classified by DECLARED semantics, never guessed from the name: a ``Path.suffix`` heuristic misreads dotted directories and extensionless @@ -100,12 +147,18 @@ def command_protected_roots( than the target (``/aliases/memory.json -> /live-data/memory.json`` would protect ``/aliases`` and accept ``/live-data/workspace``). """ + source: object = config + if source is None: + source = SimpleNamespace(tools=tools) if tools is not None else SimpleNamespace() + roots = [str(_canonical(install_root))] declared: list[tuple[object, bool]] = [ - (audit_log_path, True), # file - (trajectory_path, False), # directory - (memory_path, True), # file + (_dotted(source, dotted), is_file) for dotted, is_file in _DECLARED_STATE_PATHS ] + # The live memory.json is supplied by wiring rather than by config, so it + # is passed in rather than declared above. + declared.append((memory_path, True)) + for configured, is_file in declared: if configured is None: continue @@ -113,7 +166,9 @@ def command_protected_roots( if not text: continue resolved = _canonical(text) - roots.append(str(resolved.parent if is_file else resolved)) + root = str(resolved.parent if is_file else resolved) + if root not in roots: + roots.append(root) return roots diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index 4c220309..ac1f3d7e 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -326,11 +326,9 @@ def _live_protected_roots(bot, base: str | None) -> list[str]: """ from src.tools.workspace import DEFAULT_MEMORY_PATH, command_protected_roots - tools = getattr(getattr(bot, "config", None), "tools", None) memory_path = getattr(getattr(bot, "tool_executor", None), "_memory_path", None) return command_protected_roots( Path(base).resolve() if base else Path(__file__).resolve().parents[3], - audit_log_path=getattr(tools, "audit_log_path", None), - trajectory_path=getattr(tools, "trajectory_path", None), + getattr(bot, "config", None), memory_path=memory_path or DEFAULT_MEMORY_PATH, ) diff --git a/tests/test_branch_freshness.py b/tests/test_branch_freshness.py index dcf5dfb2..e99e9533 100644 --- a/tests/test_branch_freshness.py +++ b/tests/test_branch_freshness.py @@ -549,7 +549,7 @@ def executor(self): @pytest.mark.asyncio async def test_annotates_stale_branch_on_test_failure(self, executor): """Test failure on stale branch gets a staleness warning appended.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): if "pytest" in command: return (1, "3 failed, 10 passed") if "rev-parse" in command: @@ -574,7 +574,7 @@ async def mock_exec_command(address, command, ssh_user, timeout=None, on_output= @pytest.mark.asyncio async def test_no_annotation_on_fresh_branch(self, executor): """Test failure on fresh branch gets no warning.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): if "pytest" in command: return (1, "3 failed, 10 passed") if "rev-parse" in command: @@ -642,7 +642,7 @@ async def mock_run_on_host(alias, command): @pytest.mark.asyncio async def test_freshness_check_exception_is_safe(self, executor): """If freshness check crashes, the original result is returned.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): if "pytest" in command: return (1, "3 failed, 10 passed") raise RuntimeError("freshness check boom") @@ -659,7 +659,7 @@ async def mock_exec_command(address, command, ssh_user, timeout=None, on_output= @pytest.mark.asyncio async def test_fetch_failure_tracked(self, executor): """Fetch failures are counted in stats.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): if "pytest" in command: return (1, "3 failed") if "rev-parse" in command: @@ -690,7 +690,7 @@ def executor(self): @pytest.mark.asyncio async def test_script_with_test_annotated(self, executor): """run_script with test commands gets freshness annotation.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): if "rev-parse" in command: return (0, "master\n") if "fetch" in command: @@ -715,7 +715,7 @@ async def mock_exec_command(address, command, ssh_user, timeout=None, on_output= @pytest.mark.asyncio async def test_script_non_test_not_annotated(self, executor): """run_script without test commands doesn't get freshness annotation.""" - async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None): + async def mock_exec_command(address, command, ssh_user, timeout=None, on_output=None, **kw): return (1, "file not found") executor._exec_command = mock_exec_command diff --git a/tests/test_executor_integration_smoke.py b/tests/test_executor_integration_smoke.py index 2a951bf2..070fb7d3 100644 --- a/tests/test_executor_integration_smoke.py +++ b/tests/test_executor_integration_smoke.py @@ -812,7 +812,12 @@ async def test_run_command_multi_per_host_governor(self): assert "ok" in text assert "strict" in text assert exit_code == 1 # a host was denied → not ok - exe._run_on_host.assert_called_once_with("dev", "systemctl restart nginx") + # use_workspace=True is load-bearing: run_command_multi is a raw + # user-command route and must land in the workspace, not the install + # (PR #239). Unrelated tools deliberately omit it. + exe._run_on_host.assert_called_once_with( + "dev", "systemctl restart nginx", use_workspace=True + ) @pytest.mark.asyncio async def test_memory_manage_get_action(self, tmp_path): diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 4a501d24..9949e57e 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -365,10 +365,15 @@ def test_discord_execution_path_excludes_the_legacy_dag_surfaces( def _executor_with_workspace(workspace: Path, protected: Path): """A ToolExecutor whose local commands run in ``workspace``.""" - from src.config.schema import ToolsConfig + from src.config.schema import ToolHost, ToolsConfig from src.tools.executor import ToolExecutor - config = ToolsConfig(local_working_dir=str(workspace)) + config = ToolsConfig( + local_working_dir=str(workspace), + # A resolvable localhost so tests drive the PRODUCTION dispatch route + # rather than the shared _exec_command primitive. + hosts={"localhost": ToolHost(address="127.0.0.1")}, + ) executor = ToolExecutor(config=config) # Protected roots are normally derived from the running app; point them at # the fixture so the test exercises real validation against a fake install. @@ -376,6 +381,20 @@ def _executor_with_workspace(workspace: Path, protected: Path): return executor +async def _run_command(executor, command: str) -> tuple[int, str]: + """Drive the REAL run_command tool, end to end. + + Deliberately not `_exec_command`: since round 8 the workspace is opt-in at + the call site, because that shared primitive also backs git_ops, docker, + terraform, kubectl, claude_code and PDF host reads, whose cwd semantics + must not change. Testing the primitive would therefore no longer prove + that the tool Odin actually calls gets the workspace — removing + `use_workspace=True` from the run_command handler has to fail these tests. + """ + result = await executor.execute("run_command", {"command": command, "host": "localhost"}) + return (0 if result.ok else 1), str(result.output) + + async def test_executor_replays_the_incident_without_touching_the_install( fake_install: Path, workspace: Path, @@ -392,25 +411,22 @@ async def test_executor_replays_the_incident_without_touching_the_install( assert Path.cwd() == fake_install.resolve() executor = _executor_with_workspace(workspace, fake_install) - code, out = await executor._exec_command( - "localhost", + code, out = await _run_command( + executor, f"jar xf {ae2_jar} data/ae2/recipe/network/blocks/pattern_providers_interface.json" f" || unzip -o {ae2_jar} 'data/ae2/recipe/network/blocks/*' > /dev/null", - timeout=30, ) assert code == 0, out assert (workspace / "data/ae2/recipe/network/blocks").exists() assert not (fake_install / "data" / "ae2").exists() - code, out = await executor._exec_command( - "localhost", - "cat data/ae2/recipe/network/blocks/pattern_providers_interface.json", - timeout=30, + code, out = await _run_command( + executor, "cat data/ae2/recipe/network/blocks/pattern_providers_interface.json" ) assert code == 0 and "ae2:shaped" in out assert Path(str(workspace)).is_relative_to(tmp_path) # bounded before deleting - code, _ = await executor._exec_command("localhost", "rm -rf data", timeout=30) + code, _ = await _run_command(executor, "rm -rf data") assert code == 0 assert not (workspace / "data").exists(), "cleanup must work" assert (fake_install / "data" / "sentinel").read_text(encoding="utf-8") == "live odin state" @@ -418,7 +434,7 @@ async def test_executor_replays_the_incident_without_touching_the_install( async def test_executor_pwd_is_the_workspace(fake_install: Path, workspace: Path) -> None: executor = _executor_with_workspace(workspace, fake_install) - code, out = await executor._exec_command("localhost", "pwd", timeout=30) + code, out = await _run_command(executor, "pwd") assert code == 0 assert out.strip() == str(workspace.resolve()) @@ -428,9 +444,7 @@ async def test_executor_explicit_cd_into_install_still_works( ) -> None: """Aaron's bar: deliberately working inside his own install is untouched.""" executor = _executor_with_workspace(workspace, fake_install) - code, out = await executor._exec_command( - "localhost", f"cd {fake_install} && cat data/sentinel", timeout=30 - ) + code, out = await _run_command(executor, f"cd {fake_install} && cat data/sentinel") assert code == 0 and "live odin state" in out @@ -466,7 +480,9 @@ async def test_executor_refuses_to_run_with_an_invalid_workspace( directory — that fallback is the hazard.""" executor = _executor_with_workspace(tmp_path / "no-parent" / "ws", fake_install) with pytest.raises(WorkspaceError): - await executor._exec_command("localhost", "echo should-not-run", timeout=10) + await executor._exec_command( + "localhost", "echo should-not-run", timeout=10, use_workspace=True + ) # --- protected roots: real deployment shapes -------------------------------- @@ -547,7 +563,9 @@ async def test_executor_enforces_its_protected_roots(fake_install: Path) -> None inside.mkdir(mode=0o700) executor = _executor_with_workspace(inside, fake_install) with pytest.raises(WorkspaceError, match="overlap"): - await executor._exec_command("localhost", "echo should-not-run", timeout=10) + await executor._exec_command( + "localhost", "echo should-not-run", timeout=10, use_workspace=True + ) def test_executor_derives_real_roots_from_the_running_app() -> None: @@ -630,14 +648,16 @@ async def test_workspace_is_revalidated_before_every_command( monkeypatch.chdir(fake_install) executor = _executor_with_workspace(workspace, fake_install) - code, out = await executor._exec_command("localhost", "pwd", timeout=30) + code, out = await _run_command(executor, "pwd") assert code == 0 and out.strip() == str(workspace.resolve()) # Swap the validated directory for a symlink pointing into the install. workspace.rmdir() workspace.symlink_to(fake_install) with pytest.raises(WorkspaceError): - await executor._exec_command("localhost", "cat data/sentinel", timeout=30) + await executor._exec_command( + "localhost", "cat data/sentinel", timeout=30, use_workspace=True + ) async def test_mode_change_after_first_command_is_caught( @@ -645,11 +665,13 @@ async def test_mode_change_after_first_command_is_caught( ) -> None: """A post-validation chmod must not be ignored either.""" executor = _executor_with_workspace(workspace, fake_install) - assert (await executor._exec_command("localhost", "pwd", timeout=30))[0] == 0 + assert (await _run_command(executor, "pwd"))[0] == 0 workspace.chmod(0o755) try: with pytest.raises(WorkspaceError, match="mode"): - await executor._exec_command("localhost", "pwd", timeout=30) + await executor._exec_command( + "localhost", "pwd", timeout=30, use_workspace=True + ) finally: workspace.chmod(0o700) @@ -1352,3 +1374,192 @@ def test_all_three_callers_share_one_protected_root_derivation( assert live in executor._protected_roots() assert live in entrypoint._command_protected_roots(config) assert live in _live_protected_roots(bot, str(layout["install"])) + + +# --- Round 8: the workspace must not leak into unrelated tools --------------- + + +async def test_git_ops_with_omitted_repo_keeps_process_cwd_semantics( + tmp_path: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-8 blocker 1, reproduced by Odin as `fatal: not a git repository`. + + git_ops documents an omitted ``repo`` as ``"."`` — which has always meant + the process cwd, i.e. Odin's own install repo. Applying the workspace + unconditionally in the shared _exec_command primitive silently repointed + that at a scratch directory and broke `git_ops status`. The same class hits + docker build ``"."``, compose's implicit project directory, and terraform + without ``working_dir``. + """ + repo = tmp_path / "a-real-repo" + repo.mkdir() + for cmd in (["git", "init", "-q"], ["git", "config", "user.email", "t@t"], + ["git", "config", "user.name", "t"]): + subprocess.run(cmd, cwd=repo, check=True, capture_output=True) + (repo / "tracked.txt").write_text("x", encoding="utf-8") + monkeypatch.chdir(repo) + + executor = _executor_with_workspace(workspace, tmp_path / "unrelated-install") + result = await executor.execute("git_ops", {"action": "status", "host": "localhost"}) + + assert result.ok, result.output + assert "not a git repository" not in str(result.output) + assert "tracked.txt" in str(result.output) + + +async def test_an_unusable_workspace_does_not_disable_unrelated_tools( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The sharp version of the same contract. + + A workspace that fails validation must take down raw user commands ONLY. + If it also took down git_ops/docker/terraform/kubectl, one bad directory + would cost most of Odin's capability — far beyond the accepted mechanism. + """ + repo = tmp_path / "repo" + repo.mkdir() + for cmd in (["git", "init", "-q"], ["git", "config", "user.email", "t@t"], + ["git", "config", "user.name", "t"]): + subprocess.run(cmd, cwd=repo, check=True, capture_output=True) + monkeypatch.chdir(repo) + + protected = tmp_path / "install" + protected.mkdir() + # Overlaps the protected root: unusable by construction. + executor = _executor_with_workspace(protected / "inside", protected) + + user_command = await executor.execute( + "run_command", {"command": "echo should-not-run", "host": "localhost"} + ) + assert not user_command.ok + assert "should-not-run" not in str(user_command.output) + + git_status = await executor.execute("git_ops", {"action": "status", "host": "localhost"}) + assert git_status.ok, "an unusable workspace must not disable unrelated tools" + + +# --- Round 8: protected roots come from the FULL live configuration --------- + + +def _config_with(**overrides): + """A real Config, so the derivation is exercised against production shape.""" + from src.config.schema import Config + + return Config(discord={"token": "test-token"}, **overrides) + + +def test_relocated_state_directory_is_protected(tmp_path: Path) -> None: + """Round-8 blocker 2, reproduced by Odin with sessions.persist_directory + EQUAL to the configured workspace and accepted by every caller.""" + from src.tools.workspace import command_protected_roots + + relocated = tmp_path / "relocated-sessions" + config = _config_with( + sessions={"persist_directory": str(relocated)}, + tools={"local_working_dir": str(relocated)}, + ) + roots = command_protected_roots(tmp_path / "install", config) + assert str(relocated.resolve()) in roots + + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(relocated), protected_roots=roots) + assert not relocated.exists() + + +def test_relocated_state_file_protects_its_directory(tmp_path: Path) -> None: + """A relocated FILE protects the directory holding it — the workspace must + not sit beside live permissions/credential state either.""" + from src.tools.workspace import command_protected_roots + + live = tmp_path / "relocated-state" + live.mkdir() + config = _config_with(permissions={"overrides_path": str(live / "permissions.json")}) + roots = command_protected_roots(tmp_path / "install", config) + assert str(live.resolve()) in roots + + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(live / "workspace"), protected_roots=roots) + assert not (live / "workspace").exists() + + +def test_overlap_with_relocated_state_is_rejected_in_both_directions( + tmp_path: Path, +) -> None: + """A workspace that CONTAINS live state is as unusable as one inside it.""" + from src.tools.workspace import command_protected_roots + + parent = tmp_path / "parent" + (parent / "live-context").mkdir(parents=True) + config = _config_with(context={"directory": str(parent / "live-context")}) + roots = command_protected_roots(tmp_path / "install", config) + + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(parent), protected_roots=roots) + + +def test_every_declared_state_path_is_covered(tmp_path: Path) -> None: + """Each declared live-state path contributes a root, so relocating any one + of them cannot silently drop it from protection.""" + from src.tools.workspace import _DECLARED_STATE_PATHS, command_protected_roots + + relocations = { + "tools.audit_log_path": (tmp_path / "s-audit" / "audit.jsonl", tmp_path / "s-audit"), + "tools.trajectory_path": (tmp_path / "s-traj", tmp_path / "s-traj"), + "tools.ssh_key_path": (tmp_path / "s-ssh" / "id", tmp_path / "s-ssh"), + "tools.ssh_known_hosts_path": (tmp_path / "s-kh" / "known", tmp_path / "s-kh"), + "tools.ssh_pool.socket_dir": (tmp_path / "s-sock", tmp_path / "s-sock"), + "context.directory": (tmp_path / "s-ctx", tmp_path / "s-ctx"), + "sessions.persist_directory": (tmp_path / "s-sess", tmp_path / "s-sess"), + "logging.directory": (tmp_path / "s-log", tmp_path / "s-log"), + "usage.directory": (tmp_path / "s-usage", tmp_path / "s-usage"), + "search.search_db_path": (tmp_path / "s-search" / "db", tmp_path / "s-search"), + "permissions.overrides_path": (tmp_path / "s-perm" / "p.json", tmp_path / "s-perm"), + "openai_codex.credentials_path": (tmp_path / "s-codex" / "c.json", tmp_path / "s-codex"), + "attachments.temp_directory": (tmp_path / "s-att", tmp_path / "s-att"), + } + assert set(relocations) == {dotted for dotted, _ in _DECLARED_STATE_PATHS}, ( + "a declared state path has no relocation case — add one so protection " + "cannot be dropped silently" + ) + + overrides: dict[str, dict] = {} + for dotted, (value, _expected) in relocations.items(): + section, _, leaf = dotted.partition(".") + node = overrides.setdefault(section, {}) + while "." in leaf: # nested section, e.g. tools.ssh_pool.socket_dir + head, _, leaf = leaf.partition(".") + node = node.setdefault(head, {}) + node[leaf] = str(value) + + roots = command_protected_roots(tmp_path / "install", _config_with(**overrides)) + for dotted, (_value, expected) in relocations.items(): + assert str(expected.resolve()) in roots, f"{dotted} is not protected" + + +def test_reduced_derivation_is_a_subset_never_a_different_answer(tmp_path: Path) -> None: + """Callers holding only a ToolsConfig (the __new__ patch seam, unit tests) + must not disagree with the full derivation — they may only know less.""" + from src.config.schema import ToolsConfig + from src.tools.workspace import command_protected_roots + + tools = ToolsConfig( + audit_log_path=str(tmp_path / "a" / "audit.jsonl"), + trajectory_path=str(tmp_path / "t"), + ) + config = _config_with(tools=tools.model_dump()) + full = command_protected_roots(tmp_path / "install", config) + reduced = command_protected_roots(tmp_path / "install", tools=tools) + assert set(reduced) <= set(full) + + +def test_wiring_supplies_the_full_config_to_the_executor() -> None: + """The reduced derivation must never be what production runs on.""" + import inspect + + from src.discord import wiring + + source = inspect.getsource(wiring.build_services) + assert "app_config=config" in source, ( + "wiring must pass the full config to ToolExecutor, or production falls " + "back to the reduced protected-root derivation" + ) diff --git a/tests/test_web_api_config_admin.py b/tests/test_web_api_config_admin.py index 2cfbb678..0493ee27 100644 --- a/tests/test_web_api_config_admin.py +++ b/tests/test_web_api_config_admin.py @@ -284,6 +284,36 @@ async def test_update_config_persists_normalized_dropping_removed_keys(self): oc = YAML().load(Path("config.yml").read_text()).get("openai_codex", {}) assert "model_routing" not in oc + @pytest.mark.asyncio + async def test_blanking_the_workspace_normalizes_everywhere(self): + """PR #239 round-8 follow-up: the persisted-config path, for real. + + tools.local_working_dir accepts free strings and can be blanked through + this endpoint. Blank must normalize to the default in the RESPONSE, in + the runtime config, on disk, and on a fresh reload — otherwise the + self-update preflight and the restarted process disagree about which + directory they are validating, which is how a blank value used to + approve an update that then failed closed on every local command. + """ + from pathlib import Path + + from ruamel.yaml import YAML + + from src.config.schema import Config as _Config + + Path("config.yml").write_text("discord:\n token: fake\n") + app, bot = _app(register_discord_config) + async with TestClient(TestServer(app)) as c: + r = await c.put("/api/config", json={"tools": {"local_working_dir": " "}}) + assert r.status == 200 + + default = "/var/lib/odin-workspace" + assert bot.config.tools.local_working_dir == default, "runtime config" + on_disk = YAML().load(Path("config.yml").read_text())["tools"]["local_working_dir"] + assert on_disk == default, "persisted YAML" + reloaded = _Config(**YAML().load(Path("config.yml").read_text())) + assert reloaded.tools.local_working_dir == default, "fresh reload" + @pytest.mark.asyncio async def test_health_and_resource_and_streams(self): app, bot = _app(register_discord_config) diff --git a/tests/test_web_api_self_update.py b/tests/test_web_api_self_update.py index cf703f34..e15d28c3 100644 --- a/tests/test_web_api_self_update.py +++ b/tests/test_web_api_self_update.py @@ -184,8 +184,12 @@ async def test_refuses_when_the_workspace_cannot_be_provisioned(self): if call.args and isinstance(call.args[0], list) ), "the update must not have been committed" - async def test_refuses_a_blank_persisted_workspace_before_touching_the_repo(self): - """PR #239 round-7 blocker: the persisted-config path, end to end. + async def test_refuses_a_blank_live_workspace_before_touching_the_repo(self): + """PR #239 round-7 blocker: a blank LIVE workspace, end to end. + + The persistence half — that PUT /api/config normalizes blank away — is + pinned separately in test_web_api_config_admin; this drives the update + route against a live config that already holds a raw blank value. local_working_dir accepts free strings and can be blanked through PUT /api/config. The preflight used to treat a present-but-blank value From b0f801bf7d1a8b74160b8096de04d1ab0241a61d Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 18:06:31 -0400 Subject: [PATCH 10/21] =?UTF-8?q?fix:=20PR=20#239=20round=209=20=E2=80=94?= =?UTF-8?q?=20the=20last=20raw-command=20route,=20the=20live=20config=20fi?= =?UTF-8?q?le,=20cached=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. SkillContext.run_on_host BYPASSED THE WORKSPACE. Round 8 scoped the workspace to the raw user-command routes, and I judged this one out of scope. That was wrong: it is arbitrary command execution deliberately exposed to user-created skills, so it is exactly a raw command route. Reproduced through the real method against a fake install — the command entered the install and deleted its data/ while the workspace sat untouched. It now opts in, with a localhost incident replay and the same fail-closed pin as the other routes. Remote hosts are unaffected: the workspace applies only after local-address resolution. 2. THE ACTIVE CONFIG FILE WAS UNPROTECTED. Odin accepts `python -m src /arbitrary/path/odin.yml`, and load_config records that canonical path — but it is runtime state rather than a Config field, so the exhaustive-declaration test structurally could not catch it. An alternate config whose parent was also the configured workspace was accepted, leaving a bare relative command able to delete the file needed to restart. Its canonical parent now joins the shared derivation used by startup, executor and updater; the complete file path is resolved before .parent so an aliased config protects the real directory, and both overlap directions are pinned. A process that never loaded a config still protects nothing extra — guessing a path would reject legitimate workspaces. 3. Metrics (his non-blocking note, worth fixing now): the size/count walk ran synchronously on every unauthenticated /metrics scrape, over a directory that deliberately never prunes — an event-loop stall that only gets worse. The walk is now cached for 60s; free space and inodes stay live. Growth is an operator signal measured in hours, so staleness costs nothing. Mutation-verified: dropping the skill opt-in, the active-config root, or the metrics cache each fails its test. Gates, all four as CI runs them: 7594 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. git diff --check clean. --- src/tools/executor.py | 45 +++++++--- src/tools/skill_context.py | 8 +- src/tools/workspace.py | 22 +++++ tests/test_local_workspace.py | 159 +++++++++++++++++++++++++++++++++- tests/test_ts_ledger_fixes.py | 14 ++- 5 files changed, 233 insertions(+), 15 deletions(-) diff --git a/src/tools/executor.py b/src/tools/executor.py index 1b30dd87..7eb355b5 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -81,6 +81,11 @@ "odin_current_tool_timeout", default=None ) +# How long a workspace size/count walk is reused. Growth is an operator signal +# measured in hours, so a minute of staleness costs nothing; scraping /metrics +# in a loop against an ever-growing directory costs the event loop a great deal. +WORKSPACE_METRICS_TTL = 60.0 + def _validate_memory_shape(data: dict) -> None: """Nested-shape check for memory.json, run inside json_store's backup @@ -183,6 +188,8 @@ def __init__( # is what would restore the 2026-07-27 hazard. self._local_workspace: str | None = None self._local_workspace_resolved = False + # (monotonic_stamp, bytes, files) from the last workspace usage walk. + self._workspace_usage_cache: tuple[float, float, float] | None = None self._memory_path = Path(memory_path) if memory_path else None self._browser_manager = browser_manager self._permission_manager = permission_manager @@ -366,18 +373,32 @@ def get_workspace_metrics(self) -> dict[str, float]: root = Path(self._ensure_local_workspace()) except Exception: return {} - total_bytes = 0.0 - files = 0.0 - try: - for dirpath, _dirnames, filenames in os.walk(root, followlinks=False): - for name in filenames: - files += 1 - try: - total_bytes += os.lstat(os.path.join(dirpath, name)).st_size - except OSError: - pass - except OSError: - return {} + + # The size/count walk is CACHED for WORKSPACE_METRICS_TTL. /metrics is + # unauthenticated and can be scraped at any rate, and this deliberately + # un-pruned directory only grows — walking every file synchronously on + # each request is an event-loop stall that gets worse over time + # (PR #239 round-9 review). Free space and inodes are cheap statvfs + # calls and stay live. On a stale-cache read the previous walk's + # numbers are reused; on the first-ever call the walk runs inline. + now = time.monotonic() + cached = getattr(self, "_workspace_usage_cache", None) + if cached is not None and (now - cached[0]) < WORKSPACE_METRICS_TTL: + total_bytes, files = cached[1], cached[2] + else: + total_bytes = 0.0 + files = 0.0 + try: + for dirpath, _dirnames, filenames in os.walk(root, followlinks=False): + for name in filenames: + files += 1 + try: + total_bytes += os.lstat(os.path.join(dirpath, name)).st_size + except OSError: + pass + except OSError: + return {} + self._workspace_usage_cache = (now, total_bytes, files) metrics = {"bytes": total_bytes, "files": files} try: usage = shutil.disk_usage(root) diff --git a/src/tools/skill_context.py b/src/tools/skill_context.py index 8900dd15..75c7530f 100644 --- a/src/tools/skill_context.py +++ b/src/tools/skill_context.py @@ -164,7 +164,13 @@ async def run_on_host(self, alias: str, command: str) -> str: # _run_on_host returns (output, exit_code) for resolved hosts and a bare # denial string for unknown ones (TS-0004 — forwarding the raw tuple # violated this method's documented str contract for every real host). - raw = await self._executor._run_on_host(alias, command) + # use_workspace=True: this is arbitrary command execution exposed to + # user-created skills, so it is a raw user-command route exactly like + # run_command — leaving it out made it an alternate path back into the + # 2026-07-27 wipe, reproduced through this method (PR #239 round-9 + # review). Remote hosts are unaffected: the workspace applies only + # after local-address resolution. + raw = await self._executor._run_on_host(alias, command, use_workspace=True) if isinstance(raw, tuple): return raw[0] return raw diff --git a/src/tools/workspace.py b/src/tools/workspace.py index d3734933..42529fb9 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -109,6 +109,22 @@ def _reject_overlap( ) +def _active_config_file() -> str | None: + """Absolute path the live config was loaded from, if any. + + Imported lazily and guarded: workspace validation must never depend on the + config module being importable, and a process that never loaded a config + (tests, one-off scripts) has nothing to protect. + """ + try: + from ..config.schema import active_config_path + + path = active_config_path() + except Exception: # pragma: no cover - defensive + return None + return str(path) if path else None + + def _dotted(source: object, path: str) -> object: """Resolve ``a.b.c`` against nested config objects, tolerating absence.""" current = source @@ -158,6 +174,12 @@ def command_protected_roots( # The live memory.json is supplied by wiring rather than by config, so it # is passed in rather than declared above. declared.append((memory_path, True)) + # The ACTIVE config file is runtime state, not a Config field: Odin accepts + # `python -m src /arbitrary/path/odin.yml`. Its directory must be protected + # too, or a bare relative command can delete the file needed to restart — + # reproduced with an alternate config whose parent WAS the configured + # workspace (PR #239 round-9 review). + declared.append((_active_config_file(), True)) for configured, is_file in declared: if configured is None: diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 9949e57e..76dcff2d 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -1170,7 +1170,11 @@ def config(self): # noqa: ANN201 - deliberately raises os.environ.setdefault(name, "test-value") import yaml -cfg = tmp / "config.yml" +# The config lives in its own directory: since round 9 the ACTIVE config +# file's directory is protected, and production keeps config.yml inside the +# install root (already protected) with the workspace elsewhere. +(tmp / "etc").mkdir(exist_ok=True) +cfg = tmp / "etc" / "config.yml" parsed = yaml.safe_load(template) parsed.setdefault("tools", {})["local_working_dir"] = str(ws) cfg.write_text(yaml.safe_dump(parsed)) @@ -1563,3 +1567,156 @@ def test_wiring_supplies_the_full_config_to_the_executor() -> None: "wiring must pass the full config to ToolExecutor, or production falls " "back to the reduced protected-root derivation" ) + + +# --- Round 9: the last arbitrary-command route, and the live config file ----- + + +async def test_skill_run_on_host_replays_the_incident_safely( + fake_install: Path, workspace: Path, ae2_jar: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-9 blocker 1, reproduced by Odin: SkillContext.run_on_host is + arbitrary command execution exposed to user-created skills, and it was + still inheriting the process cwd — an alternate route straight back into + the wipe. Remote hosts are unaffected; the workspace applies only after + local-address resolution. + """ + from src.tools.skill_context import SkillContext + + monkeypatch.chdir(fake_install) # a regression can only destroy the fixture + executor = _executor_with_workspace(workspace, fake_install) + ctx = SkillContext.__new__(SkillContext) + ctx._executor = executor + + await ctx.run_on_host("localhost", "mkdir -p data && touch data/from-skill") + assert (workspace / "data" / "from-skill").exists() + + await ctx.run_on_host("localhost", "rm -rf data") + assert not (workspace / "data").exists(), "the skill's own cleanup must work" + assert (fake_install / "data" / "sentinel").read_text(encoding="utf-8") == "live odin state" + + +async def test_skill_run_on_host_fails_closed_on_an_invalid_workspace( + fake_install: Path, tmp_path: Path +) -> None: + """Same fail-closed contract as the other raw command routes.""" + from src.tools.skill_context import SkillContext + + executor = _executor_with_workspace(tmp_path / "no-parent" / "ws", fake_install) + ctx = SkillContext.__new__(SkillContext) + ctx._executor = executor + with pytest.raises(WorkspaceError): + await ctx.run_on_host("localhost", "echo should-not-run") + + +def test_active_config_file_directory_is_protected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-9 blocker 2, reproduced by Odin with an alternate config whose + parent WAS the configured workspace. + + Odin accepts `python -m src /arbitrary/path/odin.yml`. That file is runtime + state, not a Config field, so the exhaustive-declaration test cannot cover + it — and a bare relative command could delete the file needed to restart. + """ + from src.config.schema import active_config_path, set_active_config_path + from src.tools.workspace import command_protected_roots + + live = tmp_path / "live-config-and-workspace" + live.mkdir() + config_file = live / "odin-custom.yml" + config_file.write_text("discord:\n token: fake\n", encoding="utf-8") + + previous = active_config_path() + set_active_config_path(config_file) + try: + roots = command_protected_roots(tmp_path / "install") + assert str(live.resolve()) in roots + + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(live), protected_roots=roots) + # ...and in the other direction: a workspace CONTAINING the config file. + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(tmp_path), protected_roots=roots) + finally: + set_active_config_path(previous) + + +def test_active_config_symlink_protects_the_target_directory( + tmp_path: Path, +) -> None: + """The complete file path is resolved before taking .parent, so an aliased + config cannot protect the alias directory instead of the real one.""" + from src.config.schema import active_config_path, set_active_config_path + from src.tools.workspace import command_protected_roots + + real_dir = tmp_path / "real-config-dir" + real_dir.mkdir() + real_file = real_dir / "odin.yml" + real_file.write_text("discord:\n token: fake\n", encoding="utf-8") + alias_dir = tmp_path / "aliases" + alias_dir.mkdir() + (alias_dir / "odin.yml").symlink_to(real_file) + + previous = active_config_path() + set_active_config_path(alias_dir / "odin.yml") + try: + roots = command_protected_roots(tmp_path / "install") + assert str(real_dir.resolve()) in roots + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(real_dir / "ws"), protected_roots=roots) + finally: + set_active_config_path(previous) + + +def test_no_active_config_protects_nothing_extra(tmp_path: Path) -> None: + """A process that never loaded a config has nothing to protect, and must + not guess a path — guessing would reject legitimate workspaces.""" + from src.config.schema import active_config_path, set_active_config_path + from src.tools.workspace import command_protected_roots + + previous = active_config_path() + set_active_config_path(None) + try: + roots = command_protected_roots(tmp_path / "install") + assert roots == [str((tmp_path / "install").resolve()), str(Path("./data").resolve())] + finally: + set_active_config_path(previous) + + +def test_workspace_usage_walk_is_cached_between_metrics_scrapes( + fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """/metrics is unauthenticated and this directory deliberately never + prunes, so walking every file per scrape is an event-loop stall that grows + without bound (PR #239 round-9 review). Free space stays live; the walk is + reused for WORKSPACE_METRICS_TTL. + """ + import src.tools.executor as executor_module + + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "one").write_text("x" * 10, encoding="utf-8") + + walks = 0 + real_walk = executor_module.os.walk + + def counting_walk(*args, **kwargs): + nonlocal walks + walks += 1 + return real_walk(*args, **kwargs) + + monkeypatch.setattr(executor_module.os, "walk", counting_walk) + + first = executor.get_workspace_metrics() + assert first["files"] == 1 and walks == 1 + + (workspace / "two").write_text("y" * 10, encoding="utf-8") + second = executor.get_workspace_metrics() + assert walks == 1, "a second scrape inside the TTL must not re-walk" + assert second["files"] == 1, "the cached count is served" + assert "free_bytes" in second, "cheap statvfs metrics stay live" + + # Past the TTL the walk runs again and picks up the new file. + monkeypatch.setattr(executor_module, "WORKSPACE_METRICS_TTL", 0.0) + third = executor.get_workspace_metrics() + assert walks == 2 and third["files"] == 2 diff --git a/tests/test_ts_ledger_fixes.py b/tests/test_ts_ledger_fixes.py index 4a03d5eb..cc97a9af 100644 --- a/tests/test_ts_ledger_fixes.py +++ b/tests/test_ts_ledger_fixes.py @@ -22,7 +22,10 @@ def __init__(self, raw): self._raw = raw self.config = MagicMock() - async def _run_on_host(self, alias, command): + async def _run_on_host(self, alias, command, use_workspace=False): + # Recorded, not asserted: skills opt IN (arbitrary command execution), + # the audit diff tracker deliberately does NOT (PR #239 round 9). + self.last_use_workspace = use_workspace if isinstance(self._raw, Exception): raise self._raw return self._raw @@ -68,6 +71,15 @@ def _ctx(self, raw): ctx._executor = _FakeExecutor(raw) return ctx + @pytest.mark.asyncio + async def test_skill_commands_opt_into_the_workspace(self): + """PR #239 round 9: this is arbitrary command execution exposed to + user-created skills, so it is a raw command route and must land in the + workspace — omitting it left an alternate path back into the wipe.""" + ctx = self._ctx(("ok", 0)) + await ctx.run_on_host("localhost", "rm -rf data") + assert ctx._executor.last_use_workspace is True + @pytest.mark.asyncio async def test_resolved_host_returns_output_string(self): result = await self._ctx(("uptime output", 0)).run_on_host("server", "uptime") From 2cdaa52618e72d3640c38db0612e0d17554a5633 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 18:18:36 -0400 Subject: [PATCH 11/21] test: hold the whole local-execution surface closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Odin's reviews found workspace bypasses one route at a time — round 9's was SkillContext.run_on_host, arbitrary shell handed to user-written skills, still inheriting the process cwd. Finding them individually is not a contract. This enumerates every module in src/ that spawns (or replaces) a local process by AST, and requires each to be classified: routed through the workspace, or justified as not a user-command path. Adding a new spawn site fails the test until someone decides which it is — the default, inheriting Odin's cwd, is the 2026-07-27 mechanism. Twelve sites classified; two use the workspace, eight are argv-form or infrastructure paths, two are the legacy CLI surface already pinned as unreachable from Discord. Mutation-verified with a new unclassified spawn. --- tests/test_local_workspace.py | 98 +++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 76dcff2d..8123fe5f 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -1720,3 +1720,101 @@ def counting_walk(*args, **kwargs): monkeypatch.setattr(executor_module, "WORKSPACE_METRICS_TTL", 0.0) third = executor.get_workspace_metrics() assert walks == 2 and third["files"] == 2 + + +# --- The complete set of local-execution routes, held closed ----------------- + +# Every module in src/ that spawns a local process, and why it does or does not +# use the command workspace. Odin's reviews found bypasses one at a time +# (round 9: SkillContext.run_on_host); this holds the whole surface closed, so +# a NEW spawn site has to be classified rather than silently inheriting the +# process cwd. +_SPAWN_PRIMITIVES = { + "create_subprocess_shell", + "create_subprocess_exec", + "run", # subprocess.run + "Popen", + "system", # os.system + # Process REPLACEMENT counts too: it is how the self-update restarts, and + # it carries the cwd forward into the new image. + "execv", + "execve", + "execvp", + "execvpe", + "execl", + "execle", + "execlp", +} + +_CLASSIFIED_SPAWN_SITES: dict[str, str] = { + # --- uses the workspace ------------------------------------------------- + "src/tools/ssh.py": "run_local_command — takes cwd from the caller; THE seam", + "src/tools/process_manager.py": "background manage_process — resolves workspace per spawn", + # --- deliberately does not ---------------------------------------------- + "src/discord/native_tools/media.py": "argv-form ssh to a REMOTE host; local cwd is irrelevant", + "src/tools/ssh_pool.py": "argv-form ssh control-socket management, no user command text", + "src/tools/mcp_client.py": "operator-configured MCP server process, not a user command", + "src/tools/skill_manager.py": "argv-form `pip install `, no user-supplied relative path", + "src/tools/workspace.py": "`sudo -n install -d` provisioning the workspace itself", + "src/web/api/self_update.py": "argv-form git/gh during self-update, inside the install", + "src/packaging/validate.py": "build-time packaging check, not a runtime path", + "src/restart.py": "os.execve re-exec of Odin himself", + # --- legacy CLI surface, unreachable from Discord (pinned separately) ---- + "src/odin/tools/shell.py": "legacy CLI ShellTool; excluded from the Discord tool loop", + "src/odin/tools/process.py": "legacy CLI ProcessTool; excluded from the Discord tool loop", +} + + +def _modules_that_spawn_processes() -> set[str]: + """Every src/ module calling a process-spawning primitive, by AST. + + AST rather than grep so comments, docstrings and this test's own tables + cannot register as spawn sites. + """ + import ast + + repo = Path(__file__).resolve().parents[1] + found: set[str] = set() + for path in sorted((repo / "src").rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except SyntaxError: # pragma: no cover - src must always parse + continue + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not isinstance(func, ast.Attribute) or func.attr not in _SPAWN_PRIMITIVES: + continue + # `subprocess.run` / `os.system` / `asyncio.create_subprocess_*` + # only — not arbitrary `.run(...)` methods on unrelated objects. + owner = func.value + owner_name = getattr(owner, "id", None) or getattr(owner, "attr", None) + if func.attr in {"run", "Popen"} and owner_name != "subprocess": + continue + if func.attr == "system" and owner_name != "os": + continue + if func.attr.startswith("exec") and owner_name != "os": + continue + found.add(str(path.relative_to(repo))) + return found + + +def test_every_local_execution_route_is_classified() -> None: + """No unclassified way to run a local process may exist in src/. + + A new spawn site added later inherits Odin's process cwd by default — + which is exactly the 2026-07-27 mechanism. This fails until it is either + routed through the workspace or explicitly justified here. + """ + actual = _modules_that_spawn_processes() + declared = set(_CLASSIFIED_SPAWN_SITES) + + unclassified = actual - declared + assert not unclassified, ( + "these modules spawn local processes but are not classified: " + f"{sorted(unclassified)} — route them through the workspace or record why not" + ) + + stale = declared - actual + assert not stale, f"no longer spawn processes; drop from the table: {sorted(stale)}" From 074721b3d2c3a42c5e59a9a3fbbb524271344c59 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 18:34:41 -0400 Subject: [PATCH 12/21] =?UTF-8?q?fix:=20PR=20#239=20round=2010=20=E2=80=94?= =?UTF-8?q?=20the=20caller=20axis=20closed,=20config=20launch=20path,=20al?= =?UTF-8?q?ternate-config=20persistence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four blockers, and the first pair share a shape my round-9 contract missed: it classified where processes are SPAWNED, not who ASKS for one. Both fixes below are now held by a contract on that second axis. 1. validate_action executed user-supplied command text through the shared helper without the workspace, so a command check could replay the relative-path mechanism into the install. Reproduced through the real dispatch. It opts in now, with an incident replay and a fail-closed pin. 4. write_file's schema documents an absolute path but nothing enforced it, so a relative path silently wrote into the install. Rejected now with an error naming why — better than quietly redirecting into the workspace, since a write landing somewhere the caller did not choose is its own hazard. No documented capability is lost; absolute paths are untouched. Both are now covered by a table of every _exec_command/_run_on_host call site, each declared WORKSPACE or given a reason. A new call site fails until classified, and flipping an existing one's opt-in fails too. 2. The active config's LAUNCH path was unprotected. set_active_config_path canonicalizes, but restart.reexec replays sys.argv — so for an aliased config the next process opens the alias, and deleting it breaks the restart even though the target survives. Both the canonical target's directory and the launch path's own directory are protected now; the launch parent is canonicalized as a DIRECTORY so the symlink is not followed back to the target. 3. PUT /api/config persisted to a cwd-relative config.yml rather than the active config file. On an alternate-config deployment a workspace change lived in bot.config, was validated by the self-update preflight, and then vanished on re-exec — directly contradicting the preflight's contract that it validates what the restarted process will use. It now writes through active_config_path(), which llm_admin already did. Pinned end to end, including that the decoy cwd config.yml stays untouched. Metrics (his correction): the walk no longer runs on the calling thread at all — a stale cache starts a single-flight background refresh and the previous numbers are served meanwhile, with the timestamp taken on COMPLETION so a walk longer than the TTL is not stale the moment it finishes. Free space and inodes stay live on every scrape, now asserted rather than assumed. Mutation-verified: removing the validate_action opt-in, the write_file absolute check, the launch-path root, the background thread, or the active-config persistence each fails its test. Gates, all four as CI runs them: 7607 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0. git diff --check clean. --- src/config/schema.py | 15 +- src/tools/executor.py | 89 +++++--- src/tools/handlers/files_docs.py | 12 + src/tools/handlers/validation.py | 8 +- src/tools/workspace.py | 29 ++- src/web/api/config_admin.py | 10 +- tests/test_local_workspace.py | 343 ++++++++++++++++++++++++++--- tests/test_web_api_config_admin.py | 77 ++++++- 8 files changed, 512 insertions(+), 71 deletions(-) diff --git a/src/config/schema.py b/src/config/schema.py index 6ac3a5b7..ca83f1aa 100644 --- a/src/config/schema.py +++ b/src/config/schema.py @@ -905,6 +905,18 @@ def replacer(match: re.Match) -> str: # (a test or one-off script that never called load_config) cannot silently # overwrite a real deployment's config.yml from the wrong working directory. _ACTIVE_CONFIG_PATH: Path | None = None +# The path AS GIVEN (absolutized, symlinks intact). restart.reexec() replays +# sys.argv, so an alias like /etc/odin/config.yml -> /srv/real/odin.yml is what +# the restarted process opens — protecting only the canonical target would let +# a relative command delete the alias and break the next restart (PR #239 +# round-10 review, reproduced). +_LAUNCH_CONFIG_PATH: Path | None = None + + +def active_config_launch_path() -> Path | None: + """The config path as given on the command line, absolutized but with + symlinks intact — what ``restart.reexec()`` will hand the next process.""" + return _LAUNCH_CONFIG_PATH def active_config_path() -> Path | None: @@ -917,8 +929,9 @@ def set_active_config_path(path: str | Path | None) -> None: """Record (or clear) the active config path. ``load_config`` calls this on a successful load; tests/tools that persist a hand-built Config point it at their own file.""" - global _ACTIVE_CONFIG_PATH + global _ACTIVE_CONFIG_PATH, _LAUNCH_CONFIG_PATH _ACTIVE_CONFIG_PATH = Path(path).resolve() if path is not None else None + _LAUNCH_CONFIG_PATH = Path(os.path.abspath(path)) if path is not None else None def load_config(path: str | Path = "config.yml") -> Config: diff --git a/src/tools/executor.py b/src/tools/executor.py index 7eb355b5..49691524 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -5,6 +5,7 @@ import json import os import shutil +import threading import time from pathlib import Path from typing import Any @@ -188,8 +189,11 @@ def __init__( # is what would restore the 2026-07-27 hazard. self._local_workspace: str | None = None self._local_workspace_resolved = False - # (monotonic_stamp, bytes, files) from the last workspace usage walk. + # (completed_at_monotonic, bytes, files) from the last usage walk, + # refreshed off-thread; the lock makes the refresh single-flight. self._workspace_usage_cache: tuple[float, float, float] | None = None + self._workspace_usage_refreshing = False + self._workspace_usage_lock = threading.Lock() self._memory_path = Path(memory_path) if memory_path else None self._browser_manager = browser_manager self._permission_manager = permission_manager @@ -374,18 +378,52 @@ def get_workspace_metrics(self) -> dict[str, float]: except Exception: return {} - # The size/count walk is CACHED for WORKSPACE_METRICS_TTL. /metrics is - # unauthenticated and can be scraped at any rate, and this deliberately - # un-pruned directory only grows — walking every file synchronously on - # each request is an event-loop stall that gets worse over time - # (PR #239 round-9 review). Free space and inodes are cheap statvfs - # calls and stay live. On a stale-cache read the previous walk's - # numbers are reused; on the first-ever call the walk runs inline. - now = time.monotonic() + # The size/count walk NEVER runs on the calling thread. /metrics is + # unauthenticated and served on the event loop, and this directory + # deliberately never prunes, so a synchronous walk is a stall that only + # grows (PR #239 round-9/10 review). A stale cache triggers a + # single-flight background refresh and the previous numbers are served + # meanwhile; until the first refresh completes, usage is simply absent + # rather than blocking. The timestamp is recorded when the walk + # COMPLETES — stamping at the start makes a walk longer than the TTL + # instantly stale, so every scrape would launch another one. + self._refresh_workspace_usage(root) cached = getattr(self, "_workspace_usage_cache", None) - if cached is not None and (now - cached[0]) < WORKSPACE_METRICS_TTL: - total_bytes, files = cached[1], cached[2] - else: + metrics: dict[str, float] = {} + if cached is not None: + metrics["bytes"], metrics["files"] = cached[1], cached[2] + # Cheap statvfs calls: these stay LIVE on every scrape, cached usage + # or not, because free space is the number an operator alerts on. + try: + usage = shutil.disk_usage(root) + metrics["free_bytes"] = float(usage.free) + except OSError: + pass + try: + stats = os.statvfs(root) + metrics["free_inodes"] = float(stats.f_favail) + except (OSError, AttributeError): + pass + return metrics + + def _refresh_workspace_usage(self, root: Path) -> None: + """Start a single-flight background walk if the cache is stale. + + Never blocks the caller. Never raises: metrics collection must not be + able to break anything, least of all the event loop it runs on. + """ + cached = getattr(self, "_workspace_usage_cache", None) + if cached is not None and (time.monotonic() - cached[0]) < WORKSPACE_METRICS_TTL: + return + lock = getattr(self, "_workspace_usage_lock", None) + if lock is None: # __new__ patch seam + return + with lock: + if self._workspace_usage_refreshing: + return + self._workspace_usage_refreshing = True + + def _walk() -> None: total_bytes = 0.0 files = 0.0 try: @@ -397,20 +435,21 @@ def get_workspace_metrics(self) -> dict[str, float]: except OSError: pass except OSError: - return {} - self._workspace_usage_cache = (now, total_bytes, files) - metrics = {"bytes": total_bytes, "files": files} - try: - usage = shutil.disk_usage(root) - metrics["free_bytes"] = float(usage.free) - except OSError: - pass + return + finally: + with lock: + self._workspace_usage_refreshing = False + # Stamped on COMPLETION, so a long walk does not read as stale the + # moment it finishes. + self._workspace_usage_cache = (time.monotonic(), total_bytes, files) + try: - stats = os.statvfs(root) - metrics["free_inodes"] = float(stats.f_favail) - except (OSError, AttributeError): - pass - return metrics + threading.Thread( + target=_walk, name="odin-workspace-usage", daemon=True + ).start() + except Exception: # pragma: no cover - thread creation cannot realistically fail + with lock: + self._workspace_usage_refreshing = False def _ensure_local_workspace(self) -> str: """Resolve and re-validate the cwd for local user commands. diff --git a/src/tools/handlers/files_docs.py b/src/tools/handlers/files_docs.py index 1820bee6..eb75d1e6 100644 --- a/src/tools/handlers/files_docs.py +++ b/src/tools/handlers/files_docs.py @@ -46,6 +46,18 @@ async def _handle_write_file(self, inp: dict) -> str: return "Error: 'content' is required for write_file." if not host: return "Error: 'host' is required for write_file." + # The schema documents this path as absolute, but nothing enforced it, + # so a relative path silently resolved against Odin's install directory + # and wrote there (PR #239 round-10 review, reproduced). Rejecting is + # better than quietly redirecting into the workspace: a write whose + # destination the caller did not choose is its own hazard, and no + # documented capability is lost. + if not str(path).startswith("/"): + return ( + f"Error: write_file requires an absolute path, got {path!r}. " + "A relative path would resolve against Odin's working directory " + "rather than where you intend." + ) safe_path = shlex.quote(path) # Govern the write before executing — write_file reaches the filesystem # via _run_on_host, which does NOT itself govern. Check a representative diff --git a/src/tools/handlers/validation.py b/src/tools/handlers/validation.py index 988aca55..6f09499b 100644 --- a/src/tools/handlers/validation.py +++ b/src/tools/handlers/validation.py @@ -61,7 +61,13 @@ async def _exec( return 1, f"validate_action: governor check raised {type(ge).__name__}: {ge}" if not decision.allowed: return 1, f"governor-blocked: {decision.denial_message()}" - return await self._exec_command(address, command, ssh_user, timeout=timeout) + # use_workspace=True: command checks execute user-supplied command + # text, which makes this a raw command route like run_command — it + # was still inheriting the install cwd and could replay the + # relative-path wipe (PR #239 round-10 review, reproduced). + return await self._exec_command( + address, command, ssh_user, timeout=timeout, use_workspace=True + ) report = await run_bundle( raw_checks, diff --git a/src/tools/workspace.py b/src/tools/workspace.py index 42529fb9..c15b63b7 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -109,20 +109,35 @@ def _reject_overlap( ) -def _active_config_file() -> str | None: - """Absolute path the live config was loaded from, if any. +def _active_config_roots() -> list[tuple[str, bool]]: + """Config file paths the live process depends on, if any. + + BOTH the canonical target and the path as given on the command line: the + self-update re-exec replays ``sys.argv``, so an aliased config + (``/etc/odin/config.yml -> /srv/real/odin.yml``) needs its alias directory + protected too — deleting the alias breaks the next restart even though the + target survives (PR #239 round-10 review, reproduced). Imported lazily and guarded: workspace validation must never depend on the config module being importable, and a process that never loaded a config (tests, one-off scripts) has nothing to protect. """ try: - from ..config.schema import active_config_path + from ..config.schema import active_config_launch_path, active_config_path - path = active_config_path() + canonical = active_config_path() + launch = active_config_launch_path() except Exception: # pragma: no cover - defensive - return None - return str(path) if path else None + return [] + roots: list[tuple[str, bool]] = [] + if canonical: + roots.append((str(canonical), True)) + if launch: + # As a DIRECTORY: the launch path's own parent, canonicalized. Treating + # it as a file would resolve the symlink and yield the target's + # directory again — the alias directory is the one re-exec reopens. + roots.append((str(Path(launch).parent), False)) + return roots def _dotted(source: object, path: str) -> object: @@ -179,7 +194,7 @@ def command_protected_roots( # too, or a bare relative command can delete the file needed to restart — # reproduced with an alternate config whose parent WAS the configured # workspace (PR #239 round-9 review). - declared.append((_active_config_file(), True)) + declared.extend(_active_config_roots()) for configured, is_file in declared: if configured is None: diff --git a/src/web/api/config_admin.py b/src/web/api/config_admin.py index e0e9dd71..32fec9f3 100644 --- a/src/web/api/config_admin.py +++ b/src/web/api/config_admin.py @@ -17,7 +17,7 @@ from aiohttp import web from ... import restart -from ...config.schema import Config +from ...config.schema import Config, active_config_path from ...odin_log import get_logger from ...setup_wizard import ( build_config, @@ -387,7 +387,13 @@ async def update_config(request: web.Request) -> web.Response: # pre-normalized merge, so fields removed from the schema (a legacy # model_routing block, auxiliary tasks/max_tokens/credentials_path) # can't linger on disk after runtime has dropped them. - config_path = Path("config.yml") + # The ACTIVE config, not whatever config.yml sits in the cwd: Odin can + # be launched with `python -m src /somewhere/odin.yml`, and writing to + # the wrong file meant a change appeared in live config and in the + # self-update preflight but vanished on re-exec — contradicting the + # preflight's contract that it validates what the restarted process + # will use (PR #239 round-10 review). llm_admin already does this. + config_path = active_config_path() or Path("config.yml") if config_path.exists(): try: await asyncio.to_thread(_write_config, config_path, new_config.model_dump()) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 8123fe5f..83fb6ba9 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -25,9 +25,12 @@ import stat import subprocess import sys +import threading +import time import zipfile from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch import pytest @@ -728,6 +731,21 @@ def test_leaf_symlinked_data_paths_protect_the_target(tmp_path: Path) -> None: # --- workspace growth metrics (the operational half of "no auto-prune") ----- +def _metrics_after_refresh(executor, timeout: float = 5.0) -> dict[str, float]: + """Scrape, wait for the off-thread usage walk, scrape again. + + Usage is refreshed in the background since round 10 — /metrics is served on + the event loop and this directory never prunes, so the walk must not run on + the calling thread. Tests that assert on usage therefore have to let the + refresh land. + """ + executor.get_workspace_metrics() + deadline = time.monotonic() + timeout + while executor._workspace_usage_cache is None and time.monotonic() < deadline: + time.sleep(0.02) + return executor.get_workspace_metrics() + + def test_workspace_metrics_report_usage(workspace: Path, fake_install: Path) -> None: """No automatic pruning was always paired with observability, so growth is alertable and cleanup stays an explicit operator action.""" @@ -736,7 +754,7 @@ def test_workspace_metrics_report_usage(workspace: Path, fake_install: Path) -> (workspace / "nested").mkdir() (workspace / "nested" / "more.txt").write_text("hello", encoding="utf-8") - metrics = executor.get_workspace_metrics() + metrics = _metrics_after_refresh(executor) assert metrics["files"] == 2 assert metrics["bytes"] >= 2048 assert metrics["free_bytes"] > 0 @@ -758,6 +776,7 @@ def test_workspace_gauges_render_for_prometheus( executor = _executor_with_workspace(workspace, fake_install) (workspace / "f").write_bytes(b"12345") + _metrics_after_refresh(executor) # let the background usage walk land collector = MetricsCollector() collector.register_source("workspace", executor.get_workspace_metrics) rendered = collector.render() @@ -831,6 +850,9 @@ def _fail_usage(_path: object) -> object: def _fail_statvfs(_path: object) -> object: raise OSError("statvfs unavailable") + metrics = _metrics_after_refresh(executor) # usage lands before we break statfs + assert metrics["bytes"] == 3 and metrics["files"] == 1 + monkeypatch.setattr(_shutil, "disk_usage", _fail_usage) monkeypatch.setattr(os, "statvfs", _fail_statvfs) @@ -1684,42 +1706,29 @@ def test_no_active_config_protects_nothing_extra(tmp_path: Path) -> None: set_active_config_path(previous) -def test_workspace_usage_walk_is_cached_between_metrics_scrapes( - fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +def test_repeated_scrapes_inside_the_ttl_do_not_re_walk( + fake_install: Path, workspace: Path ) -> None: - """/metrics is unauthenticated and this directory deliberately never - prunes, so walking every file per scrape is an event-loop stall that grows - without bound (PR #239 round-9 review). Free space stays live; the walk is - reused for WORKSPACE_METRICS_TTL. - """ - import src.tools.executor as executor_module - + """A scrape loop must not walk continuously. /metrics is unauthenticated, + and the workspace deliberately never prunes.""" executor = _executor_with_workspace(workspace, fake_install) (workspace / "one").write_text("x" * 10, encoding="utf-8") walks = 0 - real_walk = executor_module.os.walk + real_walk = os.walk def counting_walk(*args, **kwargs): nonlocal walks walks += 1 return real_walk(*args, **kwargs) - monkeypatch.setattr(executor_module.os, "walk", counting_walk) + with patch("src.tools.executor.os.walk", counting_walk): + _metrics_after_refresh(executor) + assert walks == 1 + for _ in range(5): + executor.get_workspace_metrics() + assert walks == 1, "scrapes inside the TTL must reuse the cached walk" - first = executor.get_workspace_metrics() - assert first["files"] == 1 and walks == 1 - - (workspace / "two").write_text("y" * 10, encoding="utf-8") - second = executor.get_workspace_metrics() - assert walks == 1, "a second scrape inside the TTL must not re-walk" - assert second["files"] == 1, "the cached count is served" - assert "free_bytes" in second, "cheap statvfs metrics stay live" - - # Past the TTL the walk runs again and picks up the new file. - monkeypatch.setattr(executor_module, "WORKSPACE_METRICS_TTL", 0.0) - third = executor.get_workspace_metrics() - assert walks == 2 and third["files"] == 2 # --- The complete set of local-execution routes, held closed ----------------- @@ -1818,3 +1827,287 @@ def test_every_local_execution_route_is_classified() -> None: stale = declared - actual assert not stale, f"no longer spawn processes; drop from the table: {sorted(stale)}" + + +def test_wiring_hardcoded_state_paths_are_all_covered(tmp_path: Path) -> None: + """The third leg of the protected-root surface. + + Config-declared paths are held by the exhaustive table, and the active + config file is handled as runtime state — but wiring also hardcodes a set + of live-state paths (learned, channel config and logs, host access, skills, + schedules, audit, API tokens). They are covered today because they sit + beside memory.json under ./data, whose parent IS protected. This asserts + that rather than assuming it, so relocating one out of ./data without + declaring it fails here instead of silently losing protection. + """ + import ast + + from src.tools.workspace import command_protected_roots + + repo = Path(__file__).resolve().parents[1] + tree = ast.parse((repo / "src" / "discord" / "wiring.py").read_text(encoding="utf-8")) + hardcoded = { + node.value + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.value.startswith("./data/") + } + assert len(hardcoded) >= 8, f"expected wiring's data paths, found {sorted(hardcoded)}" + + roots = [Path(r) for r in command_protected_roots(tmp_path / "install")] + for raw in sorted(hardcoded): + resolved = Path(raw).expanduser().resolve() + covered = any(resolved == root or root in resolved.parents for root in roots) + assert covered, ( + f"{raw} is live state that no protected root covers — declare it in " + "_DECLARED_STATE_PATHS or keep it under the protected data directory" + ) + + +# --- Every _exec_command / _run_on_host CALLER classified -------------------- + +# The route-classification test above covers where processes are SPAWNED. This +# covers who ASKS for one, which is the axis Odin's round-10 review found twice +# (validate_action and write_file both reached the install cwd through the +# shared helpers). A call site is either a raw user-command route that opts +# into the workspace, or it must say why it does not. +_CLASSIFIED_COMMAND_CALLERS: dict[tuple[str, str], str] = { + # --- opt in: arbitrary user-supplied command text ----------------------- + ("src/tools/handlers/system.py", "_handle_run_command"): "WORKSPACE", + ("src/tools/handlers/system.py", "_handle_run_script"): "WORKSPACE", + ("src/tools/handlers/system.py", "_run_one"): "WORKSPACE", # run_command_multi + ("src/tools/handlers/validation.py", "_exec"): "WORKSPACE", + ("src/tools/skill_context.py", "run_on_host"): "WORKSPACE", + # --- do not: fixed command shapes with caller-supplied absolute paths ---- + ("src/tools/handlers/devops.py", "_handle_git_ops"): "documented repo default is the cwd", + ("src/tools/handlers/devops.py", "_handle_kubectl"): "fixed kubectl argv", + ("src/tools/handlers/devops.py", "_handle_docker_ops"): "docker build context is caller-given", + ("src/tools/handlers/devops.py", "_handle_terraform_ops"): "terraform working_dir is explicit", + ("src/tools/handlers/coding.py", "_handle_claude_code"): "claude_code has its own cwd config", + ("src/tools/handlers/browser_web.py", "_handle_http_probe"): "fixed curl argv", + ("src/tools/handlers/files_docs.py", "_handle_read_file"): "reads a caller-given path", + ("src/tools/handlers/files_docs.py", "_handle_write_file"): "absolute path enforced", + ("src/tools/handlers/files_docs.py", "_handle_analyze_pdf"): "reads a caller-given path", + ("src/discord/native_tools/media.py", "_handle_analyze_image"): "base64 of a given path", + ("src/audit/diff_tracker.py", "capture_before"): "cat of a governed absolute path", + # --- plumbing ----------------------------------------------------------- + ("src/tools/executor.py", "_run_on_host"): "the shared helper itself", + ("src/tools/executor.py", "__init__"): "HandlerDeps lambdas forwarding **kwargs", +} + + +def _command_call_sites() -> dict[tuple[str, str], bool]: + """(file, enclosing function) -> whether it passes use_workspace=True.""" + import ast + + repo = Path(__file__).resolve().parents[1] + sites: dict[tuple[str, str], bool] = {} + for path in sorted((repo / "src").rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except SyntaxError: # pragma: no cover + continue + parents: dict[ast.AST, ast.AST] = {} + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + parents[child] = node + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not isinstance(func, ast.Attribute): + continue + if func.attr not in {"_exec_command", "_run_on_host"}: + continue + enclosing = "" + walker: ast.AST | None = node + while walker is not None: + if isinstance(walker, (ast.FunctionDef, ast.AsyncFunctionDef)): + enclosing = walker.name + break + walker = parents.get(walker) + opts_in = any( + kw.arg == "use_workspace" + and isinstance(kw.value, ast.Constant) + and kw.value.value is True + for kw in node.keywords + ) + key = (str(path.relative_to(repo)), enclosing) + sites[key] = sites.get(key, False) or opts_in + return sites + + +def test_every_command_caller_is_classified() -> None: + """Round-10 regression, both blockers of that shape at once. + + validate_action and write_file each reached Odin's install directory + through these shared helpers because nobody had decided what they were. + Adding a new call site now fails until it is classified, and flipping an + existing one's opt-in fails too. + """ + actual = _command_call_sites() + declared = set(_CLASSIFIED_COMMAND_CALLERS) + + unclassified = set(actual) - declared + assert not unclassified, ( + f"unclassified command call sites: {sorted(unclassified)} — decide whether " + "each is a raw user-command route (use_workspace=True) or record why not" + ) + stale = declared - set(actual) + assert not stale, f"no longer call the helpers; drop from the table: {sorted(stale)}" + + for key, expected in _CLASSIFIED_COMMAND_CALLERS.items(): + should_opt_in = expected == "WORKSPACE" + assert actual[key] is should_opt_in, ( + f"{key[0]}::{key[1]} use_workspace={actual[key]}, expected {should_opt_in} " + f"({expected})" + ) + + +async def test_validate_action_command_check_runs_in_the_workspace( + fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-10 blocker 1, reproduced by Odin through the real dispatch: a + command check is user-supplied command text, so it can replay the same + relative-path mechanism.""" + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(workspace, fake_install) + + await executor.execute("validate_action", { + "host": "localhost", + "checks": [{"type": "command", "target": "touch from-validate"}], + }) + assert (workspace / "from-validate").exists(), "the check ran in the workspace" + assert not (fake_install / "from-validate").exists(), "and NOT in the install" + + +async def test_validate_action_fails_closed_on_an_invalid_workspace( + fake_install: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Same fail-closed contract as every other raw command route.""" + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(tmp_path / "no-parent" / "ws", fake_install) + + result = await executor.execute("validate_action", { + "host": "localhost", + "checks": [{"type": "command", "target": "touch should-not-run"}], + }) + assert not (fake_install / "should-not-run").exists() + assert not (workspace_leaked := (tmp_path / "should-not-run")).exists(), workspace_leaked + assert result is not None + + +@pytest.mark.parametrize("relative", ["notes.md", "./notes.md", "data/notes.md"]) +async def test_write_file_rejects_relative_paths( + fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch, relative: str +) -> None: + """Round-10 blocker 4, reproduced by Odin: the schema documents an absolute + path but nothing enforced it, so a relative one wrote into the install.""" + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(workspace, fake_install) + + result = await executor.execute("write_file", { + "host": "localhost", "path": relative, "content": "x", + }) + assert not result.ok + assert "absolute path" in str(result.output) + assert not (fake_install / relative).exists(), "nothing may be written to the install" + + +async def test_write_file_still_accepts_absolute_paths( + fake_install: Path, workspace: Path, tmp_path: Path +) -> None: + """The documented capability is untouched.""" + executor = _executor_with_workspace(workspace, fake_install) + target = tmp_path / "written.txt" + result = await executor.execute("write_file", { + "host": "localhost", "path": str(target), "content": "hello", + }) + assert result.ok, result.output + assert target.read_text(encoding="utf-8").strip() == "hello" + + +def test_aliased_config_protects_both_the_alias_and_the_target(tmp_path: Path) -> None: + """Round-10 blocker 2, reproduced by Odin. + + set_active_config_path canonicalizes, but restart.reexec replays sys.argv — + so the launch path is what the next process opens. Deleting the alias + breaks the restart even though the target survives. + """ + from src.config.schema import active_config_path, set_active_config_path + from src.tools.workspace import command_protected_roots + + real_dir = tmp_path / "real" + real_dir.mkdir() + real_file = real_dir / "odin.yml" + real_file.write_text("discord:\n token: fake\n", encoding="utf-8") + alias_dir = tmp_path / "alias" + alias_dir.mkdir() + (alias_dir / "odin.yml").symlink_to(real_file) + + previous = active_config_path() + set_active_config_path(alias_dir / "odin.yml") + try: + roots = command_protected_roots(tmp_path / "install") + assert str(real_dir.resolve()) in roots, "canonical target" + assert str(alias_dir.resolve()) in roots, "launch path re-exec will reopen" + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(alias_dir), protected_roots=roots) + finally: + set_active_config_path(previous) + + +def test_workspace_usage_is_never_walked_on_the_calling_thread( + fake_install: Path, workspace: Path +) -> None: + """Round-10 metrics correction: /metrics is served on the event loop, so + the walk must happen off-thread, and free space must stay live even when + usage is served from cache.""" + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "one").write_text("x" * 10, encoding="utf-8") + + calling_thread = threading.get_ident() + walk_threads: list[int] = [] + real_walk = os.walk + + def recording_walk(*args, **kwargs): + walk_threads.append(threading.get_ident()) + return real_walk(*args, **kwargs) + + with patch("src.tools.executor.os.walk", recording_walk): + first = executor.get_workspace_metrics() + # Free space is live from the very first scrape, with no walk yet. + assert "free_bytes" in first and "free_inodes" in first + deadline = time.monotonic() + 5 + while executor._workspace_usage_cache is None and time.monotonic() < deadline: + time.sleep(0.02) + + assert walk_threads, "the refresh never ran" + assert calling_thread not in walk_threads, "the walk must not block the caller" + + second = executor.get_workspace_metrics() + assert second["files"] == 1 and second["bytes"] >= 10 + assert "free_bytes" in second and "free_inodes" in second, "cheap metrics stay live" + + +def test_workspace_usage_timestamp_is_recorded_on_completion( + fake_install: Path, workspace: Path +) -> None: + """Stamping at the START would make any walk longer than the TTL stale the + instant it finished, so every scrape would launch another one.""" + executor = _executor_with_workspace(workspace, fake_install) + started = time.monotonic() + + def slow_walk(*args, **kwargs): + time.sleep(0.3) + return iter(()) + + with patch("src.tools.executor.os.walk", slow_walk): + executor.get_workspace_metrics() + deadline = time.monotonic() + 5 + while executor._workspace_usage_cache is None and time.monotonic() < deadline: + time.sleep(0.02) + + stamped_at = executor._workspace_usage_cache[0] + assert stamped_at >= started + 0.3, "the stamp must be taken after the walk finished" diff --git a/tests/test_web_api_config_admin.py b/tests/test_web_api_config_admin.py index 0493ee27..8d635c20 100644 --- a/tests/test_web_api_config_admin.py +++ b/tests/test_web_api_config_admin.py @@ -274,13 +274,21 @@ async def test_update_config_persists_normalized_dropping_removed_keys(self): from pathlib import Path from ruamel.yaml import YAML + + from src.config.schema import active_config_path, set_active_config_path + Path("config.yml").write_text("discord:\n token: fake\n") - app, bot = _app(register_discord_config) - async with TestClient(TestServer(app)) as c: - r = await c.put("/api/config", json={ - "openai_codex": {"model_routing": {"enabled": True}}, - }) - assert r.status == 200 + previous = active_config_path() + set_active_config_path(Path("config.yml")) + try: + app, bot = _app(register_discord_config) + async with TestClient(TestServer(app)) as c: + r = await c.put("/api/config", json={ + "openai_codex": {"model_routing": {"enabled": True}}, + }) + assert r.status == 200 + finally: + set_active_config_path(previous) oc = YAML().load(Path("config.yml").read_text()).get("openai_codex", {}) assert "model_routing" not in oc @@ -300,12 +308,20 @@ async def test_blanking_the_workspace_normalizes_everywhere(self): from ruamel.yaml import YAML from src.config.schema import Config as _Config + from src.config.schema import active_config_path, set_active_config_path Path("config.yml").write_text("discord:\n token: fake\n") - app, bot = _app(register_discord_config) - async with TestClient(TestServer(app)) as c: - r = await c.put("/api/config", json={"tools": {"local_working_dir": " "}}) - assert r.status == 200 + # Explicit: persistence targets the ACTIVE config path, and any earlier + # test that called load_config leaves that module global set. + previous = active_config_path() + set_active_config_path(Path("config.yml")) + try: + app, bot = _app(register_discord_config) + async with TestClient(TestServer(app)) as c: + r = await c.put("/api/config", json={"tools": {"local_working_dir": " "}}) + assert r.status == 200 + finally: + set_active_config_path(previous) default = "/var/lib/odin-workspace" assert bot.config.tools.local_working_dir == default, "runtime config" @@ -314,6 +330,47 @@ async def test_blanking_the_workspace_normalizes_everywhere(self): reloaded = _Config(**YAML().load(Path("config.yml").read_text())) assert reloaded.tools.local_working_dir == default, "fresh reload" + @pytest.mark.asyncio + async def test_writes_to_the_active_config_not_cwd_config_yml(self): + """PR #239 round-10 blocker 3: alternate-config deployments. + + Odin can be launched with `python -m src /somewhere/odin.yml`, and + restart.reexec replays that argument. Persisting to a cwd-relative + config.yml meant a change lived in bot.config, was validated by the + self-update preflight, and then vanished on re-exec — contradicting the + preflight's contract that it validates what the restarted process will + use. llm_admin already wrote to the active path; this one did not. + """ + from pathlib import Path + + from ruamel.yaml import YAML + + from src.config.schema import active_config_path, set_active_config_path + + alternate = Path("odin-alternate.yml") + alternate.write_text("discord:\n token: fake\n") + decoy = Path("config.yml") + decoy.write_text("discord:\n token: fake\n") + + previous = active_config_path() + set_active_config_path(alternate) + try: + app, bot = _app(register_discord_config) + async with TestClient(TestServer(app)) as c: + r = await c.put( + "/api/config", json={"tools": {"local_working_dir": "/srv/ws"}} + ) + assert r.status == 200 + finally: + set_active_config_path(previous) + + written = YAML().load(alternate.read_text())["tools"]["local_working_dir"] + assert written == "/srv/ws", "the ACTIVE config must receive the change" + assert bot.config.tools.local_working_dir == "/srv/ws", "runtime agrees" + assert "tools" not in (YAML().load(decoy.read_text()) or {}), ( + "the cwd config.yml is a decoy here and must not be written" + ) + @pytest.mark.asyncio async def test_health_and_resource_and_streams(self): app, bot = _app(register_discord_config) From 25c7fb3c34f64c920ffa4d0caaf0733df0d3234b Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 18:45:32 -0400 Subject: [PATCH 13/21] test: cover every usage-refresh branch deterministically CI's coverage ratchet caught one line my local run did not: the background usage walk made coverage TIMING-DEPENDENT. Locally the thread happened to land in a way that exercised the single-flight early return; on CI it did not, so executor.py grew a dark line and the gate failed. A test that passes because a thread won a race is not a test. Every branch of the refresher is now driven deterministically, with no reliance on scheduling: the single-flight guard (a refresh already in flight starts no second thread), the __new__ patch seam with no lock (degrades, never raises), an unreadable workspace (no usage reported AND the in-flight flag cleared, so later scrapes still work), and a file that cannot be stat'd mid-walk (counted, not sized, rather than aborting the scan). Gates: 7611 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0, git diff --check clean. --- tests/test_local_workspace.py | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 83fb6ba9..7590b4ae 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2111,3 +2111,73 @@ def slow_walk(*args, **kwargs): stamped_at = executor._workspace_usage_cache[0] assert stamped_at >= started + 0.3, "the stamp must be taken after the walk finished" + + +def test_usage_refresh_is_single_flight(fake_install: Path, workspace: Path) -> None: + """A slow walk must not have a second one piled on top by the next scrape.""" + executor = _executor_with_workspace(workspace, fake_install) + executor._workspace_usage_refreshing = True # a walk is already in flight + + started = [] + def _record(**kw): + started.append(kw) + return _NoThread() + + with patch("src.tools.executor.threading.Thread", _record): + executor.get_workspace_metrics() + assert not started, "a refresh was already running; a second must not start" + + +class _NoThread: + def start(self) -> None: # pragma: no cover - never reached when single-flight holds + raise AssertionError("thread should not have been started") + + +def test_usage_refresh_is_inert_without_executor_state(workspace: Path) -> None: + """The sanctioned __new__ patch seam builds executors without __init__, so + the lock may not exist. Metrics must degrade, never raise.""" + from src.tools.executor import ToolExecutor as _Executor + + bare = _Executor.__new__(_Executor) + bare._refresh_workspace_usage(workspace) # must not raise + assert getattr(bare, "_workspace_usage_cache", None) is None + + +def test_usage_refresh_survives_a_failing_walk(fake_install: Path, workspace: Path) -> None: + """An unreadable workspace leaves usage unreported and, critically, clears + the in-flight flag so later scrapes can still refresh.""" + executor = _executor_with_workspace(workspace, fake_install) + + def _boom(*_a, **_kw): + raise OSError("walk refused") + + with patch("src.tools.executor.os.walk", _boom): + executor.get_workspace_metrics() + deadline = time.monotonic() + 5 + while executor._workspace_usage_refreshing and time.monotonic() < deadline: + time.sleep(0.02) + + assert executor._workspace_usage_cache is None, "no usage numbers to report" + assert executor._workspace_usage_refreshing is False, "the flag must not stick" + + # ...and a later scrape still refreshes normally. + (workspace / "f").write_text("xyz", encoding="utf-8") + assert _metrics_after_refresh(executor)["files"] == 1 + + +def test_usage_refresh_skips_files_it_cannot_stat( + fake_install: Path, workspace: Path +) -> None: + """A file that vanishes mid-walk is counted but not sized, rather than + aborting the whole scan.""" + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "gone").write_text("x" * 5, encoding="utf-8") + + def _refuse(*_a, **_kw): + raise OSError("stat refused") + + with patch("src.tools.executor.os.lstat", _refuse): + metrics = _metrics_after_refresh(executor) + + assert metrics["files"] == 1 + assert metrics["bytes"] == 0 From 55b5bb206384a07cb725d83ce0ef85e3d6a757c3 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 19:09:50 -0400 Subject: [PATCH 14/21] fix: close the operator-visible gaps this change left open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by sweeping the surfaces this change touches instead of waiting for review to name them one at a time. Both are mine, both were shipped-and-silent. 1. THE KNOB WAS UNDOCUMENTED. tools.local_working_dir appeared in neither the tracked config.yml template nor docs/configuration.md, despite CONTRIBUTING requiring config examples stay aligned with reality. An operator could not discover the one setting that governs where commands run — at precisely the moment they would need it, since an unusable workspace takes local commands away. The template now carries it with the reasoning; a test pins the template against the schema so they cannot drift. The default was also spelled twice (field default and blank-normalizer). It is now DEFAULT_LOCAL_WORKING_DIR, once. 2. THE STARTUP REPORT SAID NOTHING ABOUT IT. Local commands fail closed on a bad workspace by design, but the boot diagnostics — the surface an operator actually reads — had no check for it. The symptom would have been every run_command failing with nothing to explain why. There is now a local_workspace check, registered in _CONFIG_CHECKS like every other one, deriving its roots from the SAME shared function the executor uses; a check applying different rules would be worse than none, since it would pass a workspace the runtime then refuses. Diagnostics go from 7 to 8 config checks (deploy output 8/8 -> 9/9). Registering it changed every test that runs the full report, whose config doubles are MagicMocks — a mock attribute is not a workspace. Those doubles now carry a real 0700 directory. That knock-on is the same "new entry -> all consumers" trace that should have preceded the change. Mutation-verified: unregistering the check, or removing the knob from the template, each fails its test. Gates: 7616 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0, git diff --check clean. --- config.yml | 9 ++++ src/config/schema.py | 10 +++- src/health/startup.py | 49 +++++++++++++++++++ tests/test_local_workspace.py | 79 +++++++++++++++++++++++++++++++ tests/test_startup_diagnostics.py | 31 +++++++++++- 5 files changed, 175 insertions(+), 3 deletions(-) diff --git a/config.yml b/config.yml index 480df65b..97f3f9de 100644 --- a/config.yml +++ b/config.yml @@ -92,6 +92,15 @@ sessions: tools: enabled: true + # Working directory for local user commands (run_command, run_script, + # run_command_multi, background processes, skill run_on_host). A bare + # relative path in a command resolves HERE, not in the install directory — + # which is what stopped an extracted archive's `rm -rf data` from deleting + # Odin's own live state. Must be an absolute path, owned by the service + # account, mode 0700, and outside the install and data directories. + # Provisioned automatically on start; if it is unusable, local commands + # refuse to run rather than falling back to the install directory. + local_working_dir: /var/lib/odin-workspace # Customize SSH key path for your deployment ssh_key_path: /app/.ssh/id_ed25519 # Customize SSH known hosts path for your deployment diff --git a/src/config/schema.py b/src/config/schema.py index ca83f1aa..861f20a1 100644 --- a/src/config/schema.py +++ b/src/config/schema.py @@ -187,6 +187,12 @@ class GovernorConfig(BaseModel): host_overrides: dict[str, str] = Field(default_factory=dict) +# The default local command workspace, spelled ONCE: the field default, the +# blank-value normalizer, the tracked config.yml template and the packaging +# scripts must never drift apart. +DEFAULT_LOCAL_WORKING_DIR = "/var/lib/odin-workspace" + + class ToolsConfig(BaseModel): enabled: bool = True governor: GovernorConfig = GovernorConfig() @@ -234,7 +240,7 @@ class ToolsConfig(BaseModel): # read it in the next, which would cost capability. Restart-required, not # hot-reloadable — swapping workspaces at runtime would break exactly the # cross-command continuity this preserves. - local_working_dir: str = "/var/lib/odin-workspace" + local_working_dir: str = DEFAULT_LOCAL_WORKING_DIR @field_validator("local_working_dir") @classmethod @@ -254,7 +260,7 @@ def _workspace_blank_means_default(cls, v): validation error on a persisted config would. """ if not isinstance(v, str) or not v.strip(): - return "/var/lib/odin-workspace" + return DEFAULT_LOCAL_WORKING_DIR return v.strip() @field_validator("command_timeout_seconds") diff --git a/src/health/startup.py b/src/health/startup.py index 816b9c81..9480e0d5 100644 --- a/src/health/startup.py +++ b/src/health/startup.py @@ -426,6 +426,54 @@ def check_config_sections(config: Any) -> DiagnosticResult: ) +def check_local_workspace(tools_config: Any) -> DiagnosticResult: + """Verify the local command workspace is usable. + + Local user commands FAIL CLOSED when this directory is missing, wrongly + owned, or not 0700 — deliberately, because falling back to the install + directory is the hazard this exists to remove. That makes it an operator- + visible dependency: without this check a broken workspace shows up as + every run_command failing, with nothing in the startup report to say why. + """ + from ..tools.workspace import WorkspaceError, provisioning_hint, resolve_workspace + + configured = getattr(tools_config, "local_working_dir", "") or "" + try: + workspace = resolve_workspace(configured, protected_roots=_workspace_protected_roots()) + except WorkspaceError as exc: + return DiagnosticResult( + name="local_workspace", + passed=False, + detail=f"Local command workspace unusable: {exc}", + recommendation=provisioning_hint(configured), + metadata={"configured": configured}, + ) + except Exception as exc: # pragma: no cover - defensive + return DiagnosticResult( + name="local_workspace", + passed=False, + detail=f"Local command workspace check failed: {exc}", + recommendation=provisioning_hint(configured), + metadata={"configured": configured}, + ) + return DiagnosticResult( + name="local_workspace", + passed=True, + detail=f"Local command workspace ready: {workspace}", + metadata={"path": str(workspace)}, + ) + + +def _workspace_protected_roots() -> list[str]: + """Protected roots for the diagnostic, from the same shared derivation the + executor uses — a check that applied different rules would be worse than + no check at all.""" + from ..config.schema import Config # noqa: F401 - imported for typing clarity + from ..tools.workspace import command_protected_roots + + return command_protected_roots(Path(__file__).resolve().parents[2]) + + def check_data_directories() -> DiagnosticResult: """Verify core data directories exist or can be created.""" dirs = [ @@ -508,6 +556,7 @@ def check_codex_model(codex_config: Any) -> DiagnosticResult: ("ssh_hosts", check_ssh_hosts, "tools"), ("sessions_directory", check_sessions_directory, "sessions"), ("knowledge_db", check_knowledge_db, "search"), + ("local_workspace", check_local_workspace, "tools"), ("config_consistency", check_config_sections, None), # uses full Config ] diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 7590b4ae..8b48c0ae 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2181,3 +2181,82 @@ def _refuse(*_a, **_kw): assert metrics["files"] == 1 assert metrics["bytes"] == 0 + + +# --- The operator-visible surface: startup diagnostics and the template ------ + + +def test_startup_diagnostic_reports_an_unusable_workspace(tmp_path: Path) -> None: + """Local commands fail closed on a bad workspace, so the startup report has + to SAY so. Without this the symptom is every run_command failing with + nothing in the diagnostics to explain it. + + This gap was mine, found by sweeping the operator-visible surfaces rather + than by review — the same sweep that should have preceded the change. + """ + from src.config.schema import ToolsConfig + from src.health.startup import check_local_workspace + + result = check_local_workspace(ToolsConfig(local_working_dir="relative/nope")) + assert result.passed is False + assert result.name == "local_workspace" + assert "absolute" in result.detail + assert "install -d -m 0700" in result.recommendation, "must name the fix" + + +def test_startup_diagnostic_passes_for_a_usable_workspace( + tmp_path: Path, workspace: Path +) -> None: + from src.config.schema import ToolsConfig + from src.health.startup import check_local_workspace + + result = check_local_workspace(ToolsConfig(local_working_dir=str(workspace))) + assert result.passed is True + assert str(workspace.resolve()) in result.detail + + +def test_startup_diagnostic_uses_the_shared_protected_roots() -> None: + """A diagnostic applying different rules than the executor would be worse + than no diagnostic: it would pass a workspace the runtime then refuses.""" + from src.health.startup import _workspace_protected_roots + from src.tools.workspace import command_protected_roots + + assert _workspace_protected_roots() == command_protected_roots( + Path(__file__).resolve().parents[1] + ) + + +def test_startup_report_includes_the_workspace_check() -> None: + """Registered in the real runner, not merely defined — an unregistered + check reports nothing, and this is the surface an operator reads when + local commands have stopped working.""" + from src.config.schema import Config + from src.health.startup import run_startup_diagnostics + + report = run_startup_diagnostics(yaml_config=Config(discord={"token": "fake"})) + names = [r.name for r in report.results] + assert "local_workspace" in names, f"check not run; got {names}" + + + +def test_tracked_config_template_documents_the_workspace() -> None: + """CONTRIBUTING is binding: config examples stay aligned with reality. An + undocumented knob is undiscoverable precisely when an operator needs it — + when fail-closed has taken local commands away.""" + repo = Path(__file__).resolve().parents[1] + template = (repo / "config.yml").read_text(encoding="utf-8") + assert "local_working_dir:" in template + + import re as _re + + import yaml + for name in set(_re.findall(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", template)): + os.environ.setdefault(name, "test-value") + parsed = yaml.safe_load(template) + # Against the module CONSTANT, not ToolsConfig() — conftest patches the + # field default to a temp directory for the whole test session. + from src.config.schema import DEFAULT_LOCAL_WORKING_DIR + + assert parsed["tools"]["local_working_dir"] == DEFAULT_LOCAL_WORKING_DIR, ( + "the template must not drift from the schema default" + ) diff --git a/tests/test_startup_diagnostics.py b/tests/test_startup_diagnostics.py index c6543915..eb5bb8af 100644 --- a/tests/test_startup_diagnostics.py +++ b/tests/test_startup_diagnostics.py @@ -618,6 +618,17 @@ def test_model_empty(self): # --------------------------------------------------------------------------- + +def _usable_workspace(): + """A 0700 directory owned by this user, outside the repo and data roots.""" + import tempfile + from pathlib import Path + + path = Path(tempfile.mkdtemp(prefix="odin-diag-ws-")) + path.chmod(0o700) + return path + + class TestRunStartupDiagnostics: def test_no_config(self): report = run_startup_diagnostics() @@ -644,6 +655,9 @@ def test_with_yaml_config(self, tmp_path, monkeypatch): yaml_cfg.openai_codex = MagicMock() yaml_cfg.openai_codex.enabled = False yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {} yaml_cfg.sessions = MagicMock() yaml_cfg.sessions.persist_directory = str(tmp_path / "sessions") @@ -669,6 +683,9 @@ def test_with_both_configs(self, tmp_path, monkeypatch): yaml_cfg.openai_codex = MagicMock() yaml_cfg.openai_codex.enabled = False yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {} yaml_cfg.sessions = MagicMock() yaml_cfg.sessions.persist_directory = "" @@ -705,6 +722,9 @@ def test_check_crash_handled(self, tmp_path, monkeypatch): lambda self: (_ for _ in ()).throw(RuntimeError("boom")) ) yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {} yaml_cfg.sessions = MagicMock() yaml_cfg.sessions.persist_directory = "" @@ -746,7 +766,7 @@ def test_unique_names(self): assert len(names) == len(set(names)), "Duplicate check names" def test_check_count(self): - assert len(_CONFIG_CHECKS) == 7 + assert len(_CONFIG_CHECKS) == 8 # --------------------------------------------------------------------------- @@ -841,6 +861,9 @@ def test_full_production_config(self, tmp_path, monkeypatch): yaml_cfg.openai_codex.credentials_path = str(cred_file) yaml_cfg.openai_codex.model = "gpt-4o" yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {"web1": MagicMock(), "db1": MagicMock()} yaml_cfg.tools.ssh_key_path = str(ssh_key) yaml_cfg.tools.ssh_known_hosts_path = str(known_hosts) @@ -871,6 +894,9 @@ def test_minimal_dev_config(self, tmp_path, monkeypatch): yaml_cfg.openai_codex = MagicMock() yaml_cfg.openai_codex.enabled = False yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {} yaml_cfg.sessions = MagicMock() yaml_cfg.sessions.persist_directory = "" @@ -898,6 +924,9 @@ def test_misconfigured_everything(self, tmp_path, monkeypatch): yaml_cfg.openai_codex.credentials_path = "/nonexistent/creds.json" yaml_cfg.openai_codex.model = "" yaml_cfg.tools = MagicMock() + # A real path: local_workspace is a registered check now, and a + # MagicMock attribute is not a workspace (PR #239). + yaml_cfg.tools.local_working_dir = str(_usable_workspace()) yaml_cfg.tools.hosts = {"web1": MagicMock()} yaml_cfg.tools.ssh_key_path = "/nonexistent/key" yaml_cfg.tools.ssh_known_hosts_path = "/nonexistent/known" From 09e17df2f3985c2c4e15ae4d158ff1a30feade7a Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 19:25:17 -0400 Subject: [PATCH 15/21] =?UTF-8?q?fix:=20PR=20#239=20round=2011=20=E2=80=94?= =?UTF-8?q?=20per-check-type=20workspace,=20lexical=20launch=20parent,=20p?= =?UTF-8?q?er-call=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. validate_action's opt-in was a blunt instrument: the shared exec callback runs EVERY check type, so http/port/service/process/log probes — fixed command shapes, not raw user text — all became workspace-dependent, and an unusable workspace stopped them executing. Round 9's narrowing says an invalid workspace disables raw user-command routes ONLY. The decision now travels per check from run_bundle: use_workspace=(check.type == "command"). Pinned both ways — with a broken workspace, process/port probes still produce real verdicts while the command check fails closed visibly; with a valid one, command checks run in the workspace as before. 2. Round 10's launch-path protection handled a symlinked config FILE but not a symlinked DIRECTORY earlier in the path: the derivation canonicalized the launch parent, collapsing ws/cfg -> real onto the target, so a workspace at ws/ was accepted and a relative `rm -rf cfg` would remove the component restart.reexec() replays while the canonical target survived. The launch-side parent is now kept LEXICAL (absolutized, symlinks intact), and overlap compares every root under BOTH spellings — lexical and canonical — so ordinary roots behave identically while the alias component is held. 3. My call-site classification test OR-ed a function's calls together, so an opted-in function could gain a second unopted call and stay green — the claimed "a new call site fails until classified" was not true for additions to existing functions. It now records EVERY call individually with three categories (constant opt-in / forwarded decision / none), and each classified function must be uniform. validation._exec and executor._run_on_host are CONDITIONAL: they forward, they do not decide. Blast radius enumerated before the change this time: run_bundle's exec contract has exactly one production call site and five strict test fakes, all updated together with the docstring. Mutation-verified: unconditional opt-in, canonicalizing the launch parent, canonical-only overlap, and an unopted call smuggled into an opted-in function each fail their test. Gates local: 7619 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0, git diff --check clean. (CI at the pushed SHA verified before review is requested — round 11 rightly called out a local "coverage 0" claim that CI contradicted at the exact head.) --- src/tools/handlers/validation.py | 18 ++- src/tools/post_validation.py | 19 ++- src/tools/workspace.py | 52 ++++--- tests/test_local_workspace.py | 140 ++++++++++++++++-- tests/test_next_level_pipeline_integration.py | 2 +- tests/test_post_validation.py | 10 +- 6 files changed, 192 insertions(+), 49 deletions(-) diff --git a/src/tools/handlers/validation.py b/src/tools/handlers/validation.py index 6f09499b..ac1fe85a 100644 --- a/src/tools/handlers/validation.py +++ b/src/tools/handlers/validation.py @@ -42,7 +42,12 @@ async def _handle_validate_action(self, inp: dict) -> str: governor = getattr(self, "command_governor", None) async def _exec( - address: str, command: str, ssh_user: str, *, timeout: int + address: str, + command: str, + ssh_user: str, + *, + timeout: int, + use_workspace: bool = False, ) -> tuple[int, str]: # Never mutate shared state here — concurrent checks would race. # _exec_command accepts a per-call timeout, which is honored @@ -61,12 +66,13 @@ async def _exec( return 1, f"validate_action: governor check raised {type(ge).__name__}: {ge}" if not decision.allowed: return 1, f"governor-blocked: {decision.denial_message()}" - # use_workspace=True: command checks execute user-supplied command - # text, which makes this a raw command route like run_command — it - # was still inheriting the install cwd and could replay the - # relative-path wipe (PR #239 round-10 review, reproduced). + # Forwarded per check from run_bundle: True only for type=command + # (user-supplied text, a raw command route like run_command — + # round 10); fixed-shape probes must keep pre-PR cwd semantics so + # an unusable workspace cannot disable service/process/http/port + # validation (round 11). return await self._exec_command( - address, command, ssh_user, timeout=timeout, use_workspace=True + address, command, ssh_user, timeout=timeout, use_workspace=use_workspace ) report = await run_bundle( diff --git a/src/tools/post_validation.py b/src/tools/post_validation.py index e2f0e58a..c0f38b68 100644 --- a/src/tools/post_validation.py +++ b/src/tools/post_validation.py @@ -464,7 +464,14 @@ async def run_bundle( ) -> ValidationReport: """Run a validation bundle. Host resolution: explicit > default > localhost. - exec_command signature: (address, command, ssh_user, timeout=...) -> (exit_code, output) + exec_command signature: + (address, command, ssh_user, timeout=..., use_workspace=...) -> (exit_code, output) + ``use_workspace`` is True ONLY for ``type=command`` checks — those execute + user-supplied command text, exactly the raw route the local workspace + exists for. Fixed-shape probes (http/port/service/process/log) are + generated command strings whose behaviour must not depend on the + workspace: an unusable workspace must not stop a service probe + (PR #239 round-11 review, reproduced). """ start = time.monotonic() if grace_seconds > 0: @@ -526,7 +533,15 @@ async def _run_one(idx: int, check: Check) -> CheckResult: return result try: exit_code, output = await asyncio.wait_for( - exec_command(address, command, ssh_user, timeout=check.timeout_seconds), + exec_command( + address, + command, + ssh_user, + timeout=check.timeout_seconds, + # Raw user command text opts into the workspace; + # fixed-shape probes keep pre-PR cwd semantics. + use_workspace=check.type == "command", + ), timeout=check.timeout_seconds + 5, ) except TimeoutError: diff --git a/src/tools/workspace.py b/src/tools/workspace.py index c15b63b7..0cdc1491 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -72,11 +72,16 @@ def _reject_overlap( ) -> None: """Raise if ``workspace`` overlaps any protected root, in either direction.""" for root in protected_roots or []: - canonical_root = _canonical(root) - if _overlaps(workspace, canonical_root): - raise WorkspaceError( - f"local_working_dir must not overlap {canonical_root}: {workspace}" - ) + # BOTH spellings: lexical-absolute (as given) and canonical (symlinks + # resolved). For ordinary roots they coincide; for the deliberately- + # lexical launch-path root they do not, and resolving it here would + # collapse the alias back onto its target and un-protect the component + # the restart traverses (PR #239 round-11 review). + for candidate in {Path(os.path.abspath(root)), _canonical(root)}: + if _overlaps(workspace, candidate): + raise WorkspaceError( + f"local_working_dir must not overlap {candidate}: {workspace}" + ) # Live-state paths declared by the FULL configuration, with their DECLARED @@ -109,14 +114,21 @@ def _reject_overlap( ) -def _active_config_roots() -> list[tuple[str, bool]]: - """Config file paths the live process depends on, if any. +def _active_config_roots() -> list[str]: + """Directory roots the live config depends on, as FINAL root strings. + + Two spellings, deliberately different in kind: - BOTH the canonical target and the path as given on the command line: the - self-update re-exec replays ``sys.argv``, so an aliased config - (``/etc/odin/config.yml -> /srv/real/odin.yml``) needs its alias directory - protected too — deleting the alias breaks the next restart even though the - target survives (PR #239 round-10 review, reproduced). + - the CANONICAL target's directory (symlinks fully resolved), so the real + file is protected wherever any alias points; + - the LAUNCH path's parent kept LEXICAL — absolutized but with symlinks + NOT resolved — because ``restart.reexec()`` replays ``sys.argv``. With a + symlinked ancestor component (``workspace/cfg -> real/``, launched as + ``workspace/cfg/odin.yml``), canonicalizing the parent collapses it onto + ``real`` and leaves the component the restart actually traverses + unprotected: a workspace-relative ``rm -rf cfg`` breaks the next re-exec + while the canonical target survives (PR #239 round-11 review, + reproduced; round 10 had only handled a symlinked leaf FILE). Imported lazily and guarded: workspace validation must never depend on the config module being importable, and a process that never loaded a config @@ -129,14 +141,14 @@ def _active_config_roots() -> list[tuple[str, bool]]: launch = active_config_launch_path() except Exception: # pragma: no cover - defensive return [] - roots: list[tuple[str, bool]] = [] + roots: list[str] = [] if canonical: - roots.append((str(canonical), True)) + roots.append(str(_canonical(canonical).parent)) if launch: - # As a DIRECTORY: the launch path's own parent, canonicalized. Treating - # it as a file would resolve the symlink and yield the target's - # directory again — the alias directory is the one re-exec reopens. - roots.append((str(Path(launch).parent), False)) + # os.path.abspath normalizes WITHOUT resolving symlinks — that is the + # point. (active_config_launch_path already stores an abspath; applied + # again here defensively, it is idempotent.) + roots.append(str(Path(os.path.abspath(launch)).parent)) return roots @@ -194,7 +206,9 @@ def command_protected_roots( # too, or a bare relative command can delete the file needed to restart — # reproduced with an alternate config whose parent WAS the configured # workspace (PR #239 round-9 review). - declared.extend(_active_config_roots()) + for config_root in _active_config_roots(): + if config_root not in roots: + roots.append(config_root) for configured, is_file in declared: if configured is None: diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 8b48c0ae..87323e5f 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -1877,7 +1877,9 @@ def test_wiring_hardcoded_state_paths_are_all_covered(tmp_path: Path) -> None: ("src/tools/handlers/system.py", "_handle_run_command"): "WORKSPACE", ("src/tools/handlers/system.py", "_handle_run_script"): "WORKSPACE", ("src/tools/handlers/system.py", "_run_one"): "WORKSPACE", # run_command_multi - ("src/tools/handlers/validation.py", "_exec"): "WORKSPACE", + # CONDITIONAL: forwards run_bundle's per-check decision — command + # checks opt in, fixed-shape probes keep process cwd (round 11). + ("src/tools/handlers/validation.py", "_exec"): "CONDITIONAL", ("src/tools/skill_context.py", "run_on_host"): "WORKSPACE", # --- do not: fixed command shapes with caller-supplied absolute paths ---- ("src/tools/handlers/devops.py", "_handle_git_ops"): "documented repo default is the cwd", @@ -1892,13 +1894,19 @@ def test_wiring_hardcoded_state_paths_are_all_covered(tmp_path: Path) -> None: ("src/discord/native_tools/media.py", "_handle_analyze_image"): "base64 of a given path", ("src/audit/diff_tracker.py", "capture_before"): "cat of a governed absolute path", # --- plumbing ----------------------------------------------------------- - ("src/tools/executor.py", "_run_on_host"): "the shared helper itself", + ("src/tools/executor.py", "_run_on_host"): "CONDITIONAL", # forwards its caller's decision ("src/tools/executor.py", "__init__"): "HandlerDeps lambdas forwarding **kwargs", } -def _command_call_sites() -> dict[tuple[str, str], bool]: - """(file, enclosing function) -> whether it passes use_workspace=True.""" +def _command_call_sites() -> dict[tuple[str, str], list[str]]: + """(file, enclosing function) -> category of EVERY call, in order. + + Per call, not per function: an OR over a function's calls let an opted-in + function silently gain a second, unopted call (PR #239 round-11 review). + Categories: "true" (constant opt-in), "dynamic" (forwards a decision made + upstream), "none" (no opt-in). + """ import ast repo = Path(__file__).resolve().parents[1] @@ -1927,14 +1935,18 @@ def _command_call_sites() -> dict[tuple[str, str], bool]: enclosing = walker.name break walker = parents.get(walker) - opts_in = any( - kw.arg == "use_workspace" - and isinstance(kw.value, ast.Constant) - and kw.value.value is True - for kw in node.keywords - ) + category = "none" + for kw in node.keywords: + if kw.arg != "use_workspace": + continue + if isinstance(kw.value, ast.Constant): + category = "true" if kw.value.value is True else "none" + else: + # A forwarded variable/attribute: the decision is made + # upstream (run_bundle's per-check flag). + category = "dynamic" key = (str(path.relative_to(repo)), enclosing) - sites[key] = sites.get(key, False) or opts_in + sites.setdefault(key, []).append(category) return sites @@ -1958,11 +1970,22 @@ def test_every_command_caller_is_classified() -> None: assert not stale, f"no longer call the helpers; drop from the table: {sorted(stale)}" for key, expected in _CLASSIFIED_COMMAND_CALLERS.items(): - should_opt_in = expected == "WORKSPACE" - assert actual[key] is should_opt_in, ( - f"{key[0]}::{key[1]} use_workspace={actual[key]}, expected {should_opt_in} " - f"({expected})" - ) + calls = actual[key] + if expected == "WORKSPACE": + assert calls and all(c == "true" for c in calls), ( + f"{key[0]}::{key[1]}: every call must opt in, got {calls} — a new " + "call in an opted-in function must not ride its neighbours' opt-in" + ) + elif expected == "CONDITIONAL": + assert calls and all(c == "dynamic" for c in calls), ( + f"{key[0]}::{key[1]}: every call must forward the upstream " + f"decision, got {calls}" + ) + else: + assert all(c == "none" for c in calls), ( + f"{key[0]}::{key[1]} is classified as NOT a command route " + f"({expected}) but a call passes use_workspace: {calls}" + ) async def test_validate_action_command_check_runs_in_the_workspace( @@ -2260,3 +2283,88 @@ def test_tracked_config_template_documents_the_workspace() -> None: assert parsed["tools"]["local_working_dir"] == DEFAULT_LOCAL_WORKING_DIR, ( "the template must not drift from the schema default" ) + + +# --- Round 11: per-check-type workspace, and symlinked launch ancestors ------ + + +async def test_fixed_shape_validation_checks_survive_an_unusable_workspace( + fake_install: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Round-11 blocker 1, reproduced by Odin: opting the WHOLE validate_action + exec callback into the workspace made every check type depend on it — + an unusable workspace stopped service/process/http/port probes that are + fixed command shapes, not raw user commands. The narrowed scope is: + an invalid workspace disables raw user-command routes ONLY. + """ + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(tmp_path / "no-parent" / "ws", fake_install) + + result = await executor.execute("validate_action", { + "host": "localhost", + "checks": [ + # Fixed-shape probes: must EXECUTE despite the broken workspace. + {"type": "process", "target": "init", "severity": "warn"}, + {"type": "port", "target": "1", "severity": "warn"}, + # Raw user command text: must fail closed, visibly. + {"type": "command", "target": "touch should-not-run"}, + ], + }) + report = str(result.output) + assert "local_working_dir" in report, "the command check must fail closed, visibly" + # The probes produced real pass/fail verdicts rather than workspace errors: + # exactly one check errored (the command one). + assert report.count("local_working_dir") == 1, report + assert not (fake_install / "should-not-run").exists() + + +async def test_command_checks_still_run_in_the_workspace_when_it_is_valid( + fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The other side of the round-11 narrowing: conditional must not mean + never — type=command checks keep the round-10 behaviour.""" + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(workspace, fake_install) + await executor.execute("validate_action", { + "host": "localhost", + "checks": [{"type": "command", "target": "touch from-conditional"}], + }) + assert (workspace / "from-conditional").exists() + assert not (fake_install / "from-conditional").exists() + + +def test_symlinked_ancestor_of_the_launch_config_is_protected(tmp_path: Path) -> None: + """Round-11 blocker 2, reproduced by Odin. + + Round 10 protected a symlinked config FILE; a symlinked DIRECTORY earlier + in the launch path (ws/cfg -> real/) still canonicalized onto the target, + so a workspace at ws/ was accepted and a relative `rm -rf cfg` would + remove the path restart.reexec() replays while the canonical target + survived. The launch-side parent is now kept lexical. + """ + from src.config.schema import active_config_path, set_active_config_path + from src.tools.workspace import command_protected_roots + + real = tmp_path / "real" + real.mkdir() + (real / "odin.yml").write_text("discord:\n token: fake\n", encoding="utf-8") + ws = tmp_path / "ws" + ws.mkdir(mode=0o700) + (ws / "cfg").symlink_to(real, target_is_directory=True) + + previous = active_config_path() + set_active_config_path(ws / "cfg" / "odin.yml") + try: + roots = command_protected_roots(tmp_path / "install") + assert str(real.resolve()) in roots, "canonical target still protected" + assert str(ws / "cfg") in roots, "the LEXICAL launch parent must survive" + + # The workspace containing the alias component is rejected... + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(ws), protected_roots=roots) + # ...and so is a workspace that IS the alias target reached lexically. + with pytest.raises(WorkspaceError, match="overlap"): + provision_workspace(str(ws / "cfg" / "sub"), protected_roots=roots) + assert not (real / "sub").exists() + finally: + set_active_config_path(previous) diff --git a/tests/test_next_level_pipeline_integration.py b/tests/test_next_level_pipeline_integration.py index 6b6d5cc1..6493d3ba 100644 --- a/tests/test_next_level_pipeline_integration.py +++ b/tests/test_next_level_pipeline_integration.py @@ -15,7 +15,7 @@ class TestValidateActionEndToEnd: @pytest.mark.asyncio async def test_full_pipeline_mixed_severity(self): - async def fake_exec(addr, cmd, user, *, timeout): + async def fake_exec(addr, cmd, user, *, timeout, use_workspace=False): if "curl" in cmd: return (0, "200") if "systemctl is-active" in cmd: diff --git a/tests/test_post_validation.py b/tests/test_post_validation.py index 66fe3061..a5fc4fd1 100644 --- a/tests/test_post_validation.py +++ b/tests/test_post_validation.py @@ -357,7 +357,7 @@ def test_all_errored_is_error(self): class TestRunBundleIntegration: @pytest.mark.asyncio async def test_full_bundle_mixed_results(self): - async def fake_exec(addr, cmd, user, *, timeout): + async def fake_exec(addr, cmd, user, *, timeout, use_workspace=False): if "curl" in cmd: return (0, "200") if "dev/tcp" in cmd: @@ -437,7 +437,7 @@ async def slow_exec(*a, **kw): async def test_host_resolution_order(self): seen_hosts: list[str] = [] - async def fake_exec(addr, cmd, user, *, timeout): + async def fake_exec(addr, cmd, user, *, timeout, use_workspace=False): seen_hosts.append(addr) return (0, "active") @@ -480,7 +480,7 @@ async def test_concurrent_checks_respect_independent_timeouts(self): """Round 2 review — no shared-state timeout race across concurrent checks.""" observed: list[tuple[int, float]] = [] - async def timed_exec(addr, cmd, user, *, timeout): + async def timed_exec(addr, cmd, user, *, timeout, use_workspace=False): t0 = asyncio.get_event_loop().time() # Fast/slow checks finish at different times; neither should see # the other's timeout bleed in. @@ -529,7 +529,7 @@ async def test_max_parallel_bounds_concurrency(self): max_in_flight = 0 lock = asyncio.Lock() - async def tracking_exec(addr, cmd, user, *, timeout): + async def tracking_exec(addr, cmd, user, *, timeout, use_workspace=False): nonlocal in_flight, max_in_flight async with lock: in_flight += 1 @@ -609,7 +609,7 @@ def check(self, command): gov = _FailingGovernor() - async def wrapped(addr, cmd, user, *, timeout): + async def wrapped(addr, cmd, user, *, timeout, use_workspace=False): try: gov.check(cmd) except Exception as ge: From 04f85a206ef1a745fbd9bac12bbb206b20fc96eb Mon Sep 17 00:00:00 2001 From: Odin Date: Mon, 27 Jul 2026 20:06:47 -0400 Subject: [PATCH 16/21] fix: close remaining workspace escape and root-alias gaps --- src/__main__.py | 2 +- src/health/startup.py | 30 ++++-- src/tools/executor.py | 15 +-- src/tools/post_validation.py | 12 ++- src/tools/workspace.py | 57 +++++++---- src/web/api/self_update.py | 2 +- tests/test_local_workspace.py | 184 ++++++++++++++++++++++++++++++++++ tests/test_post_validation.py | 20 +++- 8 files changed, 282 insertions(+), 40 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index cbb9b07d..c2b606da 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -326,7 +326,7 @@ def _command_protected_roots(config) -> list[str]: """ from src.tools.workspace import command_protected_roots - return command_protected_roots(Path(__file__).resolve().parents[1], config) + return command_protected_roots(Path(__file__).absolute().parents[1], config) # The entrypoint guard MUST stay the last statement in this module. Python diff --git a/src/health/startup.py b/src/health/startup.py index 9480e0d5..a1a9de4c 100644 --- a/src/health/startup.py +++ b/src/health/startup.py @@ -426,8 +426,8 @@ def check_config_sections(config: Any) -> DiagnosticResult: ) -def check_local_workspace(tools_config: Any) -> DiagnosticResult: - """Verify the local command workspace is usable. +def check_local_workspace(config: Any) -> DiagnosticResult: + """Verify the local command workspace against the full live config. Local user commands FAIL CLOSED when this directory is missing, wrongly owned, or not 0700 — deliberately, because falling back to the install @@ -437,9 +437,19 @@ def check_local_workspace(tools_config: Any) -> DiagnosticResult: """ from ..tools.workspace import WorkspaceError, provisioning_hint, resolve_workspace + # Production passes the full Config so independently relocated sessions, + # context, logs, credentials, and other state are protected exactly as the + # startup migration and executor protect them. Accept ToolsConfig directly + # only for focused callers/tests; that intentionally yields the reduced + # fallback contract. + tools_config = getattr(config, "tools", config) + full_config = config if tools_config is not config else None configured = getattr(tools_config, "local_working_dir", "") or "" try: - workspace = resolve_workspace(configured, protected_roots=_workspace_protected_roots()) + workspace = resolve_workspace( + configured, + protected_roots=_workspace_protected_roots(full_config), + ) except WorkspaceError as exc: return DiagnosticResult( name="local_workspace", @@ -464,14 +474,12 @@ def check_local_workspace(tools_config: Any) -> DiagnosticResult: ) -def _workspace_protected_roots() -> list[str]: - """Protected roots for the diagnostic, from the same shared derivation the - executor uses — a check that applied different rules would be worse than - no check at all.""" - from ..config.schema import Config # noqa: F401 - imported for typing clarity +def _workspace_protected_roots(config: object = None) -> list[str]: + """Protected roots for the diagnostic, from the same full live config and + shared derivation used by startup migration and executor.""" from ..tools.workspace import command_protected_roots - return command_protected_roots(Path(__file__).resolve().parents[2]) + return command_protected_roots(Path(__file__).absolute().parents[2], config) def check_data_directories() -> DiagnosticResult: @@ -556,7 +564,9 @@ def check_codex_model(codex_config: Any) -> DiagnosticResult: ("ssh_hosts", check_ssh_hosts, "tools"), ("sessions_directory", check_sessions_directory, "sessions"), ("knowledge_db", check_knowledge_db, "search"), - ("local_workspace", check_local_workspace, "tools"), + # Full Config, not only ToolsConfig: every relocatable live-state path + # must be judged by the same contract as runtime command execution. + ("local_workspace", check_local_workspace, None), ("config_consistency", check_config_sections, None), # uses full Config ] diff --git a/src/tools/executor.py b/src/tools/executor.py index 49691524..2c172b22 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -352,7 +352,7 @@ def _protected_roots(self) -> list[str]: """ return command_protected_roots( # Install root: the package's own location (…/src/tools/executor.py). - Path(__file__).resolve().parents[2], + Path(__file__).absolute().parents[2], # getattr-guarded throughout: the sanctioned __new__ patch seam # builds executors without __init__, so these may not exist. getattr(self, "_app_config", None), @@ -435,13 +435,16 @@ def _walk() -> None: except OSError: pass except OSError: - return - finally: with lock: self._workspace_usage_refreshing = False - # Stamped on COMPLETION, so a long walk does not read as stale the - # moment it finishes. - self._workspace_usage_cache = (time.monotonic(), total_bytes, files) + return + # Publish the completed cache and clear the single-flight flag as + # one locked transition. Clearing first leaves a race where a scrape + # can launch a duplicate walk before the fresh cache is visible. + completed_at = time.monotonic() + with lock: + self._workspace_usage_cache = (completed_at, total_bytes, files) + self._workspace_usage_refreshing = False try: threading.Thread( diff --git a/src/tools/post_validation.py b/src/tools/post_validation.py index c0f38b68..43643792 100644 --- a/src/tools/post_validation.py +++ b/src/tools/post_validation.py @@ -283,11 +283,17 @@ def _build_command(check: Check) -> str | None: h, p = tgt.rsplit(":", 1) else: h, p = "127.0.0.1", tgt - if not p.isdigit(): + if not h or not p.isdigit(): return None + # Do not interpolate a quoted host inside another quoted shell program. + # ``shlex.quote(h)`` embedded in ``bash -c '...{h}...'`` lets the outer + # shell evaluate substitutions in crafted hosts before Bash starts. + # Positional arguments keep the inner program constant and quote each + # user value exactly once for the outer shell. + script = 'cat < /dev/null > "/dev/tcp/$1/$2"' return ( - f"timeout {timeout} bash -c 'cat < /dev/null > /dev/tcp/{shlex.quote(h)}/{p}'" - " && echo OPEN || echo CLOSED" + f"timeout {timeout} bash -c {shlex.quote(script)} _ " + f"{shlex.quote(h)} {shlex.quote(p)} && echo OPEN || echo CLOSED" ) if t == "service": return f"systemctl is-active {shlex.quote(tgt)} 2>/dev/null || true" diff --git a/src/tools/workspace.py b/src/tools/workspace.py index 0cdc1491..7fd493e7 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -62,6 +62,24 @@ def _canonical(path: str | os.PathLike[str]) -> Path: return Path(path).expanduser().resolve(strict=False) +def _lexical_absolute(path: str | os.PathLike[str]) -> Path: + """Absolute spelling with ``..`` removed but symlink components intact. + + Canonical paths protect the object reached through an alias. Lexical paths + protect the alias components a running process will traverse again. Both + matter: deleting an alias can strand sessions, credentials, or the checkout + used for an in-place restart even when the canonical target survives. + """ + return Path(os.path.abspath(Path(path).expanduser())) + + +def _path_spellings(path: str | os.PathLike[str]) -> tuple[Path, ...]: + """Unique lexical and canonical spellings, in stable order.""" + lexical = _lexical_absolute(path) + canonical = _canonical(path) + return (lexical,) if lexical == canonical else (lexical, canonical) + + def _overlaps(workspace: Path, root: Path) -> bool: """True when the two paths are the same or either contains the other.""" return workspace == root or root in workspace.parents or workspace in root.parents @@ -77,7 +95,7 @@ def _reject_overlap( # lexical launch-path root they do not, and resolving it here would # collapse the alias back onto its target and un-protect the component # the restart traverses (PR #239 round-11 review). - for candidate in {Path(os.path.abspath(root)), _canonical(root)}: + for candidate in _path_spellings(root): if _overlaps(workspace, candidate): raise WorkspaceError( f"local_working_dir must not overlap {candidate}: {workspace}" @@ -185,16 +203,30 @@ def command_protected_roots( Paths are classified by DECLARED semantics, never guessed from the name: a ``Path.suffix`` heuristic misreads dotted directories and extensionless - files. Each declared path is resolved COMPLETELY before ``.parent`` is - taken, because taking the parent first protects the alias directory rather - than the target (``/aliases/memory.json -> /live-data/memory.json`` would - protect ``/aliases`` and accept ``/live-data/workspace``). + files. Each declared path is kept under both its lexical and canonical + spelling before file parents are taken: the lexical spelling protects the + alias a running subsystem traverses, while the canonical spelling protects + its target (``/aliases/memory.json -> /live-data/memory.json`` needs both). """ source: object = config if source is None: source = SimpleNamespace(tools=tools) if tools is not None else SimpleNamespace() - roots = [str(_canonical(install_root))] + roots: list[str] = [] + + def _append_path(path: object, *, is_file: bool) -> None: + """Protect both how a path is named and what that name reaches.""" + if path is None: + return + text = str(path).strip() + if not text: + return + for spelling in _path_spellings(text): + root = str(spelling.parent if is_file else spelling) + if root not in roots: + roots.append(root) + + _append_path(install_root, is_file=False) declared: list[tuple[object, bool]] = [ (_dotted(source, dotted), is_file) for dotted, is_file in _DECLARED_STATE_PATHS ] @@ -207,19 +239,10 @@ def command_protected_roots( # reproduced with an alternate config whose parent WAS the configured # workspace (PR #239 round-9 review). for config_root in _active_config_roots(): - if config_root not in roots: - roots.append(config_root) + _append_path(config_root, is_file=False) for configured, is_file in declared: - if configured is None: - continue - text = str(configured).strip() - if not text: - continue - resolved = _canonical(text) - root = str(resolved.parent if is_file else resolved) - if root not in roots: - roots.append(root) + _append_path(configured, is_file=is_file) return roots diff --git a/src/web/api/self_update.py b/src/web/api/self_update.py index ac1f3d7e..795b78a7 100644 --- a/src/web/api/self_update.py +++ b/src/web/api/self_update.py @@ -328,7 +328,7 @@ def _live_protected_roots(bot, base: str | None) -> list[str]: memory_path = getattr(getattr(bot, "tool_executor", None), "_memory_path", None) return command_protected_roots( - Path(base).resolve() if base else Path(__file__).resolve().parents[3], + Path(base).absolute() if base else Path(__file__).absolute().parents[3], getattr(bot, "config", None), memory_path=memory_path or DEFAULT_MEMORY_PATH, ) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 87323e5f..4cf9214e 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -526,6 +526,119 @@ def test_rejects_workspace_that_contains_a_protected_root(tmp_path: Path) -> Non resolve_workspace(str(parent), protected_roots=[str(install)]) +def test_declared_state_path_protects_lexical_alias_and_canonical_target( + tmp_path: Path, +) -> None: + """A live-state path may contain a symlinked directory component. Protect + both the alias traversed by runtime code and the canonical target it reaches. + """ + from src.config.schema import Config + from src.tools.workspace import command_protected_roots + + install = tmp_path / "install" + install.mkdir() + workspace = tmp_path / "workspace" + workspace.mkdir(mode=0o700) + live = tmp_path / "live" + (live / "sessions").mkdir(parents=True) + (workspace / "state").symlink_to(live, target_is_directory=True) + config = Config( + discord={"token": "fake"}, + sessions={"persist_directory": str(workspace / "state" / "sessions")}, + ) + + roots = command_protected_roots( + install, + config, + memory_path=tmp_path / "memory" / "memory.json", + ) + assert str(workspace / "state" / "sessions") in roots + assert str((live / "sessions").resolve()) in roots + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(workspace), protected_roots=roots) + + +def test_install_root_protects_lexical_alias_and_canonical_checkout(tmp_path: Path) -> None: + """In-place re-exec needs the checkout's launch alias as well as its target.""" + from src.tools.workspace import command_protected_roots + + workspace = tmp_path / "workspace" + workspace.mkdir(mode=0o700) + real_install = tmp_path / "real-install" + real_install.mkdir() + (workspace / "checkout").symlink_to(real_install, target_is_directory=True) + + roots = command_protected_roots( + workspace / "checkout", + memory_path=tmp_path / "memory" / "memory.json", + ) + assert str(workspace / "checkout") in roots + assert str(real_install.resolve()) in roots + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(workspace), protected_roots=roots) + + +def _symlinked_checkout_layout(tmp_path: Path) -> tuple[Path, Path]: + workspace = tmp_path / "workspace" + workspace.mkdir(mode=0o700) + real_install = tmp_path / "real-install" + real_install.mkdir() + checkout = workspace / "checkout" + checkout.symlink_to(real_install, target_is_directory=True) + return workspace, checkout + + +def test_executor_passes_lexical_install_root_to_shared_derivation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import src.tools.executor as executor_module + + workspace, checkout = _symlinked_checkout_layout(tmp_path) + monkeypatch.setattr(executor_module, "__file__", str(checkout / "src/tools/executor.py")) + executor = ToolExecutor( + ToolsConfig(local_working_dir=str(workspace)), + memory_path=str(tmp_path / "memory/memory.json"), + ) + with pytest.raises(WorkspaceError, match="overlap"): + executor._ensure_local_workspace() + + +def test_startup_passes_lexical_install_root_to_shared_derivation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import src.__main__ as entrypoint + from src.config.schema import Config + + workspace, checkout = _symlinked_checkout_layout(tmp_path) + monkeypatch.setattr(entrypoint, "__file__", str(checkout / "src/__main__.py")) + config = Config( + discord={"token": "fake"}, + tools={"local_working_dir": str(workspace)}, + ) + roots = entrypoint._command_protected_roots(config) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(workspace), protected_roots=roots) + + +def test_updater_passes_lexical_install_root_to_shared_derivation(tmp_path: Path) -> None: + from src.config.schema import Config + from src.web.api.self_update import _live_protected_roots + + workspace, checkout = _symlinked_checkout_layout(tmp_path) + bot = SimpleNamespace( + config=Config( + discord={"token": "fake"}, + tools={"local_working_dir": str(workspace)}, + ), + tool_executor=SimpleNamespace( + _memory_path=tmp_path / "memory/memory.json", + ), + ) + roots = _live_protected_roots(bot, str(checkout)) + with pytest.raises(WorkspaceError, match="overlap"): + resolve_workspace(str(workspace), protected_roots=roots) + + def test_rejects_wrong_owner(tmp_path: Path, fake_install: Path) -> None: """The contract is a directory owned by the execution identity.""" ws = tmp_path / "otherowner" @@ -2151,6 +2264,41 @@ def _record(**kw): assert not started, "a refresh was already running; a second must not start" +def test_usage_cache_is_published_before_single_flight_clears( + fake_install: Path, workspace: Path +) -> None: + """No scrape may observe refreshing=False while the completed cache is + still absent/stale, or it can launch a duplicate usage walk.""" + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "one").write_text("x", encoding="utf-8") + inner = threading.Lock() + invalid_transitions: list[str] = [] + + class _ObservingLock: + def __enter__(self): + inner.acquire() + return self + + def __exit__(self, *_exc): + if ( + executor._workspace_usage_refreshing is False + and executor._workspace_usage_cache is None + ): + invalid_transitions.append("refresh cleared before cache publish") + inner.release() + + executor._workspace_usage_lock = _ObservingLock() + executor.get_workspace_metrics() + + deadline = time.monotonic() + 5 + while executor._workspace_usage_refreshing and time.monotonic() < deadline: + time.sleep(0.01) + + assert invalid_transitions == [] + assert executor._workspace_usage_refreshing is False + assert executor._workspace_usage_cache is not None + + class _NoThread: def start(self) -> None: # pragma: no cover - never reached when single-flight holds raise AssertionError("thread should not have been started") @@ -2238,6 +2386,26 @@ def test_startup_diagnostic_passes_for_a_usable_workspace( assert str(workspace.resolve()) in result.detail +def test_startup_diagnostic_uses_full_config_for_relocated_state(tmp_path: Path) -> None: + """The real diagnostic registry must not say ready when runtime rejects the + same workspace for overlapping an independently relocated state directory. + """ + from src.config.schema import Config + from src.health.startup import run_startup_diagnostics + + workspace = tmp_path / "state" + workspace.mkdir(mode=0o700) + config = Config( + discord={"token": "fake"}, + tools={"local_working_dir": str(workspace)}, + sessions={"persist_directory": str(workspace)}, + ) + report = run_startup_diagnostics(yaml_config=config) + result = next(item for item in report.results if item.name == "local_workspace") + assert result.passed is False + assert "overlap" in result.detail + + def test_startup_diagnostic_uses_the_shared_protected_roots() -> None: """A diagnostic applying different rules than the executor would be worse than no diagnostic: it would pass a workspace the runtime then refuses.""" @@ -2333,6 +2501,22 @@ async def test_command_checks_still_run_in_the_workspace_when_it_is_valid( assert not (fake_install / "from-conditional").exists() +async def test_port_probe_target_cannot_execute_in_the_inherited_cwd( + fake_install: Path, workspace: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Port is a fixed-shape probe only when its host cannot escape the nested + shell. Before the fix this target created the marker in the install cwd. + """ + monkeypatch.chdir(fake_install) + executor = _executor_with_workspace(workspace, fake_install) + await executor.execute("validate_action", { + "host": "localhost", + "checks": [{"type": "port", "target": "$(touch injected-by-port):1"}], + }) + assert not (fake_install / "injected-by-port").exists() + assert not (workspace / "injected-by-port").exists() + + def test_symlinked_ancestor_of_the_launch_config_is_protected(tmp_path: Path) -> None: """Round-11 blocker 2, reproduced by Odin. diff --git a/tests/test_post_validation.py b/tests/test_post_validation.py index a5fc4fd1..cb5401af 100644 --- a/tests/test_post_validation.py +++ b/tests/test_post_validation.py @@ -3,6 +3,8 @@ import asyncio import json +import subprocess +from pathlib import Path import pytest @@ -118,14 +120,17 @@ def test_http(self): def test_port_with_host(self): cmd = _build_command(Check(type="port", target="1.2.3.4:8080")) - assert "/dev/tcp/1.2.3.4/8080" in cmd + assert '"/dev/tcp/$1/$2"' in cmd + assert cmd.endswith("_ 1.2.3.4 8080 && echo OPEN || echo CLOSED") def test_port_bare(self): cmd = _build_command(Check(type="port", target="5432")) - assert "/dev/tcp/127.0.0.1/5432" in cmd + assert '"/dev/tcp/$1/$2"' in cmd + assert cmd.endswith("_ 127.0.0.1 5432 && echo OPEN || echo CLOSED") def test_port_invalid(self): assert _build_command(Check(type="port", target="notaport")) is None + assert _build_command(Check(type="port", target=":80")) is None def test_service(self): cmd = _build_command(Check(type="service", target="nginx")) @@ -144,6 +149,17 @@ def test_log_absent_bare(self): def test_command_passthrough(self): assert _build_command(Check(type="command", target="echo hi")) == "echo hi" + def test_port_host_cannot_escape_the_constant_inner_script(self, tmp_path: Path): + """A port probe is fixed-shape only if its target cannot become shell + syntax. This payload executed in Odin's inherited cwd before the fix. + """ + marker = tmp_path / "port-target-executed" + payload = f"$(touch {marker}):1" + cmd = _build_command(Check(type="port", target=payload, timeout_seconds=1)) + assert cmd is not None + subprocess.run(cmd, shell=True, cwd=tmp_path, capture_output=True, timeout=5) + assert not marker.exists() + class TestEvaluate: def test_http_pass_default_status(self): From ee0d68d11a6760289563f2006d8fd5221fb85cf0 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 20:18:54 -0400 Subject: [PATCH 17/21] =?UTF-8?q?fix:=20cross-review=20of=2004f85a2=20?= =?UTF-8?q?=E2=80=94=20restore=20the=20finally=20guarantee=20on=20the=20us?= =?UTF-8?q?age=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Odin implemented his own round-12 findings (role-flip). Cross-reviewed his commit to full bar: read the whole diff, independently reproduced the port-probe injection before and after his fix, ran all four gates myself, and mutation-tested his three pins (reverting the port-probe fix, the dual lexical/canonical spellings, or the full-Config diagnostic each fails his tests). His work is correct and the port-probe finding is significant — a target like `$(touch marker):1` escaped the nested quoting and executed arbitrary commands. That vulnerability is PRE-EXISTING on master, not introduced by this PR. One regression found in his metrics fix. Publishing the cache and clearing the single-flight flag as one locked transition is right — clearing first lets a scrape start a duplicate walk before the fresh cache is visible. But it also removed the `finally`, so any exception that is not OSError left the flag set forever and every later scrape declined to refresh: usage metrics frozen permanently with no recovery. Reproduced with a non-OSError from os.walk — flag stuck True, no recovery on subsequent scrapes. Both properties are required and both now hold: the success path still publishes cache and flag atomically under the lock, and a `finally` guarded by a `published` sentinel guarantees the flag never survives the thread on any exception without splitting the atomic publish. Pinned by two tests — recovery after an unexpected exception, and cache-present whenever the flag is clear. Gates run independently, not taken from the report: 7630 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0, diff --check clean. --- src/tools/executor.py | 29 ++++++++++++++------- tests/test_local_workspace.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/tools/executor.py b/src/tools/executor.py index 2c172b22..3341abe2 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -426,6 +426,7 @@ def _refresh_workspace_usage(self, root: Path) -> None: def _walk() -> None: total_bytes = 0.0 files = 0.0 + published = False try: for dirpath, _dirnames, filenames in os.walk(root, followlinks=False): for name in filenames: @@ -434,17 +435,27 @@ def _walk() -> None: total_bytes += os.lstat(os.path.join(dirpath, name)).st_size except OSError: pass - except OSError: + # Publish the completed cache and clear the single-flight flag + # as ONE locked transition. Clearing first leaves a race where + # a scrape launches a duplicate walk before the fresh cache is + # visible (Odin, PR #239 round-12). + completed_at = time.monotonic() with lock: + self._workspace_usage_cache = (completed_at, total_bytes, files) self._workspace_usage_refreshing = False - return - # Publish the completed cache and clear the single-flight flag as - # one locked transition. Clearing first leaves a race where a scrape - # can launch a duplicate walk before the fresh cache is visible. - completed_at = time.monotonic() - with lock: - self._workspace_usage_cache = (completed_at, total_bytes, files) - self._workspace_usage_refreshing = False + published = True + except OSError: + pass + finally: + # The flag must never survive this thread. Clearing it only on + # the OSError and success paths wedged usage metrics FOREVER on + # any other exception — the flag stayed set, so every later + # scrape declined to start a refresh (cross-review of round 12, + # reproduced). Guarded by `published` so the success path's + # atomic publish is not split apart. + if not published: + with lock: + self._workspace_usage_refreshing = False try: threading.Thread( diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 4cf9214e..1de609aa 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2552,3 +2552,50 @@ def test_symlinked_ancestor_of_the_launch_config_is_protected(tmp_path: Path) -> assert not (real / "sub").exists() finally: set_active_config_path(previous) + + +def test_usage_refresh_flag_never_survives_the_thread( + fake_install: Path, workspace: Path +) -> None: + """Cross-review of Odin's round-12 single-flight fix. + + Publishing the cache and clearing the flag as one locked transition is + correct — clearing first lets a scrape launch a duplicate walk before the + fresh cache is visible. But dropping the `finally` alongside it meant any + exception that is NOT OSError left the flag set forever, and every later + scrape then declined to refresh: usage metrics frozen permanently, with no + recovery. Both properties are required, so both are pinned. + """ + executor = _executor_with_workspace(workspace, fake_install) + + def _unexpected(*_a, **_kw): + raise RuntimeError("not an OSError") + + with patch("src.tools.executor.os.walk", _unexpected): + executor.get_workspace_metrics() + deadline = time.monotonic() + 5 + while executor._workspace_usage_refreshing and time.monotonic() < deadline: + time.sleep(0.02) + + assert executor._workspace_usage_refreshing is False, ( + "the single-flight flag must not survive the thread on ANY exception" + ) + + # ...and metrics recover on the next scrape rather than staying frozen. + (workspace / "f").write_text("xyz", encoding="utf-8") + assert _metrics_after_refresh(executor)["files"] == 1 + + +def test_usage_cache_and_flag_are_published_atomically( + fake_install: Path, workspace: Path +) -> None: + """Odin's finding: a scrape landing between 'flag cleared' and 'cache + published' starts a redundant walk. The fresh cache must be visible to any + observer that sees the flag cleared.""" + executor = _executor_with_workspace(workspace, fake_install) + (workspace / "one").write_text("x" * 10, encoding="utf-8") + + _metrics_after_refresh(executor) + # Whenever the flag is clear, a cache is present — never the gap. + assert executor._workspace_usage_refreshing is False + assert executor._workspace_usage_cache is not None From a0d47707d35d0c54fa0a9b06282f5ed7281999df Mon Sep 17 00:00:00 2001 From: Odin Date: Mon, 27 Jul 2026 21:13:56 -0400 Subject: [PATCH 18/21] fix: close final workspace review gaps --- src/__main__.py | 10 +- src/tools/executor.py | 17 ++- src/tools/handlers/files_docs.py | 17 ++- src/tools/workspace.py | 55 ++++++++++ tests/test_local_workspace.py | 177 +++++++++++++++++++++++++++++-- 5 files changed, 260 insertions(+), 16 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index c2b606da..219f1ac8 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -129,10 +129,14 @@ def main() -> None: # Odin from starting and answering on Discord. Local commands then fail # closed individually, with the same actionable error. try: - from src.tools.workspace import WorkspaceError, provision_workspace, provisioning_hint + from src.tools.workspace import ( + WorkspaceError, + provision_startup_workspace, + provisioning_hint, + ) - workspace = provision_workspace( - config.tools.local_working_dir, + workspace = provision_startup_workspace( + config.tools, protected_roots=_command_protected_roots(config), ) log.info("Local command workspace ready: %s", workspace) diff --git a/src/tools/executor.py b/src/tools/executor.py index 3341abe2..a8da2ac2 100644 --- a/src/tools/executor.py +++ b/src/tools/executor.py @@ -419,6 +419,15 @@ def _refresh_workspace_usage(self, root: Path) -> None: if lock is None: # __new__ patch seam return with lock: + # Re-check freshness after acquiring the lock. Another scrape may + # have completed and published while this caller was waiting; the + # unlocked fast-path observation above is not authoritative once + # lock acquisition blocks. Without this second check, concurrent + # scrapes can launch an immediate duplicate walk after a fast first + # refresh finishes. + cached = getattr(self, "_workspace_usage_cache", None) + if cached is not None and (time.monotonic() - cached[0]) < WORKSPACE_METRICS_TTL: + return if self._workspace_usage_refreshing: return self._workspace_usage_refreshing = True @@ -444,7 +453,13 @@ def _walk() -> None: self._workspace_usage_cache = (completed_at, total_bytes, files) self._workspace_usage_refreshing = False published = True - except OSError: + except Exception: + # A metrics walk is best-effort. Letting an unexpected error + # escape a daemon thread still invokes threading.excepthook + # (and produces an unhandled-thread warning in pytest/logging), + # contradicting this collector's never-raises contract. The + # finally block below clears the single-flight flag so the next + # scrape can recover. pass finally: # The flag must never survive this thread. Clearing it only on diff --git a/src/tools/handlers/files_docs.py b/src/tools/handlers/files_docs.py index eb75d1e6..29cc435d 100644 --- a/src/tools/handlers/files_docs.py +++ b/src/tools/handlers/files_docs.py @@ -10,6 +10,7 @@ from __future__ import annotations import base64 +import posixpath import shlex from .deps import HandlerBase @@ -58,7 +59,16 @@ async def _handle_write_file(self, inp: dict) -> str: "A relative path would resolve against Odin's working directory " "rather than where you intend." ) + path = str(path) safe_path = shlex.quote(path) + # Compute and quote the parent as its own shell argument. Embedding a + # quoted path in ``$(dirname ...)`` and then leaving the substitution + # unquoted word-splits a parent containing spaces; ``mkdir`` can create + # those extra relative words in Odin's inherited install cwd even though + # the requested file path itself is absolute (PR #239 final review, + # reproduced). Remote managed hosts are POSIX, matching the absolute-path + # contract above. + safe_parent = shlex.quote(posixpath.dirname(path) or "/") # Govern the write before executing — write_file reaches the filesystem # via _run_on_host, which does NOT itself govern. Check a representative # redirect-to-path command so policy (e.g. writes to sensitive targets) @@ -67,8 +77,11 @@ async def _handle_write_file(self, inp: dict) -> str: if not allowed: return denial # Base64-encode content to avoid shell injection via heredoc delimiter - encoded = base64.b64encode(content.encode()).decode() - cmd = f"mkdir -p $(dirname {safe_path}) && echo '{encoded}' | base64 -d > {safe_path}" + encoded = shlex.quote(base64.b64encode(content.encode()).decode()) + cmd = ( + f"mkdir -p -- {safe_parent} && " + f"printf %s {encoded} | base64 -d > {safe_path}" + ) return await self._run_on_host(host, cmd) @staticmethod diff --git a/src/tools/workspace.py b/src/tools/workspace.py index 7fd493e7..76697830 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -406,6 +406,61 @@ def provision_workspace( ) +def provision_startup_workspace( + tools_config: object, + *, + protected_roots: Sequence[str | os.PathLike[str]] | None = None, + owner_uid: int | None = None, +) -> Path: + """Provision the workspace used by the incoming process at startup. + + Existing source/git installs have config files from before + ``local_working_dir`` existed. On their first in-place update the old + updater cannot run the new preflight, and an unprivileged developer account + commonly cannot create the schema default under ``/var/lib`` or use + ``sudo -n``. Failing closed there would preserve safety by silently losing + all local-command capability. + + Explicit configuration is authoritative and never substituted. Only a + genuinely absent legacy field may fall back, after the normal default + cannot be provisioned, to a stable private directory in the service user's + home. The selected value is written into the live ``ToolsConfig`` object so + startup, the executor, diagnostics, and the next self-update all use the + identical path for this process. A later PUT /api/config persists it through + the normal validated config path; otherwise the deterministic migration is + repeated on future starts. + """ + configured = str(getattr(tools_config, "local_working_dir", "") or "") + try: + return provision_workspace( + configured, + protected_roots=protected_roots, + owner_uid=owner_uid, + ) + except WorkspaceError as default_error: + fields_set: set[str] = set(getattr(tools_config, "model_fields_set", set())) + if "local_working_dir" in fields_set: + raise + + # A direct child of HOME needs no recursive parent creation, remains + # stable across commands/restarts, and is outside a normal source + # checkout. Packaged Odin declares HOME=/opt/odin, so a broken packaged + # default still rejects on protected-root overlap rather than hiding a + # packaging failure inside the install. + fallback = Path.home() / ".odin-workspace" + try: + workspace = provision_workspace( + str(fallback), + protected_roots=protected_roots, + owner_uid=owner_uid, + allow_sudo=False, + ) + except WorkspaceError: + raise default_error + setattr(tools_config, "local_working_dir", str(workspace)) + return workspace + + def _sudo_create(target: Path, owner_uid: int | None) -> None: """Best-effort privileged creation; failures fall through to validation, which produces the actionable error.""" diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index 1de609aa..f695464b 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -1172,6 +1172,74 @@ def test_provisioner_rejects_everything_the_runtime_rejects( assert not (tmp_path / "relative-ws").exists(), "must not create a relative workspace" +def test_legacy_source_config_falls_back_when_var_lib_cannot_be_provisioned( + tmp_path: Path, fake_install: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The first update to this feature is executed by old updater code. + + A pre-PR source config has no workspace field, and its unprivileged account + cannot normally create /var/lib/odin-workspace or use sudo. Preserve local + command capability with a stable HOME fallback, but only for that absent + legacy field. + """ + import src.tools.workspace as ws_module + from src.config.schema import DEFAULT_LOCAL_WORKING_DIR + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + # Simulate the validated shape produced from a pre-feature config file. + # The session fixture changes ToolsConfig's default for unrelated tests, so + # model this migration boundary explicitly rather than accidentally testing + # the fixture's temporary value. + tools = SimpleNamespace( + local_working_dir=DEFAULT_LOCAL_WORKING_DIR, + model_fields_set=set(), + ) + real_provision = ws_module.provision_workspace + + def _default_unavailable(configured: str, **kwargs): + if configured == DEFAULT_LOCAL_WORKING_DIR: + raise WorkspaceError("simulated root-owned /var/lib and denied sudo") + return real_provision(configured, **kwargs) + + monkeypatch.setattr(ws_module, "provision_workspace", _default_unavailable) + result = ws_module.provision_startup_workspace( + tools, protected_roots=[str(fake_install)] + ) + + assert result == (home / ".odin-workspace").resolve() + assert stat.S_IMODE(result.stat().st_mode) == 0o700 + assert tools.local_working_dir == str(result), "all runtime consumers must see the migration" + + +def test_explicit_workspace_never_uses_the_legacy_source_fallback( + tmp_path: Path, fake_install: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only a missing pre-feature field migrates; an explicit operator value is + authoritative even when unusable.""" + import src.tools.workspace as ws_module + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + explicit = tmp_path / "missing-parent" / "operator-workspace" + tools = SimpleNamespace( + local_working_dir=str(explicit), + model_fields_set={"local_working_dir"}, + ) + + def _unavailable(_configured: str, **_kwargs): + raise WorkspaceError("operator workspace unavailable") + + monkeypatch.setattr(ws_module, "provision_workspace", _unavailable) + with pytest.raises(WorkspaceError, match="operator workspace unavailable"): + ws_module.provision_startup_workspace( + tools, protected_roots=[str(fake_install)] + ) + assert tools.local_working_dir == str(explicit) + assert not (home / ".odin-workspace").exists() + + def test_startup_migration_provisions_before_commands_are_served() -> None: """PR #239 round-5 blocker 1 — the bootstrap paradox. @@ -1183,8 +1251,8 @@ def test_startup_migration_provisions_before_commands_are_served() -> None: main_src = (Path(__file__).resolve().parents[1] / "src/__main__.py").read_text( encoding="utf-8" ) - assert "provision_workspace(" in main_src, "startup must provision the workspace" - provision_at = main_src.index("provision_workspace(") + assert "provision_startup_workspace(" in main_src, "startup must provision the workspace" + provision_at = main_src.index("provision_startup_workspace(") bot_at = main_src.index("bot = OdinBot(config)") config_at = main_src.index("config = load_config(config_path)") assert config_at < provision_at < bot_at, ( @@ -2152,16 +2220,29 @@ async def test_write_file_rejects_relative_paths( async def test_write_file_still_accepts_absolute_paths( - fake_install: Path, workspace: Path, tmp_path: Path + fake_install: Path, + workspace: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """The documented capability is untouched.""" + """The documented capability is untouched, including spaces in the path. + + The old ``mkdir -p $(dirname )`` still word-split dirname's + output. An absolute ``.../intended parent/file`` therefore created a + relative ``parent`` directory in the inherited install cwd and then failed + the intended write — another fixed-shape route touching the install. + """ + monkeypatch.chdir(fake_install) executor = _executor_with_workspace(workspace, fake_install) - target = tmp_path / "written.txt" + target = tmp_path / "intended parent" / "written file.txt" result = await executor.execute("write_file", { "host": "localhost", "path": str(target), "content": "hello", }) assert result.ok, result.output - assert target.read_text(encoding="utf-8").strip() == "hello" + assert target.read_text(encoding="utf-8") == "hello" + assert not (fake_install / "parent").exists(), ( + "an absolute write must not create word-split relative directories in the install" + ) def test_aliased_config_protects_both_the_alias_and_the_target(tmp_path: Path) -> None: @@ -2264,6 +2345,57 @@ def _record(**kw): assert not started, "a refresh was already running; a second must not start" +def test_waiting_usage_refresh_rechecks_a_newly_published_cache( + fake_install: Path, workspace: Path +) -> None: + """The unlocked TTL check is only a fast path. If this caller waits for + the lock while another refresh publishes, it must re-check freshness under + the lock instead of launching an immediate duplicate walk.""" + executor = _executor_with_workspace(workspace, fake_install) + executor._workspace_usage_cache = (0.0, 1.0, 1.0) # stale at first check + real_thread = threading.Thread + attempted = threading.Event() + inner = threading.Lock() + started: list[dict] = [] + + class _GateLock: + def __enter__(self): + attempted.set() + inner.acquire() + return self + + def __exit__(self, *_exc): + inner.release() + + class _RecordedThread: + def __init__(self, **kwargs): + started.append(kwargs) + + def start(self) -> None: + return None + + executor._workspace_usage_lock = _GateLock() + inner.acquire() + caller = real_thread(target=executor._refresh_workspace_usage, args=(workspace,)) + caller.start() + try: + assert attempted.wait(5), "refresh caller never reached the locked re-check" + # Simulate the refresh that completed while this caller waited. + executor._workspace_usage_cache = (time.monotonic(), 2.0, 2.0) + executor._workspace_usage_refreshing = False + with patch("src.tools.executor.threading.Thread", _RecordedThread): + inner.release() + caller.join(5) + finally: + # Never strand the test thread on a failing assertion. + if inner.locked(): + inner.release() + caller.join(5) + + assert not caller.is_alive() + assert started == [], "a newly fresh cache must suppress the duplicate walk" + + def test_usage_cache_is_published_before_single_flight_clears( fake_install: Path, workspace: Path ) -> None: @@ -2571,12 +2703,37 @@ def test_usage_refresh_flag_never_survives_the_thread( def _unexpected(*_a, **_kw): raise RuntimeError("not an OSError") - with patch("src.tools.executor.os.walk", _unexpected): + # Observe exceptions leaving the worker directly rather than letting + # pytest reduce them to a non-failing PytestUnhandledThreadExceptionWarning. + # The exact ee0d68d CI run was green while emitting that warning, so checking + # only the recovered flag is not enough. + real_thread = threading.Thread + escaped: list[BaseException] = [] + finished = threading.Event() + + class _ObservedThread: + def __init__(self, *, target, **_kwargs): + self._target = target + + def start(self) -> None: + def _run() -> None: + try: + self._target() + except BaseException as exc: + escaped.append(exc) + finally: + finished.set() + + real_thread(target=_run, daemon=True).start() + + with ( + patch("src.tools.executor.os.walk", _unexpected), + patch("src.tools.executor.threading.Thread", _ObservedThread), + ): executor.get_workspace_metrics() - deadline = time.monotonic() + 5 - while executor._workspace_usage_refreshing and time.monotonic() < deadline: - time.sleep(0.02) + assert finished.wait(5), "usage worker did not finish" + assert escaped == [], "metrics worker exceptions must not escape the daemon thread" assert executor._workspace_usage_refreshing is False, ( "the single-flight flag must not survive the thread on ANY exception" ) From 11c64ba9271315f775800b8bf4d39491f72ff374 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 21:27:19 -0400 Subject: [PATCH 19/21] =?UTF-8?q?fix:=20cross-review=20of=20a0d4770=20?= =?UTF-8?q?=E2=80=94=20the=20legacy=20fallback=20must=20announce=20itself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-reviewed Odin's commit to full bar: read the whole diff, independently reproduced both bug claims, mutation-tested his pins, and ran all four gates myself. Three of his four changes are correct as written and now pinned: - The post-lock freshness recheck is right; the unlocked fast-path observation is not authoritative once lock acquisition blocks. - Broadening the metrics worker to `except Exception` fixes a real defect in MY ee0d68d: a RuntimeError still escaped the daemon thread and surfaced as a PytestUnhandledThreadExceptionWarning in a run I reported as green. I saw "2 warnings" and did not chase it. Now zero occurrences. - write_file's `$(dirname ...)` was unquoted, so an absolute path containing spaces word-split: reproduced the requested write FAILING while `mkdir` created three stray relative directories in the inherited install cwd. Pre-existing, like the port-probe injection. ONE finding on the legacy-config fallback. The fallback itself is the right call — losing every local command on a first upgrade is worse than a private HOME directory, and explicit operator settings are correctly never substituted (verified independently). But its stated safety property does not hold: the packaged unit sets User= and NO Environment=HOME, so HOME comes from the account record and is typically OUTSIDE the install — /home/odin on this real deployment, not /opt/odin as the comment claimed. A broken PACKAGED default therefore falls back rather than rejecting, and with only an INFO log naming the path it was indistinguishable from normal operation: an operator would never learn their packaging was broken. Keeping the behaviour, removing the silence. The comment now states what is actually true. provision_startup_workspace takes an on_fallback callback and records process state; __main__ logs a WARNING naming the failed default, the reason, and the fix command; and the startup diagnostic — the report an operator actually reads — reports "ready at FALLBACK " with the reason and a recommendation, still passing, because a usable fallback is not a failure. Mutation-verified: silencing the fallback callback/state, dropping the diagnostic's fallback branch, reverting his freshness recheck, or reverting his write_file parent quoting each fails its test. Gates run independently: 7636 passed / 4 skipped, lint-no-new 0, types-no-new 0, coverage-no-drop findings=0, diff --check clean, no unhandled-thread warnings. --- src/__main__.py | 15 ++++++ src/health/startup.py | 17 +++++++ src/tools/workspace.py | 30 ++++++++++-- tests/test_local_workspace.py | 88 +++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index 219f1ac8..952d1c42 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -135,9 +135,24 @@ def main() -> None: provisioning_hint, ) + def _warn_fallback(path, reason) -> None: + # A fallback is not a failure, but it must never look like normal + # operation: on a packaged install it means the packaged default + # could not be provisioned, which the operator needs to know + # (cross-review of PR #239 round 13). + log.warning( + "Local command workspace fell back to %s — the configured " + "default could not be provisioned (%s). Local commands work, " + "but this indicates a packaging or permissions problem. %s", + path, + reason, + provisioning_hint(config.tools.local_working_dir), + ) + workspace = provision_startup_workspace( config.tools, protected_roots=_command_protected_roots(config), + on_fallback=_warn_fallback, ) log.info("Local command workspace ready: %s", workspace) except WorkspaceError as exc: diff --git a/src/health/startup.py b/src/health/startup.py index a1a9de4c..d66e00d7 100644 --- a/src/health/startup.py +++ b/src/health/startup.py @@ -466,6 +466,23 @@ def check_local_workspace(config: Any) -> DiagnosticResult: recommendation=provisioning_hint(configured), metadata={"configured": configured}, ) + # A legacy-config fallback is usable, so this still passes — but it must + # be visible in the report an operator reads, not buried as a path they + # would have to notice (cross-review of PR #239 round 13). + from ..tools.workspace import startup_fallback + + fallback = startup_fallback() + if fallback is not None and str(workspace) == fallback[0]: + return DiagnosticResult( + name="local_workspace", + passed=True, + detail=( + f"Local command workspace ready at FALLBACK {workspace} — the " + f"configured default could not be provisioned ({fallback[1]})" + ), + recommendation=provisioning_hint(configured), + metadata={"path": str(workspace), "fallback": True}, + ) return DiagnosticResult( name="local_workspace", passed=True, diff --git a/src/tools/workspace.py b/src/tools/workspace.py index 76697830..db4749e3 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -33,7 +33,7 @@ import os import stat -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path from types import SimpleNamespace @@ -51,6 +51,16 @@ # success, then handing over to an executor that refused every command. DEFAULT_MEMORY_PATH = "./data/memory.json" +# Set once at startup when the legacy-config fallback engages. Process-wide +# startup state, like the active config path — read by the startup diagnostic +# so a fallback is VISIBLE rather than indistinguishable from normal operation. +_STARTUP_FALLBACK: tuple[str, str] | None = None + + +def startup_fallback() -> tuple[str, str] | None: + """``(active_workspace, reason)`` if this process fell back, else None.""" + return _STARTUP_FALLBACK + class WorkspaceError(RuntimeError): """The configured local workspace is unusable. Never fall back to cwd.""" @@ -411,6 +421,7 @@ def provision_startup_workspace( *, protected_roots: Sequence[str | os.PathLike[str]] | None = None, owner_uid: int | None = None, + on_fallback: Callable[[Path, WorkspaceError], None] | None = None, ) -> Path: """Provision the workspace used by the incoming process at startup. @@ -444,9 +455,16 @@ def provision_startup_workspace( # A direct child of HOME needs no recursive parent creation, remains # stable across commands/restarts, and is outside a normal source - # checkout. Packaged Odin declares HOME=/opt/odin, so a broken packaged - # default still rejects on protected-root overlap rather than hiding a - # packaging failure inside the install. + # checkout. + # + # It is NOT a safety boundary for packaged installs. The packaged unit + # sets User= but no Environment=HOME, so HOME comes from the account + # record and is typically OUTSIDE the install (verified: /home/odin on + # a real deployment) — a broken packaged default therefore falls back + # here instead of rejecting. That is deliberate, because losing every + # local command is worse; it is made VISIBLE instead, via the warning + # below and the startup diagnostic, so a packaging failure is reported + # rather than hidden (cross-review of PR #239 round 13). fallback = Path.home() / ".odin-workspace" try: workspace = provision_workspace( @@ -458,6 +476,10 @@ def provision_startup_workspace( except WorkspaceError: raise default_error setattr(tools_config, "local_working_dir", str(workspace)) + global _STARTUP_FALLBACK + _STARTUP_FALLBACK = (str(workspace), f"{configured!r} unusable: {default_error}") + if on_fallback is not None: + on_fallback(workspace, default_error) return workspace diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index f695464b..e81a44bf 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2756,3 +2756,91 @@ def test_usage_cache_and_flag_are_published_atomically( # Whenever the flag is clear, a cache is present — never the gap. assert executor._workspace_usage_refreshing is False assert executor._workspace_usage_cache is not None + + +def test_legacy_fallback_is_visible_not_silent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Cross-review of Odin's round-13 legacy fallback. + + The fallback itself is right — losing every local command on a first + upgrade is worse than using a private HOME directory. But its stated safety + property does not hold: the packaged unit sets User= and no + Environment=HOME, so HOME comes from the account record and is typically + OUTSIDE the install (/home/odin on a real deployment). A broken PACKAGED + default therefore falls back rather than rejecting, and with only an + INFO log naming the path, that is indistinguishable from normal operation — + an operator would never learn their packaging is broken. + + So the fallback must announce itself: a callback for the caller to warn on, + and process state the startup diagnostic reports. + """ + import src.tools.workspace as workspace_module + from src.config.schema import ToolsConfig + from src.tools.workspace import provision_startup_workspace + + monkeypatch.setattr(workspace_module, "_STARTUP_FALLBACK", None) + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + + legacy = ToolsConfig() + object.__setattr__(legacy, "local_working_dir", str(tmp_path / "no-parent" / "ws")) + legacy.model_fields_set.discard("local_working_dir") + + seen: list[tuple[Path, str]] = [] + workspace = provision_startup_workspace( + legacy, + protected_roots=[str(tmp_path / "install")], + on_fallback=lambda path, reason: seen.append((path, str(reason))), + ) + + assert workspace == (home / ".odin-workspace").resolve() + assert seen, "the caller must be told a fallback happened" + assert "could not be created" in seen[0][1], "the reason must be carried" + + recorded = workspace_module.startup_fallback() + assert recorded is not None and recorded[0] == str(workspace) + + +def test_startup_diagnostic_announces_a_fallback_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The report an operator actually reads must say FALLBACK, not just show + a path they would have to recognise as unusual.""" + import src.tools.workspace as workspace_module + from src.config.schema import ToolsConfig + from src.health.startup import check_local_workspace + + active = tmp_path / "fallback-ws" + active.mkdir(mode=0o700) + monkeypatch.setattr( + workspace_module, + "_STARTUP_FALLBACK", + (str(active.resolve()), "'/var/lib/odin-workspace' unusable: permission denied"), + ) + + result = check_local_workspace(ToolsConfig(local_working_dir=str(active))) + assert result.passed is True, "a usable fallback is not a failure" + assert "FALLBACK" in result.detail + assert "permission denied" in result.detail + assert result.metadata.get("fallback") is True + assert result.recommendation, "must name how to restore the intended default" + + +def test_explicit_workspace_is_never_replaced_by_the_fallback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Odin's property, re-verified independently: an operator who set the + path gets a hard error, never a silent substitution.""" + from src.config.schema import ToolsConfig + from src.tools.workspace import provision_startup_workspace + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + + explicit = ToolsConfig(local_working_dir=str(tmp_path / "no-parent" / "ws")) + with pytest.raises(WorkspaceError): + provision_startup_workspace(explicit, protected_roots=[str(tmp_path / "install")]) + assert not (home / ".odin-workspace").exists(), "nothing may be provisioned" From e457f0e508aa466fc3ecfae9359a6936b6aadae2 Mon Sep 17 00:00:00 2001 From: ATCharpentier Date: Mon, 27 Jul 2026 21:34:49 -0400 Subject: [PATCH 20/21] test: make the fallback tests identity-independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed at 11c64ba where my local run was green — the precise gap Odin called out two rounds ago, and this time the cause was my test's premise rather than my code. Both new fallback tests assumed "a path whose parent does not exist cannot be provisioned". provision_workspace falls back to `sudo -n install -d`, which creates parent chains, so on a runner with passwordless sudo the path WAS created: the fallback never engaged and the explicit-config test stopped raising. The premise silently evaporated and the tests asserted nothing. They now use a parent that is a regular FILE, which defeats both mkdir and install -d regardless of identity. Verified by running the suite BOTH as the normal user and as root with passwordless sudo — 132 passed each way. That also confirms the six pre-existing "missing parent" tests are safe: they go through resolve_workspace, which has no sudo path. Gates: full suite green, lint 0, types 0, coverage findings 0, diff --check clean. --- tests/test_local_workspace.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index e81a44bf..d85b7ac3 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2758,6 +2758,20 @@ def test_usage_cache_and_flag_are_published_atomically( assert executor._workspace_usage_cache is not None +def _unprovisionable(tmp_path: Path) -> Path: + """A path that CANNOT be created under any identity. + + Deliberately not "a missing parent": provision_workspace falls back to + `sudo -n install -d`, which creates parent chains, so on a runner with + passwordless sudo (GitHub Actions) that premise silently evaporates and the + test asserts nothing. A parent that is a regular FILE defeats both mkdir + and install -d, as root or otherwise — verified under both. + """ + blocker = tmp_path / "blocker-file" + blocker.write_text("not a directory", encoding="utf-8") + return blocker / "ws" + + def test_legacy_fallback_is_visible_not_silent( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -2785,7 +2799,7 @@ def test_legacy_fallback_is_visible_not_silent( monkeypatch.setenv("HOME", str(home)) legacy = ToolsConfig() - object.__setattr__(legacy, "local_working_dir", str(tmp_path / "no-parent" / "ws")) + object.__setattr__(legacy, "local_working_dir", str(_unprovisionable(tmp_path))) legacy.model_fields_set.discard("local_working_dir") seen: list[tuple[Path, str]] = [] @@ -2840,7 +2854,7 @@ def test_explicit_workspace_is_never_replaced_by_the_fallback( home.mkdir() monkeypatch.setenv("HOME", str(home)) - explicit = ToolsConfig(local_working_dir=str(tmp_path / "no-parent" / "ws")) + explicit = ToolsConfig(local_working_dir=str(_unprovisionable(tmp_path))) with pytest.raises(WorkspaceError): provision_startup_workspace(explicit, protected_roots=[str(tmp_path / "install")]) assert not (home / ".odin-workspace").exists(), "nothing may be provisioned" From 18402015cc6eb6087726f17c8d8e1e266c21d431 Mon Sep 17 00:00:00 2001 From: Odin Date: Mon, 27 Jul 2026 21:53:37 -0400 Subject: [PATCH 21/21] fix: preserve intended workspace in fallback remediation --- src/__main__.py | 4 ++-- src/health/startup.py | 11 ++++++++--- src/tools/workspace.py | 15 +++++++++------ tests/test_local_workspace.py | 19 ++++++++++++++----- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/__main__.py b/src/__main__.py index 952d1c42..711420d8 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -135,7 +135,7 @@ def main() -> None: provisioning_hint, ) - def _warn_fallback(path, reason) -> None: + def _warn_fallback(path, configured, reason) -> None: # A fallback is not a failure, but it must never look like normal # operation: on a packaged install it means the packaged default # could not be provisioned, which the operator needs to know @@ -146,7 +146,7 @@ def _warn_fallback(path, reason) -> None: "but this indicates a packaging or permissions problem. %s", path, reason, - provisioning_hint(config.tools.local_working_dir), + provisioning_hint(configured), ) workspace = provision_startup_workspace( diff --git a/src/health/startup.py b/src/health/startup.py index d66e00d7..935f17af 100644 --- a/src/health/startup.py +++ b/src/health/startup.py @@ -473,15 +473,20 @@ def check_local_workspace(config: Any) -> DiagnosticResult: fallback = startup_fallback() if fallback is not None and str(workspace) == fallback[0]: + _active, intended, reason = fallback return DiagnosticResult( name="local_workspace", passed=True, detail=( f"Local command workspace ready at FALLBACK {workspace} — the " - f"configured default could not be provisioned ({fallback[1]})" + f"configured default {intended!r} could not be provisioned ({reason})" ), - recommendation=provisioning_hint(configured), - metadata={"path": str(workspace), "fallback": True}, + recommendation=provisioning_hint(intended), + metadata={ + "path": str(workspace), + "configured": intended, + "fallback": True, + }, ) return DiagnosticResult( name="local_workspace", diff --git a/src/tools/workspace.py b/src/tools/workspace.py index db4749e3..ec5785f2 100644 --- a/src/tools/workspace.py +++ b/src/tools/workspace.py @@ -54,11 +54,14 @@ # Set once at startup when the legacy-config fallback engages. Process-wide # startup state, like the active config path — read by the startup diagnostic # so a fallback is VISIBLE rather than indistinguishable from normal operation. -_STARTUP_FALLBACK: tuple[str, str] | None = None +# Keep the failed configured path as well as the active fallback: startup +# mutates ToolsConfig to the fallback so every runtime consumer agrees, and +# without this copy later remediation would point at the already-working path. +_STARTUP_FALLBACK: tuple[str, str, str] | None = None -def startup_fallback() -> tuple[str, str] | None: - """``(active_workspace, reason)`` if this process fell back, else None.""" +def startup_fallback() -> tuple[str, str, str] | None: + """``(active_workspace, configured_workspace, reason)`` after fallback.""" return _STARTUP_FALLBACK @@ -421,7 +424,7 @@ def provision_startup_workspace( *, protected_roots: Sequence[str | os.PathLike[str]] | None = None, owner_uid: int | None = None, - on_fallback: Callable[[Path, WorkspaceError], None] | None = None, + on_fallback: Callable[[Path, str, WorkspaceError], None] | None = None, ) -> Path: """Provision the workspace used by the incoming process at startup. @@ -477,9 +480,9 @@ def provision_startup_workspace( raise default_error setattr(tools_config, "local_working_dir", str(workspace)) global _STARTUP_FALLBACK - _STARTUP_FALLBACK = (str(workspace), f"{configured!r} unusable: {default_error}") + _STARTUP_FALLBACK = (str(workspace), configured, str(default_error)) if on_fallback is not None: - on_fallback(workspace, default_error) + on_fallback(workspace, configured, default_error) return workspace diff --git a/tests/test_local_workspace.py b/tests/test_local_workspace.py index d85b7ac3..228708c0 100644 --- a/tests/test_local_workspace.py +++ b/tests/test_local_workspace.py @@ -2802,19 +2802,24 @@ def test_legacy_fallback_is_visible_not_silent( object.__setattr__(legacy, "local_working_dir", str(_unprovisionable(tmp_path))) legacy.model_fields_set.discard("local_working_dir") - seen: list[tuple[Path, str]] = [] + intended = str(_unprovisionable(tmp_path)) + seen: list[tuple[Path, str, str]] = [] workspace = provision_startup_workspace( legacy, protected_roots=[str(tmp_path / "install")], - on_fallback=lambda path, reason: seen.append((path, str(reason))), + on_fallback=lambda path, configured, reason: seen.append( + (path, configured, str(reason)) + ), ) assert workspace == (home / ".odin-workspace").resolve() assert seen, "the caller must be told a fallback happened" - assert "could not be created" in seen[0][1], "the reason must be carried" + assert seen[0][1] == intended + assert "could not be created" in seen[0][2], "the reason must be carried" recorded = workspace_module.startup_fallback() assert recorded is not None and recorded[0] == str(workspace) + assert recorded[1] == intended def test_startup_diagnostic_announces_a_fallback_workspace( @@ -2831,7 +2836,7 @@ def test_startup_diagnostic_announces_a_fallback_workspace( monkeypatch.setattr( workspace_module, "_STARTUP_FALLBACK", - (str(active.resolve()), "'/var/lib/odin-workspace' unusable: permission denied"), + (str(active.resolve()), "/var/lib/odin-workspace", "permission denied"), ) result = check_local_workspace(ToolsConfig(local_working_dir=str(active))) @@ -2839,7 +2844,11 @@ def test_startup_diagnostic_announces_a_fallback_workspace( assert "FALLBACK" in result.detail assert "permission denied" in result.detail assert result.metadata.get("fallback") is True - assert result.recommendation, "must name how to restore the intended default" + assert result.metadata.get("configured") == "/var/lib/odin-workspace" + assert "/var/lib/odin-workspace" in result.recommendation + assert str(active) not in result.recommendation, ( + "remediation must repair the failed default, not the working fallback" + ) def test_explicit_workspace_is_never_replaced_by_the_fallback(