diff --git a/Makefile b/Makefile index 9493494..37fa0ed 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Top-level workflow runner -.PHONY: soul restart routingsim routing-regression-live routing-regression-live-tools routing-regression-live-dry-run eval-routing eval-routing-tools eval-compare eval-run eval-reliability eval-reliability-daily monthly-scorecard route-hygiene lane-hotspots stop-gate-enforce canary-health-check canary-stage skill-discovery assistant-skill-refresh firecrawl-wire-pilot acpx-pilot acpx-smoke github-workflow-pilot assistant-agenda-refresh incident-bundle error-review session-maintenance dreaming-preview dreaming-status dreaming-help dreaming-on dreaming-off operator-health-bundle party-batch-once inbox-cycle task-loop task-loop-heartbeat task-loop-weekly looptest avatar audio-check lint dev config-validate openclaw-compat toolset-audit task-packets plan-graph test shellcheck redteam-validate redteam-gate mcp-harness-smoke policy-gate-check orion-policy-check policy-scorecard secure-preflight-check supply-chain-check llm-vuln-probe-check langfuse-bootstrap-check mcp-schema-check llm-provider-bench llm-provider-bench-dry llm-provider-configure-dry skill-guards-smoke ci +.PHONY: soul restart routingsim routing-regression-live routing-regression-live-tools routing-regression-live-dry-run eval-routing eval-routing-tools eval-compare eval-run eval-reliability eval-reliability-daily monthly-scorecard route-hygiene lane-hotspots stop-gate-enforce canary-health-check canary-stage skill-discovery assistant-skill-refresh firecrawl-wire-pilot acpx-pilot acpx-smoke github-workflow-pilot assistant-agenda-refresh incident-bundle error-review session-maintenance dreaming-preview dreaming-status dreaming-help dreaming-on dreaming-off operator-health-bundle machine-status party-batch-once inbox-cycle task-loop task-loop-heartbeat task-loop-weekly looptest avatar audio-check lint dev config-validate openclaw-compat toolset-audit task-packets plan-graph test shellcheck redteam-validate redteam-gate mcp-harness-smoke policy-gate-check orion-policy-check policy-scorecard secure-preflight-check supply-chain-check llm-vuln-probe-check langfuse-bootstrap-check mcp-schema-check llm-provider-bench llm-provider-bench-dry llm-provider-configure-dry skill-guards-smoke ci PROMPTFOO_CONFIG ?= config/promptfoo/orion-safety-gate.yaml THINKING ?= high @@ -228,6 +228,10 @@ operator-health-bundle: $$args \ --json +## Read-only machine/ORION health digest; no repairs, restarts, provider probes, or Telegram sends +machine-status: + @python3 scripts/orion_machine_status.py --repo-root . --json + ## One-shot coding-party batch (eval + reliability + canary health) party-batch-once: @python3 scripts/party_batch_once.py --repo-root . diff --git a/scripts/README.md b/scripts/README.md index 6498079..78f7bb6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -899,6 +899,32 @@ Notes: --- +## orion_machine_status.py + +### Purpose +Read-only machine/ORION digest for the recurring "is everything okay?" check. + +It summarizes: +- gateway resurrector log classification +- latest operator health bundle artifact, if present +- relevant LaunchAgent last-exit/path drift signals +- Codex/OpenClaw paths and versions +- storage watcher headline, if configured +- Remodex bridge status, if installed + +### Usage + +```bash +make machine-status +python3 scripts/orion_machine_status.py --repo-root . --json +``` + +Notes: +- This script does not repair, restart, update, probe live models, send Telegram messages, or install LaunchAgents. +- Missing optional tools degrade to `unknown` instead of failing the digest. + +--- + ## polymarket_sports_paper.py ### Purpose diff --git a/scripts/orion_machine_status.py b/scripts/orion_machine_status.py new file mode 100644 index 0000000..ae1f053 --- /dev/null +++ b/scripts/orion_machine_status.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Read-only ORION machine status digest. + +This script intentionally avoids repairs, restarts, Telegram sends, provider +probes, and LaunchAgent changes. It composes local evidence into one compact +operator-readable status object. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(text: str) -> str: + return ANSI_RE.sub("", text) + + +def _run(argv: list[str], *, cwd: Path | None = None, timeout: int = 15) -> dict[str, Any]: + try: + result = subprocess.run( + argv, + cwd=str(cwd) if cwd else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + timeout=timeout, + ) + except Exception as exc: + return {"ok": False, "returncode": None, "stdout": "", "stderr": str(exc)} + return { + "ok": result.returncode == 0, + "returncode": result.returncode, + "stdout": result.stdout or "", + "stderr": result.stderr or "", + } + + +def _warn(warnings: list[str], message: str) -> None: + if message not in warnings: + warnings.append(message) + + +def _resolve_binary(name: str, warnings: list[str]) -> dict[str, Any]: + path_result = _run(["/bin/zsh", "-lc", f"command -v {name}"]) + path = path_result["stdout"].strip().splitlines()[0] if path_result["ok"] and path_result["stdout"].strip() else None + version = None + version_result = _run([name, "--version"], timeout=20) + if version_result["ok"] and version_result["stdout"].strip(): + version = version_result["stdout"].strip().splitlines()[0] + elif path is None: + _warn(warnings, f"{name} binary not found") + else: + _warn(warnings, f"{name} version check failed") + return {"path": path, "version": version} + + +def _read_gateway_guard(logs_dir: Path, warnings: list[str]) -> dict[str, Any]: + log_path = logs_dir / "orion_resurrector.log" + if not log_path.exists(): + _warn(warnings, f"gateway guard log missing: {log_path}") + return { + "status": "unknown", + "log_path": str(log_path), + "last_classification": None, + "last_line": None, + } + + lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines() + last_line = next((line for line in reversed(lines) if line.strip()), None) + last_classification = None + for line in reversed(lines): + match = re.search(r"classification=([a-z]+)", line) + if match: + last_classification = match.group(1) + break + + status = "unknown" + if last_classification == "healthy": + status = "ok" + elif last_classification == "degraded": + status = "warn" + _warn(warnings, "gateway guard last classified gateway as degraded") + elif last_classification == "failed": + status = "warn" + _warn(warnings, "gateway guard last classified gateway as failed") + else: + _warn(warnings, "gateway guard log has no classification") + + return { + "status": status, + "log_path": str(log_path), + "last_classification": last_classification, + "last_line": last_line, + } + + +def _read_operator_health_bundle(repo_root: Path, warnings: list[str]) -> dict[str, Any]: + candidates = [ + repo_root / "tmp" / "openclaw_operator_health_bundle_latest.json", + repo_root / "tmp" / "operator-health-bundle.json", + ] + for path in candidates: + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + _warn(warnings, f"operator health bundle unreadable: {path}: {exc}") + return {"status": "warn", "path": str(path), "summary": None} + status = payload.get("status") + if status is None and isinstance(payload.get("summary"), dict): + overall_ok = payload["summary"].get("overall_ok") + if overall_ok is True: + status = "ok" + elif overall_ok is False: + status = "warn" + status = str(status or "unknown") + if status not in {"ok", "healthy"}: + _warn(warnings, f"operator health bundle status is {status}") + return { + "status": status, + "path": str(path), + "summary": { + "gateway": payload.get("gateway"), + "channels": payload.get("channels"), + }, + } + _warn(warnings, "operator health bundle not found") + return {"status": "unknown", "path": None, "summary": None} + + +def _read_launchagents(launch_agents_dir: Path, warnings: list[str]) -> dict[str, Any]: + list_result = _run(["launchctl", "list"], timeout=20) + entries: list[dict[str, Any]] = [] + if list_result["ok"]: + for raw in list_result["stdout"].splitlines(): + parts = raw.split(None, 2) + if len(parts) != 3: + continue + pid, status, label = parts + if "orion" not in label and "openclaw" not in label and "remodex" not in label: + continue + entry = {"pid": pid, "last_exit": status, "label": label} + entries.append(entry) + if pid == "-" and status not in {"0", "-"}: + _warn(warnings, f"LaunchAgent {label} last exited with {status}") + else: + _warn(warnings, "launchctl list unavailable") + + drift: list[str] = [] + if launch_agents_dir.exists(): + for path in sorted(launch_agents_dir.glob("*.plist")): + name = path.name + if "orion" not in name and "openclaw" not in name: + continue + text = path.read_text(encoding="utf-8", errors="replace") + if "/Desktop/ORION" in text and "/Desktop/ORION ->" not in text: + drift.append(str(path)) + _warn(warnings, f"LaunchAgent references Desktop/ORION: {path.name}") + else: + _warn(warnings, f"LaunchAgents dir missing: {launch_agents_dir}") + + return { + "status": "warn" if drift or any(e["pid"] == "-" and e["last_exit"] not in {"0", "-"} for e in entries) else "ok", + "entries": entries, + "path_drift": drift, + } + + +def _read_storage(storage_watch: Path | None, warnings: list[str]) -> dict[str, Any]: + if storage_watch is None or not storage_watch.exists(): + return {"status": "unknown", "headline": None, "alerts": []} + result = _run([str(storage_watch)], timeout=60) + if not result["ok"]: + _warn(warnings, "storage watcher failed") + return {"status": "warn", "headline": None, "alerts": []} + lines = [_strip_ansi(line).strip() for line in result["stdout"].splitlines() if line.strip()] + alerts = [line for line in lines if "ALERT" in line or "WARN" in line] + if alerts: + _warn(warnings, "storage watcher reported warnings") + return { + "status": "warn" if alerts else "ok", + "headline": lines[0] if lines else None, + "alerts": alerts, + } + + +def _read_remodex(warnings: list[str]) -> dict[str, Any]: + result = _run(["remodex", "status"], timeout=20) + if not result["ok"]: + return {"status": "unknown", "headline": None} + lines = [line.strip() for line in result["stdout"].splitlines() if line.strip()] + text = "\n".join(lines).lower() + status = "ok" if "connected" in text or "running" in text else "warn" + if status == "warn": + _warn(warnings, "remodex status is not clearly connected/running") + return {"status": status, "headline": lines[0] if lines else None} + + +def collect_status( + repo_root: Path, + *, + logs_dir: Path, + launch_agents_dir: Path, + storage_watch: Path | None, +) -> dict[str, Any]: + warnings: list[str] = [] + payload: dict[str, Any] = { + "generated_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + "repo_root": str(repo_root), + "status": "ok", + "warnings": warnings, + "gateway_guard": _read_gateway_guard(logs_dir, warnings), + "operator_health_bundle": _read_operator_health_bundle(repo_root, warnings), + "launchagents": _read_launchagents(launch_agents_dir, warnings), + "binaries": { + "codex": _resolve_binary("codex", warnings), + "openclaw": _resolve_binary("openclaw", warnings), + }, + "storage": _read_storage(storage_watch, warnings), + "remodex": _read_remodex(warnings), + } + payload["status"] = "warn" if warnings else "ok" + return payload + + +def _print_text(payload: dict[str, Any]) -> None: + print(f"ORION machine status: {payload['status']}") + for warning in payload["warnings"]: + print(f"- {warning}") + print(f"Gateway guard: {payload['gateway_guard']['status']} ({payload['gateway_guard']['last_classification']})") + print(f"Operator bundle: {payload['operator_health_bundle']['status']}") + print(f"LaunchAgents: {payload['launchagents']['status']}") + print(f"Codex: {payload['binaries']['codex']['version'] or 'unknown'}") + print(f"OpenClaw: {payload['binaries']['openclaw']['version'] or 'unknown'}") + print(f"Storage: {payload['storage']['status']} {payload['storage']['headline'] or ''}".rstrip()) + print(f"Remodex: {payload['remodex']['status']}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", default=str(Path(__file__).resolve().parents[1])) + parser.add_argument("--logs-dir", default=str(Path.home() / "Library" / "Logs")) + parser.add_argument("--launch-agents-dir", default=str(Path.home() / "Library" / "LaunchAgents")) + parser.add_argument("--storage-watch", default=str(Path.home() / "bin" / "storage-watch.sh")) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + storage_watch = Path(args.storage_watch).expanduser() if args.storage_watch else None + payload = collect_status( + Path(args.repo_root).expanduser().resolve(), + logs_dir=Path(args.logs_dir).expanduser(), + launch_agents_dir=Path(args.launch_agents_dir).expanduser(), + storage_watch=storage_watch, + ) + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + _print_text(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/resurrect_orion_mac.sh b/scripts/resurrect_orion_mac.sh index 9ade3dd..9e3c2a7 100755 --- a/scripts/resurrect_orion_mac.sh +++ b/scripts/resurrect_orion_mac.sh @@ -2,6 +2,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export PATH="/opt/homebrew/bin:/opt/homebrew/sbin:${HOME}/.npm-global/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:${PATH:-}" default_openclaw="${repo_root}/scripts/openclaww.sh" OPENCLAW_BIN="${OPENCLAW_BIN:-$default_openclaw}" GATEWAY_BASE_URL="${ORION_GATEWAY_BASE_URL:-http://127.0.0.1:18789}" @@ -116,18 +117,24 @@ gateway_state() { tmp="$(mktemp)" local status_rc=0 if ! "$OPENCLAW_BIN" gateway status --json >"$tmp" 2>/dev/null; then - status_rc=$? + status_rc=1 fi - local loaded runtime rpc + local loaded=0 + local runtime="unknown" + local rpc="unknown" if [[ "$status_rc" -eq 0 ]]; then - eval "$( + local parsed + if parsed="$( python3 - "$tmp" <<'PY' import json import shlex import sys -data = json.load(open(sys.argv[1], "r", encoding="utf-8")) +try: + data = json.load(open(sys.argv[1], "r", encoding="utf-8")) +except Exception: + raise SystemExit(1) service = data.get("service") or {} runtime = service.get("runtime") or {} rpc = (data.get("rpc") or {}).get("ok") @@ -142,11 +149,9 @@ print(f"loaded={'1' if service.get('loaded') else '0'}") print(f"runtime={shlex.quote(runtime_status)}") print(f"rpc={shlex.quote(rpc_status)}") PY - )" - else - loaded=0 - runtime="unknown" - rpc="unknown" + )"; then + eval "$parsed" + fi fi rm -f "$tmp" @@ -160,13 +165,13 @@ PY probe_http "${GATEWAY_BASE_URL%/}/readyz" && readyz=1 || true probe_http "${GATEWAY_BASE_URL%/}/healthz" && healthz=1 || true - local classification="degraded" - if [[ "$loaded" == "1" && "$runtime" == "running" && "$port_listening" == "1" ]]; then - if [[ "$rpc" == "1" || "$readyz" == "1" || "$healthz" == "1" ]]; then - classification="healthy" - elif [[ "$rpc" == "0" && "$readyz" == "0" && "$healthz" == "0" ]]; then - classification="failed" - fi + local classification="failed" + if [[ "$loaded" == "1" && "$runtime" == "running" && "$port_listening" == "1" && "$rpc" == "1" ]]; then + classification="healthy" + elif [[ "$readyz" == "1" || "$healthz" == "1" ]]; then + classification="degraded" + elif [[ "$loaded" == "1" && "$runtime" == "running" && "$port_listening" == "1" && "$rpc" != "0" ]]; then + classification="degraded" else classification="failed" fi diff --git a/tests/test_orion_machine_status.py b/tests/test_orion_machine_status.py new file mode 100644 index 0000000..5269535 --- /dev/null +++ b/tests/test_orion_machine_status.py @@ -0,0 +1,176 @@ +import importlib.util +import io +import json +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +def _load_status_module(): + repo_root = Path(__file__).resolve().parents[1] + script_path = repo_root / "scripts" / "orion_machine_status.py" + assert script_path.exists(), f"Missing script: {script_path}" + spec = importlib.util.spec_from_file_location("orion_machine_status", script_path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) # type: ignore[attr-defined] + return mod + + +class TestOrionMachineStatus(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.status = _load_status_module() + + def _run_status(self, root: Path, responses: dict[tuple[str, ...], object], extra_args: list[str] | None = None): + argv = [ + "orion_machine_status.py", + "--repo-root", + str(root), + "--json", + ] + if extra_args: + argv.extend(extra_args) + + def fake_run(argv, cwd=None, stdout=None, stderr=None, stdin=None, text=None, check=None, timeout=None, **_kwargs): + key = tuple(argv) + value = responses.get(key) + if value is None: + raise AssertionError(f"Unexpected command: {argv}") + if isinstance(value, Exception): + raise value + return SimpleNamespace( + returncode=value["returncode"], + stdout=value.get("stdout", ""), + stderr=value.get("stderr", ""), + ) + + stdout = io.StringIO() + with mock.patch.object(self.status.subprocess, "run", side_effect=fake_run): + with mock.patch.object(sys, "argv", argv): + with redirect_stdout(stdout): + rc = self.status.main() + return rc, json.loads(stdout.getvalue()) + + def test_digest_returns_stable_top_level_keys(self): + with tempfile.TemporaryDirectory() as td_name: + root = Path(td_name) + logs = root / "logs" + logs.mkdir() + (logs / "orion_resurrector.log").write_text( + "[2026-05-21 10:00:00] Pre-check: classification=degraded loaded=0 runtime=unknown rpc=unknown port=1 readyz=1 healthz=1\n" + "[2026-05-21 10:00:00] Gateway degraded but still serving local probes; skipping automatic restart\n", + encoding="utf-8", + ) + latest = root / "tmp" / "openclaw_operator_health_bundle_latest.json" + latest.parent.mkdir(parents=True) + latest.write_text(json.dumps({"status": "ok", "gateway": {"runtime_status": "running"}}), encoding="utf-8") + launch_agents = root / "LaunchAgents" + launch_agents.mkdir() + (launch_agents / "com.openclaw.orion.resurrector.plist").write_text( + "ProgramArguments" + f"{root}/scripts/resurrect_orion_mac.sh" + "", + encoding="utf-8", + ) + storage_watch = root / "storage-watch.sh" + storage_watch.write_text("#!/usr/bin/env bash\n", encoding="utf-8") + + responses = { + ("launchctl", "list"): { + "returncode": 0, + "stdout": "123\t0\tcom.openclaw.orion.resurrector\n-\t78\tcom.remodex.bridge\n", + }, + ("/bin/zsh", "-lc", "command -v codex"): {"returncode": 0, "stdout": "/tmp/bin/codex\n"}, + ("codex", "--version"): {"returncode": 0, "stdout": "codex-cli 0.131.0\n"}, + ("/bin/zsh", "-lc", "command -v openclaw"): {"returncode": 0, "stdout": "/tmp/bin/openclaw\n"}, + ("openclaw", "--version"): {"returncode": 0, "stdout": "OpenClaw 2026.5.12\n"}, + (str(storage_watch),): {"returncode": 0, "stdout": "Data OK 120Gi free\nhome .git ALERT 47.2G\n"}, + ("remodex", "status"): {"returncode": 0, "stdout": "Bridge: running\nConnection: connected\n"}, + } + + rc, payload = self._run_status( + root, + responses, + [ + "--logs-dir", + str(logs), + "--launch-agents-dir", + str(launch_agents), + "--storage-watch", + str(storage_watch), + ], + ) + + self.assertEqual(rc, 0) + self.assertEqual( + set(payload), + { + "generated_at", + "repo_root", + "status", + "warnings", + "gateway_guard", + "operator_health_bundle", + "launchagents", + "binaries", + "storage", + "remodex", + }, + ) + self.assertEqual(payload["gateway_guard"]["last_classification"], "degraded") + self.assertEqual(payload["operator_health_bundle"]["status"], "ok") + self.assertEqual(payload["binaries"]["codex"]["version"], "codex-cli 0.131.0") + self.assertEqual(payload["binaries"]["openclaw"]["version"], "OpenClaw 2026.5.12") + self.assertEqual(payload["storage"]["headline"], "Data OK 120Gi free") + self.assertEqual(payload["remodex"]["status"], "ok") + self.assertEqual(payload["status"], "warn") + self.assertTrue(any("com.remodex.bridge" in warning for warning in payload["warnings"])) + + def test_missing_optional_tools_degrade_to_unknown(self): + with tempfile.TemporaryDirectory() as td_name: + root = Path(td_name) + logs = root / "logs" + logs.mkdir() + launch_agents = root / "LaunchAgents" + launch_agents.mkdir() + responses = { + ("launchctl", "list"): FileNotFoundError("launchctl missing"), + ("/bin/zsh", "-lc", "command -v codex"): {"returncode": 1, "stdout": ""}, + ("codex", "--version"): FileNotFoundError("codex missing"), + ("/bin/zsh", "-lc", "command -v openclaw"): {"returncode": 1, "stdout": ""}, + ("openclaw", "--version"): FileNotFoundError("openclaw missing"), + ("remodex", "status"): FileNotFoundError("remodex missing"), + } + + rc, payload = self._run_status( + root, + responses, + [ + "--logs-dir", + str(logs), + "--launch-agents-dir", + str(launch_agents), + "--storage-watch", + str(root / "missing-storage-watch.sh"), + ], + ) + + self.assertEqual(rc, 0) + self.assertEqual(payload["status"], "warn") + self.assertEqual(payload["gateway_guard"]["status"], "unknown") + self.assertEqual(payload["operator_health_bundle"]["status"], "unknown") + self.assertEqual(payload["binaries"]["codex"]["path"], None) + self.assertEqual(payload["binaries"]["openclaw"]["version"], None) + self.assertEqual(payload["storage"]["status"], "unknown") + self.assertEqual(payload["remodex"]["status"], "unknown") + self.assertGreaterEqual(len(payload["warnings"]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_resurrect_orion_mac.py b/tests/test_resurrect_orion_mac.py index a9109c5..baf7023 100644 --- a/tests/test_resurrect_orion_mac.py +++ b/tests/test_resurrect_orion_mac.py @@ -34,13 +34,45 @@ def log_message(self, format, *args): # noqa: A003 thread.start() return server, f"http://127.0.0.1:{server.server_address[1]}" - def _write_fake_openclaw(self, root: Path, *, fail_until_restart: bool, log_path: Path) -> Path: + def _start_restart_gated_http_server(self, state_path: Path): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + if self.path in {"/readyz", "/healthz"}: + if state_path.exists() and state_path.read_text(encoding="utf-8").strip() == "restarted": + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok\n") + return + self.send_response(503) + self.end_headers() + self.wfile.write(b"starting\n") + return + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): # noqa: A003 + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://127.0.0.1:{server.server_address[1]}" + + def _write_fake_openclaw( + self, + root: Path, + *, + fail_until_restart: bool, + log_path: Path, + status_mode: str = "normal", + ) -> Path: state_path = root / "state.txt" script = root / "fake-openclaw.sh" body = f"""#!/usr/bin/env bash set -euo pipefail state_file={state_path!s} log_file={log_path!s} +status_mode={status_mode} cmd="${{1-}}" shift || true state="fresh" @@ -54,6 +86,14 @@ def _write_fake_openclaw(self, root: Path, *, fail_until_restart: bool, log_path case "$sub" in status) if [[ "${{1-}}" == "--json" ]]; then + if [[ "$status_mode" == "exit-fail" ]]; then + echo "status command unavailable" >&2 + exit 7 + fi + if [[ "$status_mode" == "invalid-json" ]]; then + echo "{{not json" + exit 0 + fi if [[ "{'1' if fail_until_restart else '0'}" == "1" && "$state" != "restarted" ]]; then cat <<'JSON' {{"service":{{"loaded":false,"runtime":{{"status":"stopped"}},"configAudit":{{"ok":true}}}},"rpc":{{"ok":false}}}} @@ -120,12 +160,78 @@ def test_healthy_gateway_skips_restart(self): self.assertIn("no action needed", result.stdout.lower()) self.assertFalse(actions.exists(), actions.read_text(encoding="utf-8") if actions.exists() else "") + def test_http_healthy_gateway_skips_restart_when_status_command_fails(self): + with tempfile.TemporaryDirectory() as td_name: + td = Path(td_name) + actions = td / "actions.log" + fake = self._write_fake_openclaw( + td, + fail_until_restart=True, + log_path=actions, + status_mode="exit-fail", + ) + server, base_url = self._start_http_server() + try: + env = dict(os.environ) + env["OPENCLAW_BIN"] = str(fake) + env["ORION_GATEWAY_BASE_URL"] = base_url + env["ORION_GATEWAY_GUARD_STATE_DIR"] = str(td / "state") + env["ORION_GATEWAY_SETTLE_SECONDS"] = "0" + result = subprocess.run( + [str(self._script_path())], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + finally: + server.shutdown() + server.server_close() + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("degraded but still serving local probes", result.stdout.lower()) + self.assertFalse(actions.exists(), actions.read_text(encoding="utf-8") if actions.exists() else "") + + def test_http_healthy_gateway_skips_restart_when_status_json_is_invalid(self): + with tempfile.TemporaryDirectory() as td_name: + td = Path(td_name) + actions = td / "actions.log" + fake = self._write_fake_openclaw( + td, + fail_until_restart=True, + log_path=actions, + status_mode="invalid-json", + ) + server, base_url = self._start_http_server() + try: + env = dict(os.environ) + env["OPENCLAW_BIN"] = str(fake) + env["ORION_GATEWAY_BASE_URL"] = base_url + env["ORION_GATEWAY_GUARD_STATE_DIR"] = str(td / "state") + env["ORION_GATEWAY_SETTLE_SECONDS"] = "0" + result = subprocess.run( + [str(self._script_path())], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + finally: + server.shutdown() + server.server_close() + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("degraded but still serving local probes", result.stdout.lower()) + self.assertFalse(actions.exists(), actions.read_text(encoding="utf-8") if actions.exists() else "") + def test_failed_gateway_restarts_and_recovers(self): with tempfile.TemporaryDirectory() as td_name: td = Path(td_name) actions = td / "actions.log" fake = self._write_fake_openclaw(td, fail_until_restart=True, log_path=actions) - server, base_url = self._start_http_server() + server, base_url = self._start_restart_gated_http_server(td / "state.txt") try: env = dict(os.environ) env["OPENCLAW_BIN"] = str(fake) @@ -149,6 +255,95 @@ def test_failed_gateway_restarts_and_recovers(self): self.assertIn("gateway recovery complete", result.stdout.lower()) self.assertEqual(actions.read_text(encoding="utf-8").strip(), "restart") + def test_cli_unknown_and_http_down_restarts(self): + with tempfile.TemporaryDirectory() as td_name: + td = Path(td_name) + actions = td / "actions.log" + fake = self._write_fake_openclaw( + td, + fail_until_restart=True, + log_path=actions, + status_mode="exit-fail", + ) + env = dict(os.environ) + env["OPENCLAW_BIN"] = str(fake) + env["ORION_GATEWAY_BASE_URL"] = "http://127.0.0.1:1" + env["ORION_GATEWAY_GUARD_STATE_DIR"] = str(td / "state") + env["ORION_GATEWAY_SETTLE_SECONDS"] = "0" + result = subprocess.run( + [str(self._script_path())], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("restarting openclaw gateway", result.stdout.lower()) + self.assertIn("gateway still unhealthy after restart", result.stdout.lower()) + self.assertEqual(actions.read_text(encoding="utf-8").strip(), "restart") + + def test_minimal_path_resolves_user_node_and_openclaw_through_wrapper(self): + with tempfile.TemporaryDirectory() as td_name: + td = Path(td_name) + home = td / "home" + bin_dir = home / ".npm-global" / "bin" + bin_dir.mkdir(parents=True) + actions = td / "actions.log" + node = bin_dir / "node" + node.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + node.chmod(node.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + openclaw = bin_dir / "openclaw" + openclaw.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +if ! command -v node >/dev/null 2>&1; then + echo "node unavailable" >&2 + exit 66 +fi +if [[ "$(command -v node)" == "/usr/local/bin/node" ]]; then + echo "unexpected node: $(command -v node)" >&2 + exit 67 +fi +if [[ "${{1-}}" == "gateway" && "${{2-}}" == "status" && "${{3-}}" == "--json" ]]; then + cat <<'JSON' +{{"service":{{"loaded":true,"runtime":{{"status":"running"}},"configAudit":{{"ok":true}}}},"rpc":{{"ok":true}}}} +JSON + echo status-ok >> {actions} + exit 0 +fi +echo "unexpected command: $*" >&2 +exit 2 +""", + encoding="utf-8", + ) + openclaw.chmod(openclaw.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + server, base_url = self._start_http_server() + try: + env = { + "HOME": str(home), + "PATH": "/usr/bin:/bin", + "ORION_GATEWAY_BASE_URL": base_url, + "ORION_GATEWAY_GUARD_STATE_DIR": str(td / "state"), + "ORION_GATEWAY_SETTLE_SECONDS": "0", + } + result = subprocess.run( + [str(self._script_path())], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + finally: + server.shutdown() + server.server_close() + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("no action needed", result.stdout.lower()) + self.assertEqual(actions.read_text(encoding="utf-8").strip(), "status-ok") + def test_restart_guard_blocks_flapping(self): with tempfile.TemporaryDirectory() as td_name: td = Path(td_name)