diff --git a/.github/workflows/runtime-isolation-shadow.yml b/.github/workflows/runtime-isolation-shadow.yml new file mode 100644 index 00000000..6a752ac2 --- /dev/null +++ b/.github/workflows/runtime-isolation-shadow.yml @@ -0,0 +1,72 @@ +name: Runtime Isolation Shadow Fixture + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + fixed-input-shadow: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout BinancePlatform + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Resolve pinned dependency refs + id: refs + shell: bash + run: | + set -euo pipefail + qpk_ref="$(grep -Eo 'QuantPlatformKit\.git@[0-9a-f]+' pyproject.toml | head -n1 | sed 's/.*@//')" + strategies_ref="$(grep -Eo 'CryptoStrategies\.git@[0-9a-f]+' pyproject.toml | head -n1 | sed 's/.*@//')" + test -n "$qpk_ref" + test -n "$strategies_ref" + echo "qpk_ref=$qpk_ref" >> "$GITHUB_OUTPUT" + echo "strategies_ref=$strategies_ref" >> "$GITHUB_OUTPUT" + + - name: Checkout QuantPlatformKit + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: QuantStrategyLab/QuantPlatformKit + ref: ${{ steps.refs.outputs.qpk_ref }} + path: external/QuantPlatformKit + + - name: Checkout CryptoStrategies + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: QuantStrategyLab/CryptoStrategies + ref: ${{ steps.refs.outputs.strategies_ref }} + path: external/CryptoStrategies + + - name: Setup Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v6 + with: + python-version: "3.11" + + - name: Install locked runtime + run: | + set -euo pipefail + python -m pip install --disable-pip-version-check uv==0.11.6 + uv sync --frozen --no-dev + uv pip install --no-deps -e external/QuantPlatformKit -e external/CryptoStrategies + + - name: Run fixed-input no-order replay + env: + BINANCE_DRY_RUN: "true" + run: | + set -euo pipefail + uv run --no-sync python run_cycle_replay.py \ + --run-id "isolation-shadow-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --output reports/runtime_isolation_shadow.json + uv run --no-sync python scripts/assert_no_order_shadow_report.py \ + reports/runtime_isolation_shadow.json + + - name: Upload redacted shadow report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runtime-isolation-shadow-${{ github.run_id }} + path: reports/runtime_isolation_shadow.json + if-no-files-found: error + retention-days: 7 diff --git a/docs/operator_runbook.md b/docs/operator_runbook.md index 825f8d82..eb9e642d 100644 --- a/docs/operator_runbook.md +++ b/docs/operator_runbook.md @@ -80,6 +80,12 @@ The current production runtime still uses a persistent self-hosted runner. Treat The broker job has repository read permission only. Successful execution reports are transferred to a separate GitHub-hosted job, which alone receives `contents: write` for the `logs` branch. This prevents the broker credentials and repository write token from sharing one job, but it does not make a persistent runner equivalent to an ephemeral one. +The staged replacement architecture, no-order shadow proof, deployment preflight, +and rollback fence are documented in +[`runtime_isolation_migration.md`](runtime_isolation_migration.md). That plan is +informational until a separately reviewed live cutover is approved; the current +runtime remains authoritative. + ## Degraded Mode Ladder Healthy mode: diff --git a/docs/runtime_isolation_migration.md b/docs/runtime_isolation_migration.md new file mode 100644 index 00000000..cd99046b --- /dev/null +++ b/docs/runtime_isolation_migration.md @@ -0,0 +1,226 @@ +# Binance Runtime Isolation Migration + +## Status and scope + +This document defines a staged migration from the persistent GitHub self-hosted +runner to a short-lived execution boundary. It does **not** authorize a live +cutover, delete the current runner, change broker permissions, or move broker +credentials. + +The current `main.yml` workflow remains the production path until a separately +reviewed cutover change is approved. The first migration phase is limited to a +fixed-input, no-order shadow replay on a clean GitHub-hosted runner. + +## Current architecture + +The current runtime is dispatched through GitHub Actions and runs on the +persistent `binance-quant-runner` self-hosted runner. The broker job checks out +the repository, authenticates to Google Cloud through GitHub OIDC, builds or +reuses a local dependency environment, and injects the Binance credentials only +into the strategy step. + +Recent hardening already provides useful boundaries: + +- the broker job has `contents: read` and cannot write the repository; +- execution-log publication happens in a separate GitHub-hosted job without + broker credentials; +- third-party actions used by the runtime are pinned to full commit SHAs; +- the Workload Identity provider accepts this repository on `refs/heads/main`; +- `RUNTIME_TARGET_JSON` and `BINANCE_DRY_RUN` must agree or the runtime fails + closed. + +The residual risk is persistence: a compromised dependency, workflow step, or +operator action can leave files or processes on the runner and affect a later +job. GitHub explicitly recommends ephemeral self-hosted runners for autoscaling +and does not recommend persistent runners for that purpose. + +## Decision + +Use an isolated **Cloud Run Job** as the preferred target. A just-in-time +ephemeral GitHub runner is the fallback only if the runtime proves incompatible +with Cloud Run networking or execution constraints. + +Cloud Run Job is the lower-complexity fit for this personal deployment because +the runtime is a bounded command, not a long-lived HTTP service. Each execution +runs in a fresh managed task, exits when complete, and writes logs to Cloud +Logging. It also avoids operating Kubernetes solely for GitHub Actions Runner +Controller. + +### Target control and data planes + +```text +external scheduler + -> dispatch-only GitHub workflow (GitHub-hosted runner, no broker secrets) + -> GitHub OIDC / Workload Identity Federation + -> dedicated Cloud Run invoker service account + -> run one pre-created Cloud Run Job + -> dedicated job runtime service account + -> Firestore state + -> narrowly scoped Secret Manager secrets + -> Binance through a fixed outbound IP + -> append-only execution evidence +``` + +Build/deploy authority and runtime invoke authority must remain separate: + +- the normal scheduler may invoke one reviewed job but may not update its image, + environment, service account, or secrets; +- deployment uses a separate identity and a digest-pinned container image; +- the Cloud Run runtime identity may read only the required Firestore data and + named secrets; it may not administer IAM, Cloud Run, or Secret Manager; +- the GitHub workflow never receives `BINANCE_API_KEY` or + `BINANCE_API_SECRET`. + +## Current GCP gaps to close before deployment + +The `binancequant` project currently has the GitHub Workload Identity provider +and a runtime service account with Firestore access, but no reviewed Cloud Run +Job, Artifact Registry repository, runtime Secret Manager entries, dedicated +invoker identity, or fixed-egress Cloud NAT path. Those are deployment +prerequisites, not defects to paper over in workflow YAML. + +Binance IP allowlisting is an important constraint. Cloud Run uses a dynamic +outbound IP pool by default. A live job must route all outbound traffic through +Direct VPC egress (or a connector) and Cloud NAT with a reserved static IP before +that IP can be allowlisted at Binance. + +## Risk boundaries + +| Boundary | Required rule | Failure behavior | +| --- | --- | --- | +| GitHub workflow | No broker secret references; `contents: read`; OIDC only in the invoke job | Do not invoke the Cloud Run Job | +| OIDC trust | Repository and `refs/heads/main` constrained; dedicated invoker service account | Authentication denied | +| Runtime image | Immutable image digest, reviewed source revision recorded | Job remains parked | +| Cloud Run execution | One task, parallelism one, timeout bounded, automatic task retries zero | Terminal failed execution; operator alert | +| Broker secret | Secret Manager resource-level access; withdrawals disabled; static-IP allowlist | Job cannot start or broker rejects request | +| Order idempotency | Stable run/execution ID plus Firestore lease and order intent keys | Duplicate/overlapping cycle aborts | +| Evidence | Append-only report with source SHA, image digest, mode, and executed/suppressed counts | Missing evidence is `PARKED`, never `READY` | +| Live authority | Existing risk envelope only; no automatic expansion of assets, leverage, or capital | Human decision required | + +Cloud Run's default task retry count is not appropriate for order execution: a +platform retry could repeat a partially completed cycle. The live job must use +`maxRetries: 0`; application-level recovery must first reconcile Firestore, +broker orders, fills, and the stable execution ID. + +## Staged migration + +### Phase 1: fixed-input isolation shadow (this change) + +- Run `run_cycle_replay.py` on `ubuntu-latest` using committed fixtures. +- Do not request OIDC and do not reference any GitHub environment or secret. +- Assert `dry_run=true`, `executed_call_count=0`, and at least one suppressed + side effect. +- Upload only the redacted structured report. + +Passing this phase proves that the strategy package can execute in a clean, +short-lived environment. It does not validate Google Cloud, Binance networking, +or live readiness. + +### Phase 2: Cloud Run fixture shadow + +- Create a shadow-only container entrypoint that runs the same committed fixture. +- Deploy a separate `binance-runtime-shadow` job with no broker secrets. +- Set one task, parallelism one, zero retries, and a bounded timeout. +- Invoke manually through a dedicated GitHub OIDC invoker identity. +- Compare the Cloud Run report digest with the GitHub-hosted Phase 1 report. + +### Phase 3: read-only forward shadow + +- Add Firestore read/write permissions required for shadow state only. +- If live market/account observations are required, use a separate Binance API + key without order or withdrawal capability. +- Add Secret Manager resource-level access only for that read-only key. +- Establish Direct VPC egress, Cloud NAT, and a reserved outbound IP. +- Run alongside the old runtime without sending orders and reconcile decisions. + +Creating or changing a Binance API key is an explicit operator action and is not +part of the automated migration. + +### Phase 4: limited live canary + +This phase requires a separate PR and human approval. Before it starts: + +- the Cloud Run job must use the existing live risk envelope or a smaller one; +- the old scheduler must be fenced so only one execution path can place orders; +- duplicate-order, partial-fill, timeout, Firestore outage, and Binance outage + recovery must be rehearsed; +- the execution image digest, job revision, service accounts, IAM bindings, + secret versions, static IP, and rollback owner must be recorded. + +### Phase 5: cutover and retirement + +Switch the scheduler only after the canary evidence is accepted. Keep the old +runner installed but disabled during the observation window. Remove it and +rotate credentials only in a later, separately authorized cleanup. + +## Deployment preflight + +All items below are mandatory before Phase 2 or later: + +- [ ] Phase 1 fixture report is deterministic and records zero executed calls. +- [ ] Container entrypoint defaults to no-order; live requires an explicit, + reviewed runtime target. +- [ ] Image is referenced by digest, not a mutable tag. +- [ ] Shadow and live are different Cloud Run Jobs and identities. +- [ ] GitHub invoker service account has `roles/run.invoker` only on the intended + job and cannot update the job. +- [ ] Job runtime service account is not the default Compute service account. +- [ ] Runtime service account has no project-wide Owner, Editor, IAM, Cloud Run + admin, or Secret Manager admin role. +- [ ] Broker secrets are absent from GitHub and plaintext environment variables; + Secret Manager access is limited to named secrets. +- [ ] Live secret versions are pinned or rotation behavior is explicitly tested. +- [ ] Task count and parallelism are one; task retries are zero; timeout is + shorter than the scheduling interval. +- [ ] Firestore lease prevents overlapping old/new runtime cycles. +- [ ] Static outbound IP is observed from the job and allowlisted at Binance. +- [ ] Binance key has withdrawals disabled and the smallest required trade scope. +- [ ] Execution report reaches durable storage without granting the runtime job + repository write permission. +- [ ] Alerts cover start failure, timeout, missing terminal report, reconciliation + mismatch, and circuit-breaker activation. +- [ ] Old live path remains unchanged and available for rollback until cutover is + separately approved. + +## Rollback + +Before live canary, rollback means deleting or disabling only the shadow trigger; +the existing runtime is unaffected. + +During live canary or cutover: + +1. Stop the new scheduler/invoker and wait for the current Cloud Run execution to + reach a terminal state. +2. Reconcile broker open orders, fills, balances, Firestore lease, and the last + durable execution report. Do not start the old path while ownership is + ambiguous. +3. Mark the Cloud Run job parked and revoke its invoker binding. Do not delete + evidence or secret versions during incident response. +4. Re-enable the old dispatch path only after the execution lease is cleared and + the broker state matches the expected portfolio. +5. Record the rollback reason and require a new canary decision before retrying. + +Rollback must never run both paths concurrently. Restoring the old runner is not +permission to bypass a triggered circuit breaker or expand the live envelope. + +## Rejected alternatives + +- Reusing the persistent VPS but deleting the workspace after each run: cleanup + cannot reliably remove a compromised process, runner service, or host secret. +- Moving broker secrets to a GitHub-hosted runner: the runner is clean, but its + outbound IP is unsuitable for a stable Binance allowlist and GitHub would still + become the broker-secret boundary. +- Deploying Actions Runner Controller: production-grade, but Kubernetes adds an + unnecessary control plane for one bounded personal runtime. +- Switching live directly to Cloud Run: it skips idempotency, networking, + reconciliation, and rollback evidence. + +## Official references + +- [GitHub self-hosted runners reference](https://docs.github.com/en/actions/reference/runners/self-hosted-runners) +- [GitHub secure use reference](https://docs.github.com/en/actions/reference/security/secure-use) +- [Google Cloud Run jobs](https://cloud.google.com/run/docs/create-jobs) +- [Execute Cloud Run jobs](https://cloud.google.com/run/docs/execute/jobs) +- [Cloud Run Job secrets](https://cloud.google.com/run/docs/configuring/jobs/secrets) +- [Cloud Run static outbound IP](https://cloud.google.com/run/docs/configuring/static-outbound-ip) +- [Workload Identity Federation for deployment pipelines](https://cloud.google.com/iam/docs/workload-identity-federation-with-deployment-pipelines) diff --git a/scripts/assert_no_order_shadow_report.py b/scripts/assert_no_order_shadow_report.py new file mode 100644 index 00000000..0d644d39 --- /dev/null +++ b/scripts/assert_no_order_shadow_report.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def validate_no_order_report(report: Any) -> list[str]: + errors: list[str] = [] + if not isinstance(report, dict): + return ["report must be a JSON object"] + + if report.get("status") != "ok": + errors.append("report status must be ok") + if report.get("dry_run") is not True: + errors.append("report dry_run must be true") + + summary = report.get("side_effect_summary") + if not isinstance(summary, dict): + errors.append("side_effect_summary must be an object") + return errors + + if summary.get("executed_call_count") != 0: + errors.append("executed_call_count must be zero") + suppressed = summary.get("suppressed_call_count") + if not isinstance(suppressed, int) or isinstance(suppressed, bool) or suppressed < 1: + errors.append("suppressed_call_count must be a positive integer") + return errors + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Fail unless a replay report proves no-order shadow execution.") + parser.add_argument("report", type=Path, help="Path to the structured replay report.") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + report = json.loads(args.report.read_text(encoding="utf-8")) + errors = validate_no_order_report(report) + if errors: + raise SystemExit("No-order shadow report rejected: " + "; ".join(errors)) + print("No-order shadow report accepted: dry_run=true executed_call_count=0") + + +if __name__ == "__main__": + main() diff --git a/tests/test_runtime_isolation_shadow.py b/tests/test_runtime_isolation_shadow.py new file mode 100644 index 00000000..c9ea0e11 --- /dev/null +++ b/tests/test_runtime_isolation_shadow.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import importlib.util +import json +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "runtime-isolation-shadow.yml" +VALIDATOR = ROOT / "scripts" / "assert_no_order_shadow_report.py" +FULL_SHA_ACTION = re.compile(r"(?:-\s+)?uses:\s+[^\s@]+@[0-9a-f]{40}(?:\s+#\s+v\d+)?$") + + +def load_validator(): + spec = importlib.util.spec_from_file_location("assert_no_order_shadow_report", VALIDATOR) + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load shadow report validator") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class RuntimeIsolationShadowWorkflowTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.workflow = WORKFLOW.read_text(encoding="utf-8") + cls.validator = load_validator() + + def test_shadow_workflow_is_manual_ephemeral_and_has_no_secret_capability(self) -> None: + workflow = self.workflow + + self.assertIn("workflow_dispatch:", workflow) + self.assertNotIn("pull_request_target:", workflow) + self.assertIn("runs-on: ubuntu-latest", workflow) + self.assertNotIn("runs-on: self-hosted", workflow) + self.assertIn("contents: read", workflow) + self.assertNotIn("id-token: write", workflow) + self.assertNotIn("environment:", workflow) + self.assertNotIn("secrets.", workflow) + self.assertNotIn("BINANCE_API_KEY", workflow) + self.assertNotIn("BINANCE_API_SECRET", workflow) + + def test_shadow_workflow_uses_fixture_replay_and_pins_actions(self) -> None: + workflow = self.workflow + action_lines = [line.strip() for line in workflow.splitlines() if "uses:" in line] + + self.assertTrue(action_lines) + self.assertTrue(all(FULL_SHA_ACTION.fullmatch(line) for line in action_lines)) + self.assertIn("run_cycle_replay.py", workflow) + self.assertIn("assert_no_order_shadow_report.py", workflow) + self.assertIn('BINANCE_DRY_RUN: "true"', workflow) + self.assertNotIn("python main.py", workflow) + + def test_validator_accepts_only_dry_run_with_zero_executed_calls(self) -> None: + accepted = { + "status": "ok", + "dry_run": True, + "side_effect_summary": {"executed_call_count": 0, "suppressed_call_count": 3}, + } + self.assertEqual(self.validator.validate_no_order_report(accepted), []) + + for field, value in ( + ("dry_run", False), + ("status", "aborted"), + ): + rejected = json.loads(json.dumps(accepted)) + rejected[field] = value + self.assertTrue(self.validator.validate_no_order_report(rejected)) + + executed = json.loads(json.dumps(accepted)) + executed["side_effect_summary"]["executed_call_count"] = 1 + self.assertTrue(self.validator.validate_no_order_report(executed)) + + +if __name__ == "__main__": + unittest.main()