Skip to content

Commit a831e6f

Browse files
MinecraftFunsMinecraftFuns
committed
feat: enhance e2e deployment scripts and add production deployment workflow
1 parent 353beec commit a831e6f

4 files changed

Lines changed: 513 additions & 49 deletions

File tree

‎.github/workflows/deploy-prod.yml‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
name: deploy-prod
2+
3+
# Real production deployment from CI.
4+
#
5+
# Manually triggered (workflow_dispatch). Pushes the current commit's
6+
# closure to live lamacloud hosts using Colmena over SSH, authenticated
7+
# with the `sayo` user and the ed25519 private key stored as a GitHub
8+
# Actions encrypted secret.
9+
#
10+
# REQUIRED SECRETS (configure in repo Settings -> Secrets and variables
11+
# -> Actions). See `docs/CI_DEPLOY.md` for the exact format of each
12+
# secret -- do not paste anything else here:
13+
#
14+
# LAMACLOUD_SAYO_PRIVATE_KEY - PEM-formatted ed25519 private key for `sayo`
15+
# LAMACLOUD_SSH_KNOWN_HOSTS - known_hosts entries for every target host
16+
#
17+
# Optional inputs let the operator override the Colmena selector (`--on`)
18+
# and the activation goal (`switch`, `boot`, `dry-activate`, `test`).
19+
#
20+
# Safety guard: this workflow refuses to run on a dirty working tree --
21+
# the commit being deployed must exist verbatim in git.
22+
23+
on:
24+
workflow_dispatch:
25+
inputs:
26+
selector:
27+
description: "Colmena --on selector (host name, glob, or @tag). Default = @lamacloud (every host)."
28+
required: false
29+
default: '@lamacloud'
30+
type: string
31+
goal:
32+
description: "Activation goal."
33+
required: true
34+
default: 'switch'
35+
type: choice
36+
options:
37+
- switch
38+
- boot
39+
- test
40+
- dry-activate
41+
reboot:
42+
description: "Reboot nodes after activation."
43+
required: false
44+
default: false
45+
type: boolean
46+
confirm:
47+
description: "Type 'I UNDERSTAND' to acknowledge this is a production deploy."
48+
required: true
49+
type: string
50+
51+
concurrency:
52+
# Serialize prod deploys -- never allow two simultaneous deploys.
53+
group: deploy-prod-global
54+
cancel-in-progress: false
55+
56+
jobs:
57+
deploy:
58+
name: colmena apply ${{ inputs.goal }} --on ${{ inputs.selector }}
59+
runs-on: ubuntu-24.04
60+
timeout-minutes: 60
61+
62+
# Environment-scoped secrets and required reviewers. Create a GitHub
63+
# environment named `production` in repo Settings and protect it with
64+
# required reviewers if you want manual approval on top of the inputs.
65+
environment: production
66+
67+
steps:
68+
- name: Confirm operator intent
69+
if: ${{ inputs.confirm != 'I UNDERSTAND' }}
70+
run: |
71+
echo "[FAIL] deploy-prod/confirm: operator did not type 'I UNDERSTAND'"
72+
exit 1
73+
74+
- uses: actions/checkout@v4
75+
76+
- name: Refuse dirty working tree
77+
run: |
78+
if [[ -n "$(git status --porcelain)" ]]; then
79+
echo "[FAIL] deploy-prod/clean-tree: working tree is dirty after checkout"
80+
git status --short
81+
exit 1
82+
fi
83+
echo "[OK] deploy-prod/clean-tree"
84+
85+
- name: Install Nix
86+
uses: cachix/install-nix-action@v27
87+
with:
88+
extra_nix_config: |
89+
experimental-features = nix-command flakes
90+
accept-flake-config = true
91+
92+
- name: Install Colmena (from pinned flake input)
93+
run: |
94+
echo "==> installing colmena built from the pinned flake input"
95+
nix profile install .#colmena
96+
which colmena
97+
colmena --version
98+
99+
# ----------------------------------------------------------- secrets
100+
- name: Materialise SSH credentials
101+
env:
102+
SAYO_PRIVATE_KEY: ${{ secrets.LAMACLOUD_SAYO_PRIVATE_KEY }}
103+
SSH_KNOWN_HOSTS: ${{ secrets.LAMACLOUD_SSH_KNOWN_HOSTS }}
104+
run: |
105+
set -eu
106+
if [[ -z "${SAYO_PRIVATE_KEY:-}" ]]; then
107+
echo "[FAIL] deploy-prod/secrets: LAMACLOUD_SAYO_PRIVATE_KEY is unset"
108+
echo " See docs/CI_DEPLOY.md for the required format."
109+
exit 1
110+
fi
111+
if [[ -z "${SSH_KNOWN_HOSTS:-}" ]]; then
112+
echo "[FAIL] deploy-prod/secrets: LAMACLOUD_SSH_KNOWN_HOSTS is unset"
113+
echo " See docs/CI_DEPLOY.md for the required format."
114+
exit 1
115+
fi
116+
117+
mkdir -p ~/.ssh
118+
chmod 700 ~/.ssh
119+
120+
# Write the key. The secret MUST end with a trailing newline (the
121+
# documented format adds it via `cat key.pem | gh secret set`).
122+
printf '%s\n' "$SAYO_PRIVATE_KEY" > ~/.ssh/sayo
123+
chmod 600 ~/.ssh/sayo
124+
125+
printf '%s\n' "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
126+
chmod 644 ~/.ssh/known_hosts
127+
128+
cat >~/.ssh/lamacloud_config <<'EOF'
129+
Host *
130+
User sayo
131+
IdentityFile ~/.ssh/sayo
132+
IdentitiesOnly yes
133+
UserKnownHostsFile ~/.ssh/known_hosts
134+
StrictHostKeyChecking yes
135+
ServerAliveInterval 30
136+
ServerAliveCountMax 4
137+
EOF
138+
chmod 600 ~/.ssh/lamacloud_config
139+
140+
echo "[OK] deploy-prod/secrets: SSH credentials installed"
141+
142+
# ----------------------------------------------------------- verify
143+
- name: Repository integrity check
144+
run: ./foundation/scripts/lamacloud check --strict
145+
146+
- name: Build hive without pushing (sanity check)
147+
env:
148+
SSH_CONFIG_FILE: /home/runner/.ssh/lamacloud_config
149+
run: |
150+
echo "==> colmena build --on '${{ inputs.selector }}'"
151+
colmena build --on '${{ inputs.selector }}' --show-trace
152+
153+
# ----------------------------------------------------------- deploy
154+
- name: colmena apply
155+
env:
156+
SSH_CONFIG_FILE: /home/runner/.ssh/lamacloud_config
157+
run: |
158+
set -e
159+
args=(apply '${{ inputs.goal }}' --on '${{ inputs.selector }}' --verbose)
160+
if [[ '${{ inputs.reboot }}' == 'true' ]]; then
161+
args+=(--reboot)
162+
fi
163+
echo "==> colmena ${args[*]}"
164+
colmena "${args[@]}"
165+
166+
- name: Deployment summary
167+
if: always()
168+
run: |
169+
echo "==> deploy-prod outcome: ${{ job.status }}"
170+
echo " selector: ${{ inputs.selector }}"
171+
echo " goal: ${{ inputs.goal }}"
172+
echo " reboot: ${{ inputs.reboot }}"
173+
echo " commit: ${{ github.sha }}"

‎docs/CI_DEPLOY.md‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
# CI Deployment Guide
2+
3+
This document describes how the GitHub Actions workflows interact with
4+
real lamacloud infrastructure, what secrets the production deploy
5+
workflow needs, and the exact format each secret must take.
6+
7+
If a secret is missing, malformed, or contains the wrong content, the
8+
deploy workflow exits early with a `[FAIL] deploy-prod/<stage>: ...`
9+
line — there is no silent failure path.
10+
11+
---
12+
13+
## Workflows at a glance
14+
15+
| Workflow | Trigger | Touches real infra? |
16+
| ------------------------------------- | ----------------------------- | ------------------- |
17+
| `.github/workflows/ci.yml` | push / pull_request | No |
18+
| `.github/workflows/e2e-deploy.yml` | push / pull_request | No (QEMU VM only) |
19+
| `.github/workflows/deploy-prod.yml` | manual (`workflow_dispatch`) | **YES** |
20+
21+
Only `deploy-prod.yml` reads any of the secrets below. The other two
22+
workflows use the committed `tests/fixtures/` material and a throwaway
23+
VM, so they require **zero secret configuration**.
24+
25+
---
26+
27+
## Required GitHub Actions secrets
28+
29+
Configure these in the repository's
30+
**Settings → Environments → `production` → Add secret**. They MUST live
31+
in the `production` environment (not in the repo-wide secrets pane) so
32+
that GitHub's environment protection rules (required reviewers,
33+
deployment branch restrictions) apply automatically.
34+
35+
### 1. `LAMACLOUD_SAYO_PRIVATE_KEY` — *(required)*
36+
37+
The ed25519 **private** key matching the `sayo` public key recorded in
38+
`sayo.json`. Colmena uses this to SSH into every host as user `sayo`.
39+
40+
**Format:** OpenSSH PEM, **including the BEGIN/END lines and a trailing
41+
newline**. The exact bytes you would get from:
42+
43+
```bash
44+
ssh-keygen -t ed25519 -N '' -C 'sayo@lamacloud-ci' -f /tmp/sayo
45+
cat /tmp/sayo # <-- this is the secret value
46+
```
47+
48+
**Setting it:**
49+
50+
```bash
51+
gh secret set LAMACLOUD_SAYO_PRIVATE_KEY \
52+
--env production \
53+
--body "$(cat /tmp/sayo)"
54+
```
55+
56+
**Validation:** the workflow's "Materialise SSH credentials" step
57+
writes the secret to `~/.ssh/sayo` with mode `0600` and immediately
58+
fails if the variable is empty. A malformed key surfaces as an SSH
59+
error in the next step (e.g. `Load key "/home/runner/.ssh/sayo": invalid format`).
60+
61+
**Public-key counterpart:** the matching public key must already appear
62+
verbatim in `sayo.json` at the repo root, under `publicKey.keys[]`.
63+
Regenerate `sayo.json` with `lamacloud creds new --sayo` after rotating
64+
the keypair, commit it, and rotate this secret in the same PR.
65+
66+
### 2. `LAMACLOUD_SSH_KNOWN_HOSTS` — *(required)*
67+
68+
A `known_hosts` file containing one or more entries for every target
69+
host the workflow will deploy to. The deploy workflow sets
70+
`StrictHostKeyChecking yes`, so an unknown host fingerprint is a hard
71+
failure (this is intentional — it eliminates the MITM-attack window
72+
that `accept-new` opens).
73+
74+
**Format:** one `known_hosts` line per host, concatenated. Example
75+
contents:
76+
77+
```
78+
hk01.lamacloud.onlylama.fans ssh-ed25519 AAAAC3Nz...EXAMPLE
79+
hk01.lamacloud.onlylama.fans ecdsa-sha2-nistp256 AAAAE2Vj...EXAMPLE
80+
[hk01.lamacloud.onlylama.fans]:19312 ssh-ed25519 AAAAC3Nz...EXAMPLE
81+
```
82+
83+
Note the `[host]:port` form is needed for hosts whose
84+
`lamacloud.json` entry specifies a non-default port (e.g. hk01 uses 19312).
85+
86+
**Generating the entries:**
87+
88+
```bash
89+
# Repeat for every (host, port) tuple in lamacloud.json
90+
ssh-keyscan -p 19312 hk01.lamacloud.onlylama.fans >> /tmp/known_hosts
91+
ssh-keyscan hk01.lamacloud.onlylama.fans >> /tmp/known_hosts
92+
# ... etc
93+
94+
gh secret set LAMACLOUD_SSH_KNOWN_HOSTS \
95+
--env production \
96+
--body "$(cat /tmp/known_hosts)"
97+
```
98+
99+
**Rotation:** whenever a target host's SSH host key changes (fresh
100+
install, key rotation), regenerate this secret. Forgetting to do so is
101+
a *safe failure* — the workflow refuses to connect rather than silently
102+
trusting the new key.
103+
104+
---
105+
106+
## Optional GitHub Actions configuration
107+
108+
### Environment protection rules
109+
110+
Open **Settings → Environments → `production`** and enable:
111+
112+
- **Required reviewers:** at least one project owner. This forces a
113+
human to approve every `deploy-prod.yml` run before it touches real
114+
infra, even if someone has push access to `main`.
115+
- **Deployment branches:** restrict to `main` only so accidental
116+
feature-branch deploys are impossible.
117+
- **Wait timer:** optional 1–5 minute delay to give reviewers time to
118+
cancel an erroneous dispatch.
119+
120+
### Audit log
121+
122+
GitHub's audit log records every `workflow_dispatch` event including
123+
the inputs. Combined with the `deploy-prod.yml` "Deployment summary"
124+
step (which prints commit SHA, selector, goal, reboot flag) this gives
125+
a complete who-deployed-what-when trail without extra tooling.
126+
127+
---
128+
129+
## Running a production deploy
130+
131+
1. Go to **Actions → deploy-prod → Run workflow**.
132+
2. Fill in the inputs:
133+
- `selector` — Colmena `--on` selector. Defaults to `@lamacloud` (every host). Use `lc-entrypoint-hk01` to target one host, `'@infra-lax'` to target a tag, etc.
134+
- `goal` — `switch` (apply immediately), `boot` (apply on next reboot), `test` (apply without registering as default), or `dry-activate` (no-op).
135+
- `reboot` — reboot every node after activation.
136+
- `confirm` — type `I UNDERSTAND` literally. Any other value aborts the run.
137+
3. Click **Run workflow**. The job will block waiting for a reviewer if you configured required reviewers.
138+
4. Approve. Watch the Actions log for `[OK]` / `[FAIL]` lines.
139+
140+
---
141+
142+
## What the workflow does step by step
143+
144+
1. **`deploy-prod/confirm`** — refuse to proceed unless `confirm == "I UNDERSTAND"`.
145+
2. **`deploy-prod/clean-tree`** — `git status --porcelain` must be empty.
146+
3. Install Nix + Colmena (from our pinned flake input — never `nixpkgs#colmena`, so the binary version matches the hive evaluator exactly).
147+
4. **`deploy-prod/secrets`** — materialise `~/.ssh/sayo`, `~/.ssh/known_hosts`, and `~/.ssh/lamacloud_config` with the right modes. Fail with a clear error if either secret is missing.
148+
5. **Repo integrity** — `lamacloud check --strict` validates that every host has `creds.json` and the sayo creds are syntactically intact.
149+
6. **Hive sanity build** — `colmena build --on <selector>` ensures every selected closure builds locally before any push.
150+
7. **`colmena apply <goal>`** — push closures, activate, optionally reboot.
151+
8. **Deployment summary** — always-on summary line printing selector / goal / reboot / commit SHA.
152+
153+
---
154+
155+
## Failure modes & remediation
156+
157+
| Symptom in log | Cause | Fix |
158+
| ---------------------------------------------------------------------------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------ |
159+
| `[FAIL] deploy-prod/confirm: operator did not type 'I UNDERSTAND'` | Operator typo in dispatch form. | Re-dispatch with the exact string `I UNDERSTAND`. |
160+
| `[FAIL] deploy-prod/clean-tree: working tree is dirty after checkout` | A workflow earlier in the run wrote files outside `~/...`. | Investigate; clean trees are a deployment invariant. |
161+
| `[FAIL] deploy-prod/secrets: LAMACLOUD_SAYO_PRIVATE_KEY is unset` | Secret missing or stored at repo level instead of env. | Add to `production` environment. See §1 above. |
162+
| `Load key "/home/runner/.ssh/sayo": invalid format` | Secret value is not a PEM key, or lost its newline. | Re-set with `--body "$(cat key.pem)"` (NOT `--body "$(cat key.pem | base64)"`). |
163+
| `Host key verification failed` | `LAMACLOUD_SSH_KNOWN_HOSTS` missing the relevant host:port. | `ssh-keyscan -p <port> <host>` then re-set the secret. |
164+
| `[FAIL] deploy-prod/build` | A host's closure no longer builds on x86_64 / aarch64. | Reproduce locally with `lamacloud build <host>`. |
165+
| `colmena ... activation failed` | New configuration is invalid on the target. | Re-dispatch with `goal = dry-activate` or `test` to diagnose without bricking. |
166+
167+
---
168+
169+
## Why we never use `accept-new` for known_hosts
170+
171+
The first SSH connection to a host with `StrictHostKeyChecking=accept-new`
172+
silently learns whatever key the server presents. If an attacker is
173+
in-path between the runner and the target (e.g. compromised DNS, BGP
174+
hijack on the runner's egress), they can stand up an interception proxy
175+
that the workflow will trust permanently. By requiring a pre-populated
176+
`known_hosts`, we move that trust decision into a human-reviewed PR
177+
that updates `LAMACLOUD_SSH_KNOWN_HOSTS`. The cost is one extra
178+
`ssh-keyscan` when a host's key legitimately rotates; the benefit is
179+
eliminating an entire class of supply-chain attack.

0 commit comments

Comments
 (0)