Skip to content

Commit 0fa3005

Browse files
karngyanclaude
andauthored
feat: daemon restarts reattach live holder sessions
The boot sequence gains a first pass: every live holder under the holders root is dialed and its session rebuilt — same id, same ring, same running child. Dead holder dirs are swept, and the revive pass that follows only touches snapshots for sessions that are not already back. Holders answer SIGTERM (logout, reboot) by writing exactly the revival snapshot the daemon used to, so the reboot story is unchanged. Service units harden to match: systemd gains KillMode=process so a unit stop cannot reap the holders, launchd gains AbandonProcessGroup, and flue restart / flue update converge the installed unit file before bouncing so existing installs pick the settings up on their next update. An end-to-end test builds the real binary, spawns through a real daemon, SIGTERMs it, starts a successor, and finds the same session: same id, same child pid, scrollback intact. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 627fc34 commit 0fa3005

14 files changed

Lines changed: 659 additions & 28 deletions

File tree

cmd/flue/main.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,17 @@ func cmdServe(args []string) error {
198198
}
199199
}
200200
}
201+
// First, the sessions that never died: every live holder under the
202+
// holders root gets its registry entry back — same id, same ring, same
203+
// running process. This is the pass that makes a daemon restart a
204+
// non-event for sessions. Dead holder dirs (a reboot, a self-reap) are
205+
// swept here; what they saved for revival is picked up just below.
206+
if cfgDir, err := config.Dir(); err == nil {
207+
reattached, swept := session.ReattachHolders(reg, filepath.Join(cfgDir, "holders"))
208+
if reattached > 0 || swept > 0 {
209+
logger.Info("holder reattach", "reattached", reattached, "swept", swept)
210+
}
211+
}
201212
// Bring back what the previous daemon saved on its way out: each session
202213
// returns under its old id with its scrollback and a fresh shell in its
203214
// directory. Failures are reported and skipped — revival is a courtesy,
@@ -211,6 +222,15 @@ func cmdServe(args []string) error {
211222
// wedge: LoadSnapshots ages out a snapshot that keeps failing (see
212223
// maxReviveAttempts) instead of retrying it forever.
213224
for _, snap := range session.LoadSnapshots(stateDir) {
225+
if _, live := reg.Get(snap.ID); live {
226+
// The session the snapshot describes is already here, alive,
227+
// reattached above — a snapshot written by a holder whose
228+
// SIGTERM turned out not to be the end (a logout that came
229+
// back, a canceled shutdown). The live session wins and the
230+
// stale record has nothing left to say.
231+
session.ClearSnapshot(stateDir, snap.ID)
232+
continue
233+
}
214234
if _, err := reg.Revive(snap); err != nil {
215235
fmt.Fprintf(os.Stderr, "flue: could not revive session %s: %v\n", snap.ID, err)
216236
session.RecordReviveFailure(stateDir, snap)
@@ -1574,6 +1594,14 @@ func runRestart(w io.Writer, wait time.Duration) error {
15741594
if !st.Installed {
15751595
return errors.New("login service is not installed; run \"flue enable\" to install and start it")
15761596
}
1597+
// Converge the unit file first, so the restart boots the service under
1598+
// the current template — this is how a unit change (the KillMode and
1599+
// AbandonProcessGroup settings holder sessions depend on) reaches an
1600+
// install from before it. Best-effort: a refresh that fails leaves
1601+
// exactly the restart the user asked for.
1602+
if ur, ok := mgr.(service.UnitRefresher); ok {
1603+
_ = ur.RefreshUnit()
1604+
}
15771605
if err := mgr.Restart(); err != nil {
15781606
return err
15791607
}

cmd/flue/restart_e2e_test.go

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"net"
8+
"net/http"
9+
"os"
10+
"os/exec"
11+
"path/filepath"
12+
"regexp"
13+
"strconv"
14+
"strings"
15+
"syscall"
16+
"testing"
17+
"time"
18+
19+
"github.com/coder/websocket"
20+
)
21+
22+
// TestSessionsSurviveDaemonRestart is the whole feature, end to end, against
23+
// the real binary: spawn a session through a real daemon, SIGTERM the
24+
// daemon, start a new one, and find the same session — same id, same child
25+
// process, scrollback intact. Skipped under -short because it builds the
26+
// binary and runs two daemons.
27+
func TestSessionsSurviveDaemonRestart(t *testing.T) {
28+
if testing.Short() {
29+
t.Skip("builds the flue binary and runs real daemons")
30+
}
31+
32+
// The config dir must sit shallow enough for the holder socket path; a
33+
// nested t.TempDir can blow the platform's sockaddr limit.
34+
xdg, err := os.MkdirTemp("", "fe")
35+
if err != nil {
36+
t.Fatalf("MkdirTemp: %v", err)
37+
}
38+
t.Cleanup(func() { os.RemoveAll(xdg) })
39+
40+
// Not xdg/flue: config.Dir wants to create a directory by that name.
41+
bin := filepath.Join(xdg, "bin", "flue")
42+
if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil {
43+
t.Fatal(err)
44+
}
45+
build := exec.Command("go", "build", "-o", bin, ".")
46+
if out, err := build.CombinedOutput(); err != nil {
47+
t.Fatalf("go build: %v\n%s", err, out)
48+
}
49+
50+
port := freePort(t)
51+
env := append(os.Environ(), "XDG_CONFIG_HOME="+xdg)
52+
53+
daemon1 := startDaemon(t, bin, port, env)
54+
token := readToken(t, xdg)
55+
56+
// A child that names its own pid is the witness: the same line after
57+
// the restart proves the same process, not a lookalike.
58+
ws := dialWS(t, port, token)
59+
send(t, ws, map[string]any{
60+
"type": "spawn", "reqId": 1, "cols": 80, "rows": 24, "cwd": "/",
61+
"cmd": []string{"/bin/sh", "-c", `echo "MARK-$$"; exec sleep 300`},
62+
})
63+
sessionID := awaitAttached(t, ws)
64+
backlog := awaitMark(t, ws)
65+
childPid := pidFromMark(t, backlog)
66+
ws.Close(websocket.StatusNormalClosure, "")
67+
68+
// The restart: graceful SIGTERM, as a service manager would deliver it.
69+
if err := daemon1.Process.Signal(syscall.SIGTERM); err != nil {
70+
t.Fatalf("SIGTERM daemon: %v", err)
71+
}
72+
if _, err := daemon1.Process.Wait(); err != nil {
73+
t.Fatalf("daemon did not exit: %v", err)
74+
}
75+
76+
daemon2 := startDaemon(t, bin, port, env)
77+
defer func() {
78+
_ = daemon2.Process.Signal(syscall.SIGTERM)
79+
_, _ = daemon2.Process.Wait()
80+
}()
81+
82+
infos, err := fetchSessions(port, token)
83+
if err != nil {
84+
t.Fatalf("fetchSessions: %v", err)
85+
}
86+
var found bool
87+
for _, s := range infos {
88+
if s.ID == sessionID {
89+
found = true
90+
if s.State != "running" {
91+
t.Fatalf("session %s State = %q after restart, want running", s.ID, s.State)
92+
}
93+
}
94+
}
95+
if !found {
96+
t.Fatalf("session %s is gone after the restart; got %+v", sessionID, infos)
97+
}
98+
99+
if err := syscall.Kill(childPid, 0); err != nil {
100+
t.Fatalf("child %d is dead after the restart: %v", childPid, err)
101+
}
102+
103+
ws2 := dialWS(t, port, token)
104+
defer ws2.Close(websocket.StatusNormalClosure, "")
105+
send(t, ws2, map[string]any{"type": "attach", "id": sessionID, "lastSeq": 0, "reqId": 2})
106+
backlog2 := awaitMark(t, ws2)
107+
if pid2 := pidFromMark(t, backlog2); pid2 != childPid {
108+
t.Fatalf("marker pid changed across restart: %d then %d", childPid, pid2)
109+
}
110+
111+
// Retire the session so nothing outlives the test.
112+
send(t, ws2, map[string]any{"type": "close", "id": sessionID})
113+
deadline := time.Now().Add(5 * time.Second)
114+
for syscall.Kill(childPid, 0) == nil {
115+
if time.Now().After(deadline) {
116+
t.Fatalf("child %d survived the close", childPid)
117+
}
118+
time.Sleep(20 * time.Millisecond)
119+
}
120+
}
121+
122+
func freePort(t *testing.T) int {
123+
t.Helper()
124+
ln, err := net.Listen("tcp", "127.0.0.1:0")
125+
if err != nil {
126+
t.Fatalf("Listen: %v", err)
127+
}
128+
port := ln.Addr().(*net.TCPAddr).Port
129+
ln.Close()
130+
return port
131+
}
132+
133+
func startDaemon(t *testing.T, bin string, port int, env []string) *exec.Cmd {
134+
t.Helper()
135+
cmd := exec.Command(bin, "serve", "--port", strconv.Itoa(port))
136+
cmd.Env = env
137+
cmd.Stdout = os.Stderr
138+
cmd.Stderr = os.Stderr
139+
if err := cmd.Start(); err != nil {
140+
t.Fatalf("start daemon: %v", err)
141+
}
142+
deadline := time.Now().Add(10 * time.Second)
143+
for {
144+
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 200*time.Millisecond)
145+
if err == nil {
146+
conn.Close()
147+
return cmd
148+
}
149+
if time.Now().After(deadline) {
150+
t.Fatal("daemon never bound its port")
151+
}
152+
time.Sleep(50 * time.Millisecond)
153+
}
154+
}
155+
156+
func readToken(t *testing.T, xdg string) string {
157+
t.Helper()
158+
b, err := os.ReadFile(filepath.Join(xdg, "flue", "token"))
159+
if err != nil {
160+
t.Fatalf("read token: %v", err)
161+
}
162+
return strings.TrimSpace(string(b))
163+
}
164+
165+
func dialWS(t *testing.T, port int, token string) *websocket.Conn {
166+
t.Helper()
167+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
168+
t.Cleanup(cancel)
169+
ws, _, err := websocket.Dial(ctx, fmt.Sprintf("ws://127.0.0.1:%d/ws", port), &websocket.DialOptions{
170+
HTTPHeader: http.Header{"X-Flue-Token": []string{token}},
171+
})
172+
if err != nil {
173+
t.Fatalf("ws dial: %v", err)
174+
}
175+
ws.SetReadLimit(1 << 22)
176+
return ws
177+
}
178+
179+
func send(t *testing.T, ws *websocket.Conn, msg map[string]any) {
180+
t.Helper()
181+
b, _ := json.Marshal(msg)
182+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
183+
defer cancel()
184+
if err := ws.Write(ctx, websocket.MessageText, b); err != nil {
185+
t.Fatalf("ws write %v: %v", msg["type"], err)
186+
}
187+
}
188+
189+
// awaitAttached reads until the attached control message and returns the
190+
// session id it names.
191+
func awaitAttached(t *testing.T, ws *websocket.Conn) string {
192+
t.Helper()
193+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
194+
defer cancel()
195+
for {
196+
typ, b, err := ws.Read(ctx)
197+
if err != nil {
198+
t.Fatalf("ws read awaiting attached: %v", err)
199+
}
200+
if typ != websocket.MessageText {
201+
continue
202+
}
203+
var m struct {
204+
Type string `json:"type"`
205+
ID string `json:"id"`
206+
Msg string `json:"msg"`
207+
}
208+
if json.Unmarshal(b, &m) != nil {
209+
continue
210+
}
211+
if m.Type == "error" {
212+
t.Fatalf("daemon answered error: %s", m.Msg)
213+
}
214+
if m.Type == "attached" {
215+
return m.ID
216+
}
217+
}
218+
}
219+
220+
// awaitMark accumulates stream output until the MARK-<pid> line is whole.
221+
// Output frames are binary; everything else is control chatter to skip.
222+
func awaitMark(t *testing.T, ws *websocket.Conn) string {
223+
t.Helper()
224+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
225+
defer cancel()
226+
var out strings.Builder
227+
for {
228+
typ, b, err := ws.Read(ctx)
229+
if err != nil {
230+
t.Fatalf("ws read awaiting mark: %v (have %q)", err, out.String())
231+
}
232+
if typ == websocket.MessageBinary {
233+
out.Write(b)
234+
}
235+
if markRE.MatchString(out.String()) {
236+
return out.String()
237+
}
238+
}
239+
}
240+
241+
var markRE = regexp.MustCompile(`MARK-(\d+)`)
242+
243+
func pidFromMark(t *testing.T, s string) int {
244+
t.Helper()
245+
m := markRE.FindStringSubmatch(s)
246+
if m == nil {
247+
t.Fatalf("no MARK line in %q", s)
248+
}
249+
pid, err := strconv.Atoi(m[1])
250+
if err != nil {
251+
t.Fatalf("bad pid in mark: %v", err)
252+
}
253+
return pid
254+
}

cmd/flue/update.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/karnstack/flue/internal/daemon"
25+
"github.com/karnstack/flue/internal/service"
2526
"github.com/karnstack/flue/internal/transport/local"
2627
)
2728

@@ -350,6 +351,12 @@ func extractFlue(archive []byte) ([]byte, error) {
350351
func restartForUpdate(w io.Writer, latest string) error {
351352
if mgr, err := newServiceManager(); err == nil {
352353
if st, err := mgr.Status(); err == nil && st.Installed {
354+
// Same convergence runRestart does: the restart should boot the
355+
// new build under the current unit template, not whatever an
356+
// older flue wrote at enable time.
357+
if ur, ok := mgr.(service.UnitRefresher); ok {
358+
_ = ur.RefreshUnit()
359+
}
353360
if err := mgr.Restart(); err != nil {
354361
return fmt.Errorf("restart the login service: %w", err)
355362
}

internal/holder/holder.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@ import (
1313
"io"
1414
"net"
1515
"os"
16+
"os/signal"
1617
"path/filepath"
1718
"sync"
1819
"syscall"
1920
"time"
2021

22+
"github.com/karnstack/flue/internal/config"
2123
"github.com/karnstack/flue/internal/holdwire"
2224
"github.com/karnstack/flue/internal/session"
2325
)
@@ -81,6 +83,23 @@ func Serve(dir, version string, ready io.WriteCloser) error {
8183
h := &holder{dir: dir, version: version, born: time.Now(), done: make(chan struct{})}
8284
go h.watch()
8385

86+
// SIGTERM is the machine going down or the user's login ending — the
87+
// events a holder cannot outlive. That is exactly when the old
88+
// daemon-side snapshot used to be written, so the holder writes it now:
89+
// scrollback, identity, and the agent resume hint land in the snapshots
90+
// directory for the next boot's revive pass. The daemon's close verb is
91+
// the opposite case — a deliberate retirement — and snapshots nothing.
92+
sigc := make(chan os.Signal, 1)
93+
signal.Notify(sigc, syscall.SIGTERM, syscall.SIGINT)
94+
go func() {
95+
defer signal.Stop(sigc)
96+
select {
97+
case <-sigc:
98+
h.shutdownOnSignal()
99+
case <-h.done:
100+
}
101+
}()
102+
84103
go func() {
85104
<-h.done
86105
ln.Close()
@@ -368,6 +387,31 @@ func (h *holder) pushEvents(s *session.Session) {
368387
}
369388
}
370389

390+
// shutdownOnSignal is the holder's half of a graceful machine or login
391+
// shutdown: snapshot for revival, close the session, exit. The holder dir
392+
// is left in place on purpose — the next boot's reattach pass finds its
393+
// socket dead, sweeps the dir, and the revive pass brings the session back
394+
// from the snapshot written here.
395+
func (h *holder) shutdownOnSignal() {
396+
h.mu.Lock()
397+
s := h.sess
398+
closing := h.closing
399+
h.closing = true
400+
h.mu.Unlock()
401+
if closing {
402+
return
403+
}
404+
if s != nil {
405+
if snap, ok := s.SnapshotForShutdown(); ok {
406+
if dir, err := config.Dir(); err == nil {
407+
_ = session.SaveSnapshots(filepath.Join(dir, session.SnapshotsDirName), []session.Snapshot{snap})
408+
}
409+
}
410+
_ = s.Close()
411+
}
412+
close(h.done)
413+
}
414+
371415
// watch is the self-reap guard: a holder whose child has exited and whose
372416
// daemon has not spoken for orphanGrace removes its directory and exits.
373417
// While a daemon holds a control connection the holder never self-reaps —

0 commit comments

Comments
 (0)