From 629ed8c9649dee0ba3c06e3e9aa800e59cb291f4 Mon Sep 17 00:00:00 2001 From: whitmo Date: Mon, 2 Mar 2026 17:08:58 -0800 Subject: [PATCH] fix: recover crashed agents by recreating missing tmux windows Previously, when an agent's tmux window disappeared (crash, manual kill, etc.), both the restart command and health check loop would fail to recover it - the restart command returned an error saying the window needed to be recreated, and the health check immediately marked the agent for cleanup. Now: - handleRestartAgent recreates the tmux window when missing, then restarts Claude in it, preserving session context via --resume - Health check loop attempts window recreation + restart for persistent agents before falling back to cleanup - Both paths validate the agent's worktree still exists before attempting recovery - restartAgent validates worktree existence early to fail fast with a clear error message Co-Authored-By: Claude Opus 4.6 --- internal/daemon/daemon.go | 51 +++++++++++++++++++++++++++++++- internal/daemon/handlers_test.go | 22 ++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index c755412..92429b8 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -329,6 +329,31 @@ func (d *Daemon) checkAgentHealth() { } if !hasWindow { + // For persistent agents, try to recreate the window and restart + if agent.Type.IsPersistent() { + d.logger.Info("Agent %s window missing, attempting to recreate and restart", agentName) + worktreePath := agent.WorktreePath + if worktreePath == "" { + worktreePath = d.paths.RepoDir(repoName) + } + // Verify worktree still exists before recreating + if _, statErr := os.Stat(worktreePath); statErr == nil { + cmd := exec.Command("tmux", "new-window", "-d", "-t", repo.TmuxSession, "-n", agentName, "-c", worktreePath) + runErr := cmd.Run() + if runErr == nil { + if restartErr := d.restartAgent(repoName, agentName, agent, repo); restartErr != nil { + d.logger.Error("Failed to restart agent %s after window recreation: %v", agentName, restartErr) + appendToSliceMap(deadAgents, repoName, agentName) + } else { + d.logger.Info("Successfully recreated window and restarted agent %s", agentName) + } + continue + } + d.logger.Error("Failed to recreate tmux window for agent %s: %v", agentName, runErr) + } else { + d.logger.Warn("Agent %s worktree path %s no longer exists", agentName, worktreePath) + } + } d.logger.Warn("Agent %s window not found, marking for cleanup", agentName) appendToSliceMap(deadAgents, repoName, agentName) continue @@ -1128,12 +1153,29 @@ func (d *Daemon) handleRestartAgent(req socket.Request) socket.Response { return socket.ErrorResponse("repository '%s' not found in state", repoName) } + // Verify the agent's worktree path still exists + if agent.WorktreePath != "" { + if _, err := os.Stat(agent.WorktreePath); os.IsNotExist(err) { + return socket.ErrorResponse("agent '%s' worktree path '%s' no longer exists - the agent may need to be fully recreated", agentName, agent.WorktreePath) + } + } + hasWindow, err := d.tmux.HasWindow(d.ctx, repo.TmuxSession, agentName) if err != nil { return socket.ErrorResponse("failed to check tmux window: %v", err) } if !hasWindow { - return socket.ErrorResponse("tmux window '%s' does not exist - the agent may need to be recreated", agentName) + // Recreate the tmux window so the agent can be restarted + d.logger.Info("Tmux window '%s' missing for agent %s, recreating", agentName, agentName) + worktreePath := agent.WorktreePath + if worktreePath == "" { + worktreePath = d.paths.RepoDir(repoName) + } + cmd := exec.Command("tmux", "new-window", "-d", "-t", repo.TmuxSession, "-n", agentName, "-c", worktreePath) + if err := cmd.Run(); err != nil { + return socket.ErrorResponse("failed to recreate tmux window for agent '%s': %v", agentName, err) + } + d.logger.Info("Recreated tmux window '%s' for agent %s", agentName, agentName) } // Check if agent is already running @@ -2146,6 +2188,13 @@ func (d *Daemon) writePromptFileWithPrefix(repoName string, agentType state.Agen // It uses --resume to continue the existing session if history exists. // This works for all agent types: supervisor, merge-queue, workspace, workers, and review agents. func (d *Daemon) restartAgent(repoName, agentName string, agent state.Agent, repo *state.Repository) error { + // Verify the agent's worktree path exists + if agent.WorktreePath != "" { + if _, err := os.Stat(agent.WorktreePath); os.IsNotExist(err) { + return fmt.Errorf("agent worktree path '%s' no longer exists", agent.WorktreePath) + } + } + // Check if the session has history home, err := os.UserHomeDir() if err != nil { diff --git a/internal/daemon/handlers_test.go b/internal/daemon/handlers_test.go index 9e66ebb..884195c 100644 --- a/internal/daemon/handlers_test.go +++ b/internal/daemon/handlers_test.go @@ -1420,6 +1420,28 @@ func TestHandleRestartAgentTableDriven(t *testing.T) { wantSuccess: false, wantError: "complete", }, + { + name: "agent with missing worktree path", + args: map[string]interface{}{ + "repo": "test-repo", + "agent": "broken-agent", + }, + setupState: func(s *state.State) { + s.AddRepo("test-repo", &state.Repository{ + GithubURL: "https://github.com/test/repo", + TmuxSession: "test-session", + Agents: make(map[string]state.Agent), + }) + s.AddAgent("test-repo", "broken-agent", state.Agent{ + Type: state.AgentTypeWorker, + TmuxWindow: "broken-window", + WorktreePath: "/nonexistent/path/that/does/not/exist", + CreatedAt: time.Now(), + }) + }, + wantSuccess: false, + wantError: "worktree path", + }, } for _, tt := range tests {