feat(api): force-stop endpoint for sandboxes stuck in a state change #5
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| // Copyright 2025 Daytona Platforms Inc. | ||
| // SPDX-License-Identifier: AGPL-3.0 | ||
|
|
||
| package session | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strconv" | ||
| "strings" | ||
| "syscall" | ||
|
|
||
| common_errors "github.com/daytonaio/common-go/pkg/errors" | ||
| ) | ||
|
|
||
| // inputHolderPidFileName is written by cmdWrapperFormat next to the command's | ||
| // log file and holds the PID of the async stdin-holder process. | ||
| const inputHolderPidFileName = "input_holder.pid" | ||
|
|
||
| // CommandLogPaths returns the log file and exit-code file paths for a | ||
| // command, so streaming consumers (e.g. the exec WebSocket endpoint) can tail | ||
| // output and detect completion without going through the REST endpoints. | ||
| func (s *SessionService) CommandLogPaths(sessionId, commandId string) (logPath, exitCodePath string, err error) { | ||
| session, ok := s.sessions.Get(sessionId) | ||
| if !ok { | ||
| return "", "", common_errors.NewNotFoundError(errors.New("session not found")) | ||
| } | ||
|
|
||
| command, ok := session.commands.Get(commandId) | ||
| if !ok { | ||
| return "", "", common_errors.NewNotFoundError(errors.New("command not found")) | ||
| } | ||
|
|
||
| logPath, exitCodePath = command.LogFilePath(session.Dir(s.configDir)) | ||
| return logPath, exitCodePath, nil | ||
| } | ||
|
|
||
| // WriteInput writes raw bytes to a running command's stdin FIFO. Unlike | ||
| // SendInput it adds no trailing newline and does not echo into the log — | ||
| // semantics required by byte-exact protocols (exec-over-WebSocket stdin | ||
| // frames). The FIFO is opened non-blocking first so a missing reader | ||
| // (command already gone) fails fast instead of hanging the caller. | ||
| func (s *SessionService) WriteInput(sessionId, commandId string, data []byte) error { | ||
| session, ok := s.sessions.Get(sessionId) | ||
| if !ok { | ||
| return common_errors.NewNotFoundError(errors.New("session not found")) | ||
| } | ||
|
|
||
| if session.cmd == nil || session.cmd.Process == nil { | ||
| return common_errors.NewGoneError(errors.New("session process is not running")) | ||
| } | ||
|
|
||
| if session.cmd.ProcessState != nil && session.cmd.ProcessState.Exited() { | ||
| return common_errors.NewGoneError(errors.New("session process has exited")) | ||
| } | ||
|
|
||
| command, ok := session.commands.Get(commandId) | ||
| if !ok { | ||
| return common_errors.NewNotFoundError(errors.New("command not found")) | ||
| } | ||
|
|
||
| if command.ExitCode != nil { | ||
| return common_errors.NewGoneError(fmt.Errorf("command has already completed with exit code %d", *command.ExitCode)) | ||
| } | ||
|
|
||
| inputFilePath := command.InputFilePath(session.Dir(s.configDir)) | ||
|
|
||
| fd, err := syscall.Open(inputFilePath, syscall.O_WRONLY|syscall.O_NONBLOCK, 0) | ||
| if err != nil { | ||
| if errors.Is(err, syscall.ENXIO) || os.IsNotExist(err) { | ||
| return common_errors.NewGoneError(errors.New("command stdin is closed")) | ||
| } | ||
| return common_errors.NewInternalServerError(fmt.Errorf("failed to open input pipe: %w", err)) | ||
| } | ||
|
Comment on lines
+45
to
+76
There was a problem hiding this comment. 🩺 Stability & Availability | 🟠 Major | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== locate exec_support.go and session_exec.go =="
fd -a 'exec_support\.go$|session_exec\.go$|execute\.go$' . | sed 's#^\./##'
echo
echo "== relevant source snippets =="
for f in apps/daemon/pkg/session/exec_support.go apps/daemon/pkg/session/session_exec.go apps/daemon/pkg/session/execute.go; do
if [ -f "$f" ]; then
echo "--- $f ($(wc -l < "$f") lines) ---"
if [ "$f" = "apps/daemon/pkg/session/exec_support.go" ]; then
sed -n '1,120p' "$f"
elif [ "$f" = "apps/daemon/pkg/session/session_exec.go" ]; then
sed -n '1,220p' "$f"
elif [ "$f" = "apps/daemon/pkg/session/execute.go" ]; then
sed -n '1,220p' "$f"
fi
fi
done
echo
echo "== search for WriteInput/CloseInput usage and wrapper symbols =="
rg -n "WriteInput|CloseInput|cmdWrapperFormat|Start\(|pump|Execute\\(" apps/daemon/pkg -SRepository: arrrrny/daytona Length of output: 19389 🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== cmdWrapperFormat tail =="
sed -n '200,244p' apps/daemon/pkg/session/execute.go
echo
echo "== toolbox exec controller/session startup and pump/readiness =="
sed -n '1,220p' apps/daemon/pkg/toolbox/process/exec/controller.go
echo "--- session_exec.go ---"
sed -n '1,175p' apps/daemon/pkg/toolbox/process/exec/session_exec.go
echo
echo "== websocket/transport write path around Start/stdin frames =="
rg -n "start|StartFrame|stdin|WriteInput\\(|WriteStdin|Start\\(" apps/daemon/pkg/toolbox/process/exec apps/daemon/pkg/toolbox -S
echo
echo "== FIFO ENXIO behavior probe =="
python3 - <<'PY'
import os, subprocess, time, fcntl, select, stat
pipe = '/tmp/coderabbit_fifo_probe_{}'.format(os.getpid())
try:
os.mkfifo(pipe)
# Spawn /bin/cat /dev/null > "$pipe" like the wrapper; wait briefly.
p = subprocess.Popen(['/bin/bash', '-c', 'cat /dev/null > "$pipe"'], env={'pipe': pipe})
time.sleep(0.02)
try:
fd = os.open(pipe, os.O_WRONLY|os.O_NONBLOCK)
os.write(fd, b'hello')
print('open: SUCCESS-write')
os.close(fd)
except FileExistsError as e:
r, _, _ = select.select([p.stdout.fileno()], [], [], 0.5)
s = p.stdout.readline() or p.stderr.readline() or b''
print('open: ENXIO?', 'ENXIO' in str(e))
print('open: errno:', e.errno if hasattr(e, 'errno') else None)
time.sleep(0.05)
# Open after reader has attached.
fd = os.open(pipe, os.O_WRONLY)
os.write(fd, b'hello')
os.close(fd)
print('late_open: SUCCESS-write')
finally:
p.terminate()
p.wait(timeout=2)
finally:
os.unlink(pipe)
PYRepository: arrrrny/daytona Length of output: 49490 Avoid treating an unready FIFO as closed stdin.
🤖 Prompt for AI Agents |
||
| defer func() { _ = syscall.Close(fd) }() | ||
|
|
||
| // Restore blocking semantics for the write itself so large frames don't | ||
| // fail with EAGAIN on a full pipe buffer. | ||
| if err := syscall.SetNonblock(fd, false); err != nil { | ||
| return common_errors.NewInternalServerError(fmt.Errorf("failed to configure input pipe: %w", err)) | ||
| } | ||
|
|
||
| // write(2) may return fewer bytes than requested (partial write), so loop | ||
| // until the whole frame has been delivered. | ||
| for remaining := data; len(remaining) > 0; { | ||
| n, err := syscall.Write(fd, remaining) | ||
| if err != nil { | ||
| return common_errors.NewInternalServerError(fmt.Errorf("failed to write to input pipe: %w", err)) | ||
| } | ||
| remaining = remaining[n:] | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // CloseInput delivers stdin EOF to a running command by tearing down the | ||
| // input-holder process that cmdWrapperFormat keeps alive for async commands. | ||
| // The holder is a single process (see execute.go), so killing it drops the | ||
| // FIFO's last writer and the command's stdin sees EOF — SSH channel EOF | ||
| // semantics. Best effort: if the holder is not up yet or already gone, the | ||
| // command's stdin stays as-is and nil is returned. | ||
| func (s *SessionService) CloseInput(sessionId, commandId string) error { | ||
| session, ok := s.sessions.Get(sessionId) | ||
| if !ok { | ||
| return common_errors.NewNotFoundError(errors.New("session not found")) | ||
| } | ||
|
|
||
| command, ok := session.commands.Get(commandId) | ||
| if !ok { | ||
| return common_errors.NewNotFoundError(errors.New("command not found")) | ||
| } | ||
|
|
||
| if command.ExitCode != nil { | ||
| return common_errors.NewGoneError(fmt.Errorf("command has already completed with exit code %d", *command.ExitCode)) | ||
| } | ||
|
|
||
| pidFilePath := filepath.Join(session.Dir(s.configDir), commandId, inputHolderPidFileName) | ||
| pidBytes, err := os.ReadFile(pidFilePath) | ||
| if err != nil { | ||
| return nil | ||
| } | ||
|
|
||
| pid, err := strconv.Atoi(strings.TrimSpace(string(pidBytes))) | ||
| if err != nil || pid <= 0 { | ||
| return nil | ||
| } | ||
|
|
||
| // Kill the holder (and any descendants, defensively) so no process keeps | ||
| // the FIFO's write end open. | ||
| _ = s.signalProcessTree(pid, syscall.SIGKILL) | ||
| if holder, err := os.FindProcess(pid); err == nil { | ||
| _ = holder.Signal(syscall.SIGKILL) | ||
| } | ||
|
|
||
| // The pid file is single-use: remove it so a later CloseInput doesn't | ||
| // signal a recycled PID. | ||
| _ = os.Remove(pidFilePath) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // SignalDescendants delivers sig to every descendant of the session's shell | ||
| // process — i.e. the currently running command pipeline (command subshell, | ||
| // labelers, stdin holder) — without touching the shell itself, so the wrapper | ||
| // survives to record the command's exit code (e.g. 130 for SIGINT). | ||
| func (s *SessionService) SignalDescendants(sessionId string, sig syscall.Signal) error { | ||
| session, ok := s.sessions.Get(sessionId) | ||
| if !ok { | ||
| return common_errors.NewNotFoundError(errors.New("session not found")) | ||
| } | ||
|
|
||
| if session.cmd == nil || session.cmd.Process == nil { | ||
| return common_errors.NewGoneError(errors.New("session process is not running")) | ||
| } | ||
|
|
||
| if session.cmd.ProcessState != nil && session.cmd.ProcessState.Exited() { | ||
| return common_errors.NewGoneError(errors.New("session process has exited")) | ||
| } | ||
|
|
||
| return s.signalProcessTree(session.cmd.Process.Pid, sig) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🟠 Major | 🏗️ Heavy lift
🧩 Analysis chain
🏁 Script executed:
Repository: arrrrny/daytona
Length of output: 8212
🏁 Script executed:
Repository: arrrrny/daytona
Length of output: 50372
🏁 Script executed:
Repository: arrrrny/daytona
Length of output: 50372
🏁 Script executed:
Repository: arrrrny/daytona
Length of output: 17781
Force-stop should prevent late in-flight snapshots from winning.
JobStatusrejectsIN_PROGRESS → COMPLETEDafterforceStophas set jobs toFAILED, so a delayed job status update cannot move the sandbox back toSNAPSHOTTING. The remaining race is v0 Docker snapshots:runV0SnapshotFromSandboxpersists the snapshot and returnspreviousState: SNAPSHOTTINGwithout checking whetherforceStopalready setERROR/STOPPED, so a successful snapshot can still be created after force-stop. Add a shared cancellation signal/check increateSnapshotFromSandbox/v0 snapshot path or gate the final persistence by the sandbox state.🤖 Prompt for AI Agents