diff --git a/cmd/mithril/configcmd/configcmd.go b/cmd/mithril/configcmd/configcmd.go index 0f1f7ed9c..76fa32f23 100644 --- a/cmd/mithril/configcmd/configcmd.go +++ b/cmd/mithril/configcmd/configcmd.go @@ -3,6 +3,7 @@ package configcmd import ( "bufio" "fmt" + "math" "os" "path/filepath" "regexp" @@ -117,9 +118,7 @@ func runConfigInit() { } func generateStarterConfig() string { - // Pick storage paths that work for the current environment: production - // /mnt/mithril-* when scripts/disk-setup.sh has been run, ~/.mithril/* - // otherwise. See pkg/config/defaults.go for detection details. + // Storage paths default to /mnt/mithril-* or ~/.mithril/*; see pkg/config/defaults.go. s := config.DefaultStoragePaths() return fmt.Sprintf(`# Mithril Configuration # Generated by: mithril config init @@ -155,7 +154,10 @@ source = "rpc" # "rpc" | "lightbringer" | "turbine" # [lightbringer] # enabled = false # binary_path = "./lightbringer" -# gossip_entrypoint = "1.2.3.4:8000" +# gossip_entrypoint = "entrypoint.mainnet-beta.solana.com:8001" +# gossip_port = 65400 # Public Solana gossip UDP port +# port_range_start = 65401 # Public Solana repair/TVU UDP range +# port_range_end = 65500 # shredstore stored in [storage] section # rpc_addr = "127.0.0.1:3000" # grpc_addr = "127.0.0.1:3001" @@ -302,10 +304,13 @@ func formatTOMLValue(value string) string { if _, err := fmt.Sscanf(value, "%d", &n); err == nil && fmt.Sprintf("%d", n) == strings.TrimSpace(value) { return fmt.Sprintf("%d", n) } - // Check if it's a float — return the parsed number, not raw input - var f float64 - if _, err := fmt.Sscanf(value, "%f", &f); err == nil && !strings.ContainsAny(value, "\n\r") { - return strconv.FormatFloat(f, 'f', -1, 64) + // Float only if it parses whole and round-trips (keeps IPs/versions as strings). + if t := strings.TrimSpace(value); t != "" { + if f, err := strconv.ParseFloat(t, 64); err == nil && !math.IsInf(f, 0) && !math.IsNaN(f) { + if canon := strconv.FormatFloat(f, 'f', -1, 64); canon == t { + return canon + } + } } // Check if it's a boolean diff --git a/cmd/mithril/configcmd/edit.go b/cmd/mithril/configcmd/edit.go index 30bbc7156..32a4e8d92 100644 --- a/cmd/mithril/configcmd/edit.go +++ b/cmd/mithril/configcmd/edit.go @@ -8,6 +8,7 @@ import ( "runtime" "strconv" "strings" + "unicode/utf8" "github.com/Overclock-Validator/mithril/pkg/config" "github.com/Overclock-Validator/mithril/pkg/tui" @@ -50,6 +51,9 @@ const ( edScrRPC edScrLightbringer edScrGossip + edScrLBGossipPort + edScrLBPortRangeStart + edScrLBPortRangeEnd edScrLightbringerQuiet edScrStorage edScrAccountsPath @@ -91,6 +95,9 @@ type editModel struct { rpcEndpoint string lbEnabled bool gossipEntry string + lbGossipPort string + lbRangeStart string + lbRangeEnd string lbQuiet bool accountsPath string snapshotsPath string @@ -98,6 +105,8 @@ type editModel struct { txpar string blockMaxRPS string blockInflight string + blockSource string + lbEndpoint string rpcPort string logLevel string bootstrapMode string @@ -120,6 +129,9 @@ func newEditModel(cf string, v *viper.Viper) editModel { cluster = "mainnet-beta" } rpcSlice := v.GetStringSlice("network.rpc") + if len(rpcSlice) == 0 { + rpcSlice = v.GetStringSlice("rpc.rpc") + } rpcEndpoint := "" if len(rpcSlice) > 0 { rpcEndpoint = rpcSlice[0] @@ -154,6 +166,14 @@ func newEditModel(cf string, v *viper.Viper) editModel { if logsPath == "" { logsPath = v.GetString("log.dir") } + accountsPath := v.GetString("storage.accounts") + if accountsPath == "" { + accountsPath = v.GetString("ledger.accounts_path") + } + snapshotsPath := v.GetString("snapshot.download_path") + if snapshotsPath == "" { + snapshotsPath = v.GetString("storage.snapshots") + } return editModel{ configFile: cf, @@ -165,9 +185,14 @@ func newEditModel(cf string, v *viper.Viper) editModel { txparWasSet: txparWasSet, lbEnabled: v.GetBool("lightbringer.enabled"), gossipEntry: v.GetString("lightbringer.gossip_entrypoint"), + lbGossipPort: v.GetString("lightbringer.gossip_port"), + lbRangeStart: v.GetString("lightbringer.port_range_start"), + lbRangeEnd: v.GetString("lightbringer.port_range_end"), lbQuiet: v.GetBool("lightbringer.quiet"), - accountsPath: v.GetString("storage.accounts"), - snapshotsPath: v.GetString("storage.snapshots"), + blockSource: v.GetString("block.source"), + lbEndpoint: v.GetString("block.lightbringer_endpoint"), + accountsPath: accountsPath, + snapshotsPath: snapshotsPath, logsPath: logsPath, txpar: txpar, blockMaxRPS: blockMaxRPS, @@ -219,6 +244,7 @@ func (m *editModel) goBack() { func (m editModel) isInputScreen(scr int) bool { switch scr { case edScrRPC, edScrGossip, edScrAccountsPath, edScrSnapshotsPath, + edScrLBGossipPort, edScrLBPortRangeStart, edScrLBPortRangeEnd, edScrLogsPath, edScrTuning, edScrBlockRPS, edScrBlockInflight, edScrRPCPort: return true } @@ -231,6 +257,12 @@ func (m editModel) inputValueForScreen(scr int) string { return m.rpcEndpoint case edScrGossip: return m.gossipEntry + case edScrLBGossipPort: + return m.lbGossipPort + case edScrLBPortRangeStart: + return m.lbRangeStart + case edScrLBPortRangeEnd: + return m.lbRangeEnd case edScrAccountsPath: return m.accountsPath case edScrSnapshotsPath: @@ -260,9 +292,11 @@ func (m editModel) currentItems() []edItem { if m.lbQuiet { lbStatus += ", quiet" } + } else if m.blockSource == "lightbringer" && m.lbEndpoint != "" { + lbStatus = "external: " + truncate(config.RedactEndpointForDisplay(m.lbEndpoint), 28) } return []edItem{ - {label: "Network", value: "network", desc: fmt.Sprintf("cluster=%s rpc=%s", m.cluster, truncate(m.rpcEndpoint, 35))}, + {label: "Network", value: "network", desc: fmt.Sprintf("cluster=%s rpc=%s", m.cluster, truncate(config.RedactEndpointForDisplay(m.rpcEndpoint), 35))}, {label: "Lightbringer", value: "lightbringer", desc: lbStatus}, {label: "Storage", value: "storage", desc: truncate(m.accountsPath, 30)}, {label: "Tuning", value: "tuning", desc: fmt.Sprintf("txpar=%s", m.txpar)}, @@ -282,8 +316,12 @@ func (m editModel) currentItems() []edItem { {label: "← Back", value: "_back"}, } case edScrLightbringer: + disableDesc := "Use RPC only" + if m.blockSource == "lightbringer" && m.lbEndpoint != "" { + disableDesc = "Disable managed sidecar; keep external endpoint" + } items := []edItem{ - {label: "Disable", value: "disable", desc: "Use RPC only"}, + {label: "Disable", value: "disable", desc: disableDesc}, {label: "Enable", value: "enable", desc: "Sidecar for lower-latency block streaming"}, } if m.lbEnabled { @@ -291,6 +329,24 @@ func (m editModel) currentItems() []edItem { if m.lbQuiet { quietDesc = "on (only warn/error in lightbringer.log)" } + gossipPort := m.lbGossipPort + if gossipPort == "" { + gossipPort = "65400 default" + } + rangeStart := m.lbRangeStart + if rangeStart == "" { + rangeStart = "65401 default" + } + rangeEnd := m.lbRangeEnd + if rangeEnd == "" { + rangeEnd = "65500 default" + } + items = append(items, + edItem{label: "Gossip entrypoint", value: "gossip", desc: truncate(m.gossipEntry, 30)}, + edItem{label: "Gossip UDP port", value: "gossip_port", desc: gossipPort}, + edItem{label: "UDP range start", value: "range_start", desc: rangeStart}, + edItem{label: "UDP range end", value: "range_end", desc: rangeEnd}, + ) items = append(items, edItem{label: "Quiet logs", value: "quiet", desc: quietDesc}) } items = append(items, edItem{isSep: true}, edItem{label: "← Back", value: "_back"}) @@ -439,11 +495,28 @@ func (m *editModel) handleSelect(value string) { switch value { case "enable": m.lbEnabled = true + if m.lbGossipPort == "" { + m.lbGossipPort = "65400" + } + if m.lbRangeStart == "" { + m.lbRangeStart = "65401" + } + if m.lbRangeEnd == "" { + m.lbRangeEnd = "65500" + } m.pushInput(edScrGossip) case "disable": m.lbEnabled = false m.lbQuiet = config.LightbringerQuietDefault // Reset dependent state so disable→re-enable starts clean. m.goBack() + case "gossip": + m.pushInput(edScrGossip) + case "gossip_port": + m.pushInput(edScrLBGossipPort) + case "range_start": + m.pushInput(edScrLBPortRangeStart) + case "range_end": + m.pushInput(edScrLBPortRangeEnd) case "quiet": m.pushMenu(edScrLightbringerQuiet) } @@ -473,6 +546,15 @@ func (m *editModel) handleSelect(value string) { } func (m editModel) updateInput(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + // Sanitize pasted/typed runes (strip control chars, newlines, ESC) to block TOML + terminal-escape injection. + if msg.Type == tea.KeyRunes && len(msg.Runes) > 0 { + text := config.SanitizeUserInput(string(msg.Runes)) + if text != "" { + m.inputVal = m.inputVal[:m.inputCur] + text + m.inputVal[m.inputCur:] + m.inputCur += len(text) + } + return m, nil + } switch msg.String() { case "esc": m.goBack() @@ -484,16 +566,19 @@ func (m editModel) updateInput(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } case "backspace": if m.inputCur > 0 { - m.inputVal = m.inputVal[:m.inputCur-1] + m.inputVal[m.inputCur:] - m.inputCur-- + _, size := utf8.DecodeLastRuneInString(m.inputVal[:m.inputCur]) + m.inputVal = m.inputVal[:m.inputCur-size] + m.inputVal[m.inputCur:] + m.inputCur -= size } case "left": if m.inputCur > 0 { - m.inputCur-- + _, size := utf8.DecodeLastRuneInString(m.inputVal[:m.inputCur]) + m.inputCur -= size } case "right": if m.inputCur < len(m.inputVal) { - m.inputCur++ + _, size := utf8.DecodeRuneInString(m.inputVal[m.inputCur:]) + m.inputCur += size } case "ctrl+a": m.inputCur = 0 @@ -522,12 +607,12 @@ func (m *editModel) validateAndApplyInput() bool { case edScrGossip: if val == "" { - m.inputErr = "Format: IP:port (e.g., 1.2.3.4:8000)" + m.inputErr = "Format: host:port (e.g., entrypoint.mainnet-beta.solana.com:8001)" return false } host, portStr, err := net.SplitHostPort(val) if err != nil || host == "" { - m.inputErr = "Format: IP:port (e.g., 1.2.3.4:8000)" + m.inputErr = "Format: host:port (e.g., entrypoint.mainnet-beta.solana.com:8001)" return false } if p, perr := strconv.Atoi(portStr); perr != nil || p < 1 || p > 65535 { @@ -536,6 +621,26 @@ func (m *editModel) validateAndApplyInput() bool { } m.gossipEntry = val + case edScrLBGossipPort, edScrLBPortRangeStart, edScrLBPortRangeEnd: + gossipPort := m.lbGossipPort + rangeStart := m.lbRangeStart + rangeEnd := m.lbRangeEnd + switch m.screen { + case edScrLBGossipPort: + gossipPort = val + case edScrLBPortRangeStart: + rangeStart = val + case edScrLBPortRangeEnd: + rangeEnd = val + } + if err := validateLightbringerUDPPorts(gossipPort, rangeStart, rangeEnd); err != nil { + m.inputErr = err.Error() + return false + } + m.lbGossipPort = gossipPort + m.lbRangeStart = rangeStart + m.lbRangeEnd = rangeEnd + case edScrAccountsPath: if val == "" { m.inputErr = "Path is required" @@ -558,9 +663,14 @@ func (m *editModel) validateAndApplyInput() bool { m.logsPath = filepath.Clean(val) case edScrTuning: + if val == "" { + m.txpar = "" + m.txparWasSet = true + return true + } n, err := strconv.Atoi(val) if err != nil || n < 0 { - m.inputErr = "Must be 0 (sequential) or a positive integer" + m.inputErr = "Must be empty, 0 (sequential), or a positive integer" return false } m.txpar = val @@ -595,14 +705,56 @@ func (m *editModel) validateAndApplyInput() bool { return true } +func validateLightbringerUDPPorts(gossipPortRaw, rangeStartRaw, rangeEndRaw string) error { + gossipPort, err := parseLightbringerUDPPort(gossipPortRaw, "gossip_port", 65400) + if err != nil { + return err + } + rangeStart, err := parseLightbringerUDPPort(rangeStartRaw, "port_range_start", 65401) + if err != nil { + return err + } + rangeEnd, err := parseLightbringerUDPPort(rangeEndRaw, "port_range_end", 65500) + if err != nil { + return err + } + if rangeStart > rangeEnd { + return fmt.Errorf("port_range_start must be <= port_range_end") + } + if rangeEnd-rangeStart < 25 { + return fmt.Errorf("port range must be at least 25 ports wide") + } + if rangeEnd+6 > 65535 { + return fmt.Errorf("port_range_end must be <= 65529") + } + if gossipPort >= rangeStart && gossipPort <= rangeEnd { + return fmt.Errorf("gossip_port must not overlap port_range_start..port_range_end") + } + return nil +} + +func parseLightbringerUDPPort(raw, field string, fallback int) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 65535 { + return 0, fmt.Errorf("%s must be 1-65535", field) + } + return value, nil +} + func (m *editModel) advanceFromInput() { switch m.screen { case edScrRPC: m.goBack() // back to sections m.goBack() // pop cluster too case edScrGossip: - m.goBack() // back to sections - m.goBack() // pop lightbringer too + // Single goBack returns to Lightbringer menu for both the enable flow and standalone gossip edit. + m.goBack() + case edScrLBGossipPort, edScrLBPortRangeStart, edScrLBPortRangeEnd: + m.goBack() // back to lightbringer case edScrAccountsPath, edScrSnapshotsPath, edScrLogsPath: m.goBack() // back to storage case edScrTuning: @@ -628,8 +780,9 @@ func (m *editModel) saveConfig() { content := string(data) content = setTomlValue(content, "network", "cluster", fmt.Sprintf("%q", m.cluster)) - // Preserve failover RPC endpoints — update first, keep rest - rpcArray := m.rpcFull + // Preserve failover RPC endpoints — update first, keep rest. + // Copy so we don't mutate m.rpcFull's backing array in place. + rpcArray := append([]string(nil), m.rpcFull...) if len(rpcArray) > 0 { rpcArray[0] = m.rpcEndpoint } else { @@ -642,44 +795,60 @@ func (m *editModel) saveConfig() { content = setTomlValue(content, "network", "rpc", "["+strings.Join(rpcParts, ", ")+"]") if m.accountsPath != "" { content = setTomlValue(content, "storage", "accounts", fmt.Sprintf("%q", filepath.Clean(m.accountsPath))) + content = removeTomlValue(content, "ledger", "accounts_path") } if m.snapshotsPath != "" { content = setTomlValue(content, "storage", "snapshots", fmt.Sprintf("%q", filepath.Clean(m.snapshotsPath))) + content = removeTomlValue(content, "snapshot", "download_path") } content = setTomlValue(content, "block", "max_rps", m.blockMaxRPS) content = setTomlValue(content, "block", "max_inflight", m.blockInflight) - // Only write txpar if it was originally in the config or user explicitly set a value - if m.txparWasSet && m.txpar != "" { - content = setTomlValue(content, "tuning", "txpar", m.txpar) + // Empty txpar means sequential runtime mode; remove both canonical and + // legacy keys so a pinned worker count can be cleared. + if m.txparWasSet { + if m.txpar == "" { + content = removeTomlValue(content, "tuning", "txpar") + content = removeTomlValue(content, "replay", "txpar") + } else { + content = setTomlValue(content, "tuning", "txpar", m.txpar) + content = removeTomlValue(content, "replay", "txpar") + } } content = setTomlValue(content, "rpc", "port", m.rpcPort) content = setTomlValue(content, "log", "level", fmt.Sprintf("%q", m.logLevel)) content = setTomlValue(content, "bootstrap", "mode", fmt.Sprintf("%q", m.bootstrapMode)) if m.lbEnabled { + if err := validateLightbringerUDPPorts(m.lbGossipPort, m.lbRangeStart, m.lbRangeEnd); err != nil { + m.err = err + return + } content = setTomlValue(content, "block", "source", "\"lightbringer\"") // Clear stale external endpoint so runtime uses managed sidecar's grpc_addr content = setTomlValue(content, "block", "lightbringer_endpoint", "\"\"") - if !strings.Contains(content, "[lightbringer]") { - content += fmt.Sprintf("\n[lightbringer]\nenabled = true\nbinary_path = \"./lightbringer\"\ngossip_entrypoint = %q\ngrpc_addr = \"127.0.0.1:3001\"\nrpc_addr = \"127.0.0.1:3000\"\n", m.gossipEntry) + if !hasTomlSection(content, "lightbringer") { + content += fmt.Sprintf("\n[lightbringer]\nenabled = true\nbinary_path = \"./lightbringer\"\ngossip_entrypoint = %q\ngossip_port = %s\nport_range_start = %s\nport_range_end = %s\ngrpc_addr = \"127.0.0.1:3001\"\nrpc_addr = \"127.0.0.1:3000\"\n", m.gossipEntry, defaultString(m.lbGossipPort, "65400"), defaultString(m.lbRangeStart, "65401"), defaultString(m.lbRangeEnd, "65500")) } else { content = setTomlValue(content, "lightbringer", "enabled", "true") if m.gossipEntry != "" { content = setTomlValue(content, "lightbringer", "gossip_entrypoint", fmt.Sprintf("%q", m.gossipEntry)) } } + content = setTomlValue(content, "lightbringer", "gossip_port", defaultString(m.lbGossipPort, "65400")) + content = setTomlValue(content, "lightbringer", "port_range_start", defaultString(m.lbRangeStart, "65401")) + content = setTomlValue(content, "lightbringer", "port_range_end", defaultString(m.lbRangeEnd, "65500")) if m.lbQuiet { content = setTomlValue(content, "lightbringer", "quiet", "true") } else { content = setTomlValue(content, "lightbringer", "quiet", "false") } } else { - // Only force block.source="rpc" if no external lightbringer_endpoint is configured. - // External LB mode (enabled=false + endpoint set) is a valid runtime config. - if m.v.GetString("block.lightbringer_endpoint") == "" { + // Fall back to rpc only when leaving lightbringer mode — never clobber a + // "turbine" (or other) source. External LB mode (endpoint set) stays as-is. + if m.v.GetString("block.lightbringer_endpoint") == "" && m.blockSource == "lightbringer" { content = setTomlValue(content, "block", "source", "\"rpc\"") } - if strings.Contains(content, "[lightbringer]") { + if hasTomlSection(content, "lightbringer") { content = setTomlValue(content, "lightbringer", "enabled", "false") } } @@ -745,7 +914,13 @@ func (m editModel) inputTitleDesc() (string, string) { case edScrRPC: return "RPC Endpoint", "Primary Solana RPC endpoint URL" case edScrGossip: - return "Gossip Entrypoint", "IP:port of a Solana validator running gossip" + return "Gossip Entrypoint", "Host:port of a Solana gossip entrypoint" + case edScrLBGossipPort: + return "Lightbringer Gossip UDP Port", "Public UDP gossip port. Firewall must allow inbound/outbound traffic for mainnet use." + case edScrLBPortRangeStart: + return "Lightbringer UDP Range Start", "Start of public Solana repair/TVU UDP range." + case edScrLBPortRangeEnd: + return "Lightbringer UDP Range End", "End of public Solana repair/TVU UDP range." case edScrAccountsPath: return "AccountsDB Path", "Path for AccountsDB storage (~500GB, fastest NVMe)" case edScrSnapshotsPath: @@ -867,8 +1042,10 @@ func edRenderInput(title, description, value, errMsg string, cursorPos int) stri after := text[cursorPos:] cursor := lipgloss.NewStyle().Background(edTeal).Foreground(lipgloss.Color("#000000")).Render(" ") if cursorPos < len(text) { - cursor = lipgloss.NewStyle().Background(edTeal).Foreground(lipgloss.Color("#000000")).Render(string(after[0])) - after = after[1:] + // Step a full rune so multibyte input doesn't render a lone lead byte as mojibake. + _, size := utf8.DecodeRuneInString(after) + cursor = lipgloss.NewStyle().Background(edTeal).Foreground(lipgloss.Color("#000000")).Render(after[:size]) + after = after[size:] } text = before + cursor + after } @@ -928,7 +1105,7 @@ func setTomlValue(content, section, key, value string) string { inSection := false for i, line := range lines { trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "[[") { + if sectionName, ok := tomlSectionName(trimmed); ok { if inSection { // Section found but key missing — insert before next section header result := make([]string, 0, len(lines)+1) @@ -937,7 +1114,6 @@ func setTomlValue(content, section, key, value string) string { result = append(result, lines[i:]...) return strings.Join(result, "\n") } - sectionName := strings.Trim(trimmed, "[] ") inSection = sectionName == section continue } @@ -964,9 +1140,63 @@ func setTomlValue(content, section, key, value string) string { return strings.Join(lines, "\n") } +func removeTomlValue(content, section, key string) string { + lines := strings.Split(content, "\n") + inSection := false + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if sectionName, ok := tomlSectionName(trimmed); ok { + inSection = sectionName == section + continue + } + if inSection && (strings.HasPrefix(trimmed, key+" ") || strings.HasPrefix(trimmed, key+"=")) { + lines[i] = "# " + line + return strings.Join(lines, "\n") + } + } + return content +} + func truncate(s string, max int) string { - if len(s) <= max { + // Slice on runes, not bytes, to avoid splitting multibyte chars. + r := []rune(s) + if len(r) <= max { return s } - return s[:max-3] + "..." + if max < 3 { + return string(r[:max]) + } + return string(r[:max-3]) + "..." +} + +func defaultString(value, fallback string) string { + if strings.TrimSpace(value) == "" { + return fallback + } + return value +} + +func hasTomlSection(content, section string) bool { + for _, line := range strings.Split(content, "\n") { + if sectionName, ok := tomlSectionName(line); ok && sectionName == section { + return true + } + } + return false +} + +func tomlSectionName(line string) (string, bool) { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || !strings.HasPrefix(trimmed, "[") || strings.HasPrefix(trimmed, "[[") { + return "", false + } + end := strings.Index(trimmed, "]") + if end <= 1 { + return "", false + } + tail := strings.TrimSpace(trimmed[end+1:]) + if tail != "" && !strings.HasPrefix(tail, "#") { + return "", false + } + return strings.TrimSpace(trimmed[1:end]), true } diff --git a/cmd/mithril/configcmd/edit_test.go b/cmd/mithril/configcmd/edit_test.go index 82e4dd2b2..dd5e03ff8 100644 --- a/cmd/mithril/configcmd/edit_test.go +++ b/cmd/mithril/configcmd/edit_test.go @@ -1,12 +1,15 @@ package configcmd import ( + "os" + "strings" "testing" + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) -// TestEditor_ScrLightbringerQuiet sets m.lbQuiet from a menu selection. func TestEditor_ScrLightbringerQuiet_True(t *testing.T) { m := &editModel{screen: edScrLightbringerQuiet, lbEnabled: true, lbQuiet: false} m.handleSelect("true") @@ -22,14 +25,15 @@ func TestEditor_ScrLightbringerQuiet_False(t *testing.T) { // TestEditor_DisableLB_ResetsQuiet verifies that "disable" resets m.lbQuiet to // the default so a later re-enable starts clean (no stale quiet state carried over). func TestEditor_DisableLB_ResetsQuietDefault(t *testing.T) { + // Seed lbQuiet opposite the default so the reset is observable. m := &editModel{ screen: edScrLightbringer, lbEnabled: true, - lbQuiet: true, // previously enabled quiet + lbQuiet: !config.LightbringerQuietDefault, } m.handleSelect("disable") assert.False(t, m.lbEnabled, "disable should set lbEnabled=false") - assert.True(t, m.lbQuiet, "disable should reset lbQuiet to the default to avoid stale state on re-enable") + assert.Equal(t, config.LightbringerQuietDefault, m.lbQuiet, "disable should reset lbQuiet to the default to avoid stale state on re-enable") } // TestEditor_EnableLB_PreservesQuiet ensures enable does not clobber quiet. @@ -44,8 +48,7 @@ func TestEditor_EnableLB_PreservesQuiet(t *testing.T) { assert.True(t, m.lbQuiet, "enable should not modify lbQuiet") } -// TestEditor_QuietMenuItem_ShownOnlyWhenLBEnabled verifies the conditional -// menu rendering — the "Quiet logs" entry must not appear when LB is off. +// Quiet logs entry must not appear when LB is off. func TestEditor_QuietMenuItem_HiddenWhenLBDisabled(t *testing.T) { m := editModel{screen: edScrLightbringer, lbEnabled: false} items := m.currentItems() @@ -66,3 +69,261 @@ func TestEditor_QuietMenuItem_ShownWhenLBEnabled(t *testing.T) { } assert.True(t, found, "Quiet logs entry should appear when lbEnabled=true") } + +func TestHasTomlSectionIgnoresCommentedHeaders(t *testing.T) { + content := ` +# [lightbringer] +[network] # active network section +cluster = "mainnet-beta" +` + if hasTomlSection(content, "lightbringer") { + t.Fatal("commented section header must not count as an existing section") + } + if !hasTomlSection(content, "network") { + t.Fatal("real section header should be detected") + } +} + +func TestSetTomlValueUpdatesInlineCommentSection(t *testing.T) { + content := ` +[lightbringer] # managed sidecar +enabled = false +` + updated := setTomlValue(content, "lightbringer", "enabled", "true") + + assert.Contains(t, updated, `[lightbringer] # managed sidecar`) + assert.Contains(t, updated, `enabled = true`) + assert.Equal(t, 1, strings.Count(updated, "[lightbringer]")) +} + +func TestNewEditModelFallsBackToLegacyRPCList(t *testing.T) { + v := viper.New() + config.ApplyDefaults(v) + v.Set("rpc.rpc", []string{ + "https://legacy-primary.example.invalid", + "https://legacy-backup.example.invalid", + }) + + m := newEditModel("config.toml", v) + if m.rpcEndpoint != "https://legacy-primary.example.invalid" { + t.Fatalf("unexpected primary endpoint: %q", m.rpcEndpoint) + } + if len(m.rpcFull) != 2 || m.rpcFull[1] != "https://legacy-backup.example.invalid" { + t.Fatalf("legacy failover endpoints were not preserved: %#v", m.rpcFull) + } +} + +func TestNewEditModelFallsBackToRuntimeStoragePaths(t *testing.T) { + v := viper.New() + config.ApplyDefaults(v) + v.Set("ledger.accounts_path", "/legacy/accounts") + v.Set("snapshot.download_path", "/legacy/snapshots") + + m := newEditModel("config.toml", v) + + assert.Equal(t, "/legacy/accounts", m.accountsPath) + assert.Equal(t, "/legacy/snapshots", m.snapshotsPath) +} + +func TestEditor_TxparAllowsEmptyToClearPinnedValue(t *testing.T) { + m := &editModel{screen: edScrTuning, inputVal: "", txpar: "8", txparWasSet: true} + + if !m.validateAndApplyInput() { + t.Fatalf("empty txpar should be accepted, got error: %s", m.inputErr) + } + assert.Equal(t, "", m.txpar) + assert.True(t, m.txparWasSet) +} + +func TestSaveConfig_EmptyTxparRemovesCanonicalAndLegacyKeys(t *testing.T) { + path := t.TempDir() + "/config.toml" + content := ` +[network] +cluster = "mainnet-beta" +rpc = ["https://rpc.example.invalid"] + +[block] +max_rps = 5 +max_inflight = 2 + +[rpc] +port = 8899 + +[log] +level = "info" + +[bootstrap] +mode = "auto" + +[tuning] +txpar = 8 + +[replay] +txpar = 9 +` + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + v := viper.New() + config.ApplyDefaults(v) + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + t.Fatal(err) + } + + m := &editModel{ + configFile: path, + v: v, + cluster: "mainnet-beta", + rpcEndpoint: "https://rpc.example.invalid", + rpcFull: []string{"https://rpc.example.invalid"}, + blockMaxRPS: "5", + blockInflight: "2", + rpcPort: "8899", + logLevel: "info", + bootstrapMode: "auto", + txpar: "", + txparWasSet: true, + } + m.saveConfig() + if m.err != nil { + t.Fatal(m.err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + updated := string(data) + if strings.Contains(updated, "\ntxpar = 8") || strings.Contains(updated, "\ntxpar = 9") { + t.Fatalf("txpar keys should be commented out, got:\n%s", updated) + } + assert.Contains(t, updated, "# txpar = 8") + assert.Contains(t, updated, "# txpar = 9") +} + +func TestSaveConfig_SnapshotsPathClearsShadowingDownloadPath(t *testing.T) { + path := t.TempDir() + "/config.toml" + content := ` +[network] +cluster = "mainnet-beta" +rpc = ["https://rpc.example.invalid"] + +[storage] +snapshots = "/old/storage-snapshots" + +[snapshot] +download_path = "/old/download-path" + +[block] +max_rps = 5 +max_inflight = 2 + +[rpc] +port = 8899 + +[log] +level = "info" + +[bootstrap] +mode = "auto" +` + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + v := viper.New() + config.ApplyDefaults(v) + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + t.Fatal(err) + } + + m := newEditModel(path, v) + m.snapshotsPath = "/new/snapshots" + m.saveConfig() + if m.err != nil { + t.Fatal(m.err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + updated := string(data) + assert.Contains(t, updated, `snapshots = "/new/snapshots"`) + assert.Contains(t, updated, `# download_path = "/old/download-path"`) +} + +// Saving with Lightbringer disabled must not rewrite a turbine block source to rpc. +func TestSaveConfig_PreservesTurbineSource(t *testing.T) { + path := t.TempDir() + "/config.toml" + content := ` +[network] +cluster = "mainnet-beta" +rpc = ["https://rpc.example.invalid"] + +[block] +source = "turbine" + +[lightbringer] +enabled = false +` + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + v := viper.New() + config.ApplyDefaults(v) + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + t.Fatal(err) + } + + m := newEditModel(path, v) + m.saveConfig() + if m.err != nil { + t.Fatal(m.err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + updated := string(data) + assert.Contains(t, updated, `source = "turbine"`, "turbine source must survive a save with Lightbringer disabled") + assert.NotContains(t, updated, `source = "rpc"`) +} + +// Positive path: disabling Lightbringer while source was "lightbringer" must fall back to rpc. +func TestSaveConfig_DisablingLBFromLightbringerLeavesRPC(t *testing.T) { + path := t.TempDir() + "/config.toml" + content := ` +[network] +cluster = "mainnet-beta" +rpc = ["https://rpc.example.invalid"] + +[block] +source = "lightbringer" + +[lightbringer] +enabled = false +` + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + v := viper.New() + config.ApplyDefaults(v) + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + t.Fatal(err) + } + m := newEditModel(path, v) + m.saveConfig() + if m.err != nil { + t.Fatal(m.err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + assert.Contains(t, string(data), `source = "rpc"`, "disabling LB from lightbringer mode must fall back to rpc") +} diff --git a/cmd/mithril/configcmd/format_test.go b/cmd/mithril/configcmd/format_test.go new file mode 100644 index 000000000..e80f1e171 --- /dev/null +++ b/cmd/mithril/configcmd/format_test.go @@ -0,0 +1,48 @@ +package configcmd + +import "testing" + +// Canonicalizes `mithril config set` values: IPs/versions/host:port/inf/nan stay +// quoted strings; genuine ints/floats/bools/arrays pass through. +func TestFormatTOMLValue(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"int", "42", "42"}, + {"negative int", "-7", "-7"}, + {"float", "3.14", "3.14"}, + // Non-canonical floats are quoted, not rewritten (1.50 -> 1.5). + {"non-canonical float quoted", "1.50", `"1.50"`}, + {"bool true", "true", "true"}, + {"bool false", "false", "false"}, + {"array passthrough", "[1, 2, 3]", "[1, 2, 3]"}, + + // Numeric-looking values that must stay strings. + {"ipv4 not float", "192.168.1.1", `"192.168.1.1"`}, + {"ipv4 leading octet", "203.0.113.10", `"203.0.113.10"`}, + {"version not float", "1.0.0", `"1.0.0"`}, + {"host:port not float", "0.0.0.0:8001", `"0.0.0.0:8001"`}, + {"float with trailing garbage", "1.5abc", `"1.5abc"`}, + + // Non-finite floats must not become +Inf/NaN (invalid TOML). + {"inf stays string", "inf", `"inf"`}, + {"Inf stays string", "Inf", `"Inf"`}, + {"+inf stays string", "+inf", `"+inf"`}, + {"nan stays string", "nan", `"nan"`}, + {"NaN stays string", "NaN", `"NaN"`}, + + // Plain strings and control chars. + {"plain string", "hello", `"hello"`}, + {"string with space", "hello world", `"hello world"`}, + {"newline escaped", "a\nb", `"a\nb"`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := formatTOMLValue(c.in); got != c.want { + t.Errorf("formatTOMLValue(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} diff --git a/cmd/mithril/configcmd/paste_test.go b/cmd/mithril/configcmd/paste_test.go new file mode 100644 index 000000000..7b7401959 --- /dev/null +++ b/cmd/mithril/configcmd/paste_test.go @@ -0,0 +1,75 @@ +package configcmd + +import ( + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +// pasteMsg builds the KeyRunes message bubbletea delivers for a paste. +func pasteMsg(s string) tea.KeyMsg { + return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s), Paste: true} +} + +// A multi-rune paste is inserted as a whole string. +func TestUpdateInput_PasteInserted(t *testing.T) { + m := editModel{inputVal: "", inputCur: 0} + out, _ := m.updateInput(pasteMsg("203.0.113.10:8000")) + got := out.(editModel) + if got.inputVal != "203.0.113.10:8000" { + t.Fatalf("paste not inserted: got %q", got.inputVal) + } + if got.inputCur != len("203.0.113.10:8000") { + t.Errorf("cursor = %d, want %d", got.inputCur, len("203.0.113.10:8000")) + } +} + +func TestUpdateInput_PasteIntoMiddle(t *testing.T) { + m := editModel{inputVal: "abXY", inputCur: 2} + out, _ := m.updateInput(pasteMsg("CD")) + got := out.(editModel) + if got.inputVal != "abCDXY" { + t.Errorf("mid-insert wrong: got %q want abCDXY", got.inputVal) + } + if got.inputCur != 4 { + t.Errorf("cursor = %d, want 4 (after inserted CD)", got.inputCur) + } +} + +func TestUpdateInput_PasteAllControlIsNoOp(t *testing.T) { + m := editModel{inputVal: "abc", inputCur: 3} + out, _ := m.updateInput(pasteMsg("\n\t\x1b\x00")) + got := out.(editModel) + if got.inputVal != "abc" || got.inputCur != 3 { + t.Errorf("all-control paste should be a no-op: got %q cur %d", got.inputVal, got.inputCur) + } +} + +// Embedded newlines are stripped from a paste (TOML-injection guard). +func TestUpdateInput_PasteStripsNewlineInjection(t *testing.T) { + m := editModel{} + out, _ := m.updateInput(pasteMsg("1.2.3.4:8000\nadmin = \"evil\"")) + got := out.(editModel).inputVal + if strings.ContainsAny(got, "\n\r") { + t.Fatalf("newline survived paste (TOML-injection risk): %q", got) + } +} + +// Pasted ANSI/ESC bytes are stripped. +func TestUpdateInput_PasteStripsEscape(t *testing.T) { + m := editModel{} + out, _ := m.updateInput(pasteMsg("ip\x1b[31mX")) + if got := out.(editModel).inputVal; strings.ContainsRune(got, 0x1b) { + t.Fatalf("ESC survived paste: %q", got) + } +} + +// A single typed character inserts. +func TestUpdateInput_SingleCharStillWorks(t *testing.T) { + m := editModel{inputVal: "ab", inputCur: 2} + out, _ := m.updateInput(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("c")}) + if got := out.(editModel).inputVal; got != "abc" { + t.Errorf("typing broke: got %q want abc", got) + } +} diff --git a/cmd/mithril/dashboardcmd/components.go b/cmd/mithril/dashboardcmd/components.go index faf896fe2..5cc65a74d 100644 --- a/cmd/mithril/dashboardcmd/components.go +++ b/cmd/mithril/dashboardcmd/components.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/Overclock-Validator/mithril/pkg/procctl" "github.com/Overclock-Validator/mithril/pkg/tui" "github.com/charmbracelet/lipgloss" ) @@ -35,7 +36,7 @@ type statusBarConfig struct { cluster string slot uint64 epoch uint64 - online bool // at least one service responding + runStatus procctl.Status hasConfig bool } @@ -51,11 +52,14 @@ func renderStatusBar(cfg statusBarConfig, width int) string { if cfg.slot > 0 { parts = append(parts, label.Render("slot ")+value.Render(formatNumber(cfg.slot))) parts = append(parts, label.Render("epoch ")+value.Render(fmt.Sprintf("%d", cfg.epoch))) - // Only show Online/Offline when node has actually produced state - if cfg.online { - parts = append(parts, lipgloss.NewStyle().Foreground(tui.ColorSuccess).Render("● Online")) - } else { - parts = append(parts, lipgloss.NewStyle().Foreground(tui.ColorError).Render("● Offline")) + // Match the body's Running/Stopped/Crashed wording; Stopped is muted, not an error. + switch cfg.runStatus { + case procctl.StatusRunning: + parts = append(parts, lipgloss.NewStyle().Foreground(tui.ColorSuccess).Render("● Running")) + case procctl.StatusCrashed: + parts = append(parts, lipgloss.NewStyle().Foreground(tui.ColorError).Render("✕ Crashed")) + default: + parts = append(parts, lipgloss.NewStyle().Foreground(tui.ColorTextMuted).Render("○ Stopped")) } } } else { @@ -67,7 +71,7 @@ func renderStatusBar(cfg statusBarConfig, width int) string { border := lipgloss.NewStyle(). BorderStyle(lipgloss.NormalBorder()). BorderForeground(tui.ColorBorder). - Width(width - 2). + Width(width-2). Padding(0, 1) return border.Render(line) @@ -76,7 +80,6 @@ func renderStatusBar(cfg statusBarConfig, width int) string { // ── Footer Bar ────────────────────────────────────────────────────────── type footerConfig struct { - version string configFile string } @@ -87,9 +90,6 @@ func renderFooter(cfg footerConfig, width int) string { parts := []string{ lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true).Render(" ◎ Mithril"), } - if cfg.version != "" { - parts = append(parts, value.Render(cfg.version)) - } if cfg.configFile != "" { parts = append(parts, value.Render(cfg.configFile)) } @@ -107,6 +107,45 @@ type splitViewConfig struct { focusLeft bool } +type singlePaneConfig struct { + title string + content string + focus bool +} + +func renderSinglePane(cfg singlePaneConfig, width, height int) string { + if width < 10 || height < 3 { + return "Terminal too small" + } + innerWidth := width - 2 + contentWidth := width - 4 + + borderStyle := lipgloss.NewStyle().Foreground(tui.ColorBorder) + titleStyle := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + indicator := "── " + if cfg.focus { + borderStyle = lipgloss.NewStyle().Foreground(tui.MithrilTeal) + titleStyle = lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + indicator = "─► " + } + + title := borderStyle.Render(indicator) + titleStyle.Render(cfg.title) + borderStyle.Render(" ") + titlePad := innerWidth - lipgloss.Width(title) + if titlePad < 0 { + titlePad = 0 + } + + top := borderStyle.Render("┌") + title + borderStyle.Render(strings.Repeat("─", titlePad)) + borderStyle.Render("┐") + contentLines := padLines(cfg.content, contentWidth, height) + + rows := []string{top} + for _, line := range contentLines { + rows = append(rows, borderStyle.Render("│")+" "+line+" "+borderStyle.Render("│")) + } + rows = append(rows, borderStyle.Render("└")+borderStyle.Render(strings.Repeat("─", innerWidth))+borderStyle.Render("┘")) + return strings.Join(rows, "\n") +} + func renderSplitView(cfg splitViewConfig, width, height int) string { if width < 10 || height < 3 { return "Terminal too small" @@ -234,15 +273,9 @@ func renderStackedView(cfg splitViewConfig, width, height int) string { func padLines(content string, width, height int) []string { lines := strings.Split(content, "\n") result := make([]string, height) - truncStyle := lipgloss.NewStyle().MaxWidth(width) for i := 0; i < height; i++ { if i < len(lines) { - line := truncStyle.Render(lines[i]) - pad := width - lipgloss.Width(line) - if pad > 0 { - line += strings.Repeat(" ", pad) - } - result[i] = line + result[i] = padStyledLine(lines[i], width) } else { result[i] = strings.Repeat(" ", width) } @@ -250,6 +283,35 @@ func padLines(content string, width, height int) []string { return result } +func fitTerminalFrame(content string, width, height int) string { + if width <= 0 || height <= 0 { + return content + } + lines := strings.Split(content, "\n") + if len(lines) > height { + lines = lines[:height] + } + for len(lines) < height { + lines = append(lines, "") + } + for i, line := range lines { + lines[i] = padStyledLine(line, width) + } + return strings.Join(lines, "\n") +} + +func padStyledLine(line string, width int) string { + if width <= 0 { + return "" + } + line = strings.ReplaceAll(line, "\n", " ") + line = lipgloss.NewStyle().Inline(true).MaxWidth(width).Render(line) + if pad := width - lipgloss.Width(line); pad > 0 { + line += strings.Repeat(" ", pad) + } + return line +} + // ── Menu rendering for left pane ──────────────────────────────────────── type menuItem struct { diff --git a/cmd/mithril/dashboardcmd/dashboard.go b/cmd/mithril/dashboardcmd/dashboard.go index 6ee4fc455..aa36c8c2e 100644 --- a/cmd/mithril/dashboardcmd/dashboard.go +++ b/cmd/mithril/dashboardcmd/dashboard.go @@ -8,10 +8,12 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/Overclock-Validator/mithril/cmd/mithril/setupcmd" + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" "github.com/Overclock-Validator/mithril/pkg/tui" - "github.com/Overclock-Validator/mithril/pkg/version" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" @@ -36,6 +38,7 @@ func init() { const ( screenOverview = iota + screenProcess // guided run/stop flow backed by procctl process controls screenConfig screenEdit // inline config editing screenDoctor @@ -80,13 +83,29 @@ type dataRefreshedMsg struct { checks []checkResult mithrilLines []string lbLines []string + progress []progressEvent + snapshot snapshotActivity + accounts accountsActivity + preflightErr string } type diskRefreshedMsg struct { disks []diskUsage } -func fetchDataCmd(cfgFile string) tea.Cmd { +// procDetectedMsg carries a procctl.Detect result. err set = render error; +// det nil and err empty = not yet fetched. +type procDetectedMsg struct { + det *procctl.Detection + err string +} + +type configFixResultMsg struct { + summary string + err string +} + +func fetchDataCmd(cfgFile string, spawnLogs dashboardSpawnLogs) tea.Cmd { return func() tea.Msg { var cfg *configData var state *nodeState @@ -105,9 +124,17 @@ func fetchDataCmd(cfgFile string) tea.Cmd { checks := runDoctorChecks(cfgFile, cfg) var mithrilLines, lbLines []string + var progressEvents []progressEvent + var snapshot snapshotActivity + var accounts accountsActivity + var preflightErr string if cfg != nil { - mithrilLines = readLogTail(cfg.logsPath, "mithril.log", 50) - lbLines = readLogTail(cfg.logsPath, "lightbringer.log", 50) + mithrilLines = mithrilLogLines(cfg.logsPath, 50, spawnLogs) + lbLines = lightbringerLogLines(cfg, 50) + progressEvents = readProgressEvents(cfg.logsPath, 12) + snapshot = readSnapshotActivity(cfg.snapshotsPath) + accounts = readAccountsActivity(cfg.accountsPath) + preflightErr = preflightCheck(cfg, cfg.accountsPath) } return dataRefreshedMsg{ @@ -118,22 +145,52 @@ func fetchDataCmd(cfgFile string) tea.Cmd { checks: checks, mithrilLines: mithrilLines, lbLines: lbLines, + progress: progressEvents, + snapshot: snapshot, + accounts: accounts, + preflightErr: preflightErr, } } } +func (m model) fetchDataCmd() tea.Cmd { + return fetchDataCmd(m.configFile, m.lastSpawnLogs) +} + func fetchDiskCmd(cfg *configData) tea.Cmd { return func() tea.Msg { return diskRefreshedMsg{disks: getDiskUsage(cfg)} } } +// fetchProcessCmd runs procctl.Detect and returns a procDetectedMsg. +// Serialize via proc.inflight so a tick racing a refresh can't stack calls. +func fetchProcessCmd(accountsDir string) tea.Cmd { + return func() tea.Msg { + det, err := procctl.Detect(procctl.DefaultPidFile(), procctl.DefaultLockFile(), accountsDir) + if err != nil { + return procDetectedMsg{err: err.Error()} + } + return procDetectedMsg{det: det} + } +} + +// triggerProcessFetch fetches process state, or returns nil if a fetch is +// already in flight. Sets the inflight gate, so needs a pointer receiver. +func (m *model) triggerProcessFetch() tea.Cmd { + if m.proc.inflight { + return nil + } + m.proc.inflight = true + return fetchProcessCmd(m.procAccountsDir()) +} + func tickCmd() tea.Cmd { return tea.Tick(2*time.Second, func(t time.Time) tea.Msg { return tickMsg(t) }) } func slowTickCmd() tea.Cmd { - return tea.Tick(30*time.Second, func(t time.Time) tea.Msg { return slowTickMsg(t) }) + return tea.Tick(10*time.Second, func(t time.Time) tea.Msg { return slowTickMsg(t) }) } type tickMsg time.Time @@ -174,28 +231,88 @@ type model struct { rightScroll int // Data - cfg *configData - state *nodeState - services []serviceStatus - disks []diskUsage - checks []checkResult - mithrilLines []string - lbLines []string - logScroll int // scroll offset for focused log pane - logFocused bool // true when user is scrolling logs with ↑↓ - logPane int // 0=mithril (left), 1=lightbringer (right) - disksLoaded bool // true after first disk fetch completes + cfg *configData + state *nodeState + services []serviceStatus + disks []diskUsage + checks []checkResult + mithrilLines []string + lbLines []string + progress []progressEvent + snapshot snapshotActivity + accounts accountsActivity + logScroll int // scroll offset for focused log pane + logFocused bool // true when user is scrolling logs with ↑↓ + logPane int // 0=mithril (left), 1=lightbringer (right) + logRawMode bool // true renders a full-width terminal log tail + lastSpawnLogs dashboardSpawnLogs + disksLoaded bool // true after first disk fetch completes + runFocused bool // true when the Run Node action list owns arrow/enter + runActionIdx int // selected action in the Run Node right pane + startFlow startFlowState + + // proc holds process state plus in-flight Start/Stop/Restart action state. + proc procState + + // Confirmation modal over the right pane. onYes runs on confirm. + confirmActive bool + confirmTitle string + confirmBody string + confirmOnYes func(*model) tea.Cmd // Menu items []menuItem } +// procState groups everything the dashboard knows about the mithril process. +type procState struct { + // Last Detect result; preserved across error refreshes so the badge + // doesn't flicker on a transient PID-file read failure. + detection *procctl.Detection + + // fetchedAt: last detect (ok or err). lastOkAt: last successful detect — + // the split lets the error view show "last good check N seconds ago". + fetchedAt time.Time + lastOkAt time.Time + + // Last detect error, or empty. String, not error, since it crosses goroutines. + fetchErr string + + // True while a fetchProcessCmd is running, to gate parallel Detect calls. + inflight bool + + // Active user action: "", "starting", "stopping", or "restarting". Set on + // confirm, cleared by actionResultMsg. A second action press while set is rejected. + inFlightOp string + opStartedAt time.Time + opErr string // last action error (or empty) + + // Ring buffer of status lines for the active action, capped at 8. + progressLines []string + + // Tail of the spawned mithril's stderr when Start fails early; shown verbatim. + startFailStderr string + + // Set when a Stop times out; surfaces [f] Force Stop. Cleared on next non-running detect. + stuck bool + + // preflightErr: Start blocked before spawn (supervisor conflict, unsafe lock, + // ownership, unwritable logs, incompatible AccountsDB). preflightInfo: fix success msg. + preflightErr string + preflightInfo string + + // Result of a failed guided fix. Separate field because preflightErr is + // overwritten each data tick, which would wipe it before the user sees it. + configFixErr string +} + func newModel(cf string) model { return model{ configFile: cf, screen: screenOverview, items: []menuItem{ {label: "Overview", value: "overview"}, + {label: "Run Node", value: "process"}, {label: "Config", value: "config"}, {label: "Edit Config", value: "edit"}, {label: "Doctor", value: "doctor"}, @@ -219,11 +336,17 @@ func newModel(cf string) model { {section: "turbine", key: "gossip_bind_addr", label: "Gossip UDP"}, {section: "turbine", key: "advertised_ip", label: "Advertised IP"}, {section: "turbine", key: "shred_version", label: "Shred Version"}, + {section: "block", key: "lightbringer_endpoint", label: "External LB Endpoint"}, {section: "block", key: "max_rps", label: "Block Max RPS"}, {section: "block", key: "max_inflight", label: "Block Max Inflight"}, {isSep: true}, {section: "lightbringer", key: "enabled", label: "Lightbringer"}, + {section: "lightbringer", key: "binary_path", label: "LB Binary Path"}, + {section: "lightbringer", key: "config_dir", label: "LB Config Dir"}, {section: "lightbringer", key: "gossip_entrypoint", label: "Gossip Entrypoint"}, + {section: "lightbringer", key: "gossip_port", label: "LB Gossip UDP Port"}, + {section: "lightbringer", key: "port_range_start", label: "LB UDP Range Start"}, + {section: "lightbringer", key: "port_range_end", label: "LB UDP Range End"}, {section: "lightbringer", key: "grpc_addr", label: "LB gRPC Address"}, {section: "lightbringer", key: "rpc_addr", label: "LB HTTP Address"}, {section: "lightbringer", key: "quiet", label: "LB Quiet Logs"}, @@ -238,14 +361,34 @@ func newModel(cf string) model { } func (m model) Init() tea.Cmd { - // Non-blocking: fetch data asynchronously on startup + // No process detect here: cfg isn't loaded, so an empty accountsDir would + // flicker Stopped->Crashed. First tick handles it once cfg is set. return tea.Batch( - fetchDataCmd(m.configFile), + m.fetchDataCmd(), tickCmd(), slowTickCmd(), ) } +// procAccountsDir returns the AccountsDB path, or "" if no config yet. +// Detect uses it to classify Crashed vs Stopped; "" skips that. +func (m model) procAccountsDir() string { + if m.cfg == nil { + return "" + } + return m.cfg.accountsPath +} + +func (m *model) rememberSpawnLogs(det *procctl.Detection) { + if det == nil || (det.StdoutPath == "" && det.StderrPath == "") { + return + } + m.lastSpawnLogs = dashboardSpawnLogs{ + stdoutPath: det.StdoutPath, + stderrPath: det.StderrPath, + } +} + // ── Update ────────────────────────────────────────────────────────────── func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -258,7 +401,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.childM = nil return m, nil } - // Esc on the wizard's first screen (mode selection) exits back to dashboard + // Esc on the setup TUI's first screen (mode selection) exits back to dashboard if keyMsg.String() == "esc" && setupcmd.SetupIsFirstScreen(m.childM) { m.mode = modeDashboard m.childM = nil @@ -280,7 +423,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.mode = modeDashboard m.childM = nil return m, tea.Batch( - fetchDataCmd(m.configFile), + m.fetchDataCmd(), fetchDiskCmd(m.cfg), ) } @@ -308,9 +451,17 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.WindowSizeMsg: m.width = msg.Width m.height = msg.Height + if m.screen == screenLogs { + m.logScroll = 0 + m.rightScroll = 0 + } return m, nil case dataRefreshedMsg: + oldLogLineCount := 0 + if m.logFocused && m.logScroll > 0 { + oldLogLineCount = m.currentLogLineCount() + } m.hasConfig = msg.hasConfig m.cfg = msg.cfg m.state = msg.state @@ -318,6 +469,23 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.checks = msg.checks m.mithrilLines = msg.mithrilLines m.lbLines = msg.lbLines + m.progress = msg.progress + m.snapshot = msg.snapshot + m.accounts = msg.accounts + if oldLogLineCount > 0 { + if newLogLineCount := m.currentLogLineCount(); newLogLineCount > oldLogLineCount { + m.logScroll += newLogLineCount - oldLogLineCount + } + if maxScroll := m.maxLogScroll(); m.logScroll > maxScroll { + m.logScroll = maxScroll + } + } + if m.proc.inFlightOp == "" { + m.proc.preflightErr = msg.preflightErr + if msg.preflightErr != "" { + m.proc.preflightInfo = "" + } + } // Trigger disk fetch once config is loaded (first time only) if !m.disksLoaded && m.cfg != nil { return m, fetchDiskCmd(m.cfg) @@ -332,44 +500,229 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } return m, nil + case procDetectedMsg: + // Always release the inflight gate so the next tick can fetch. + m.proc.inflight = false + m.proc.fetchedAt = time.Now() + m.proc.fetchErr = msg.err + // Keep the last-good detection on transient errors so the badge + // doesn't flicker on a single fetch hiccup; next success replaces it. + if msg.det != nil { + m.proc.detection = msg.det + m.rememberSpawnLogs(msg.det) + m.proc.lastOkAt = m.proc.fetchedAt + m.clampRunActionCursor() + // Process gone: a prior Stop timeout finished on its own, clear Stuck. + if msg.det.Status != procctl.StatusRunning { + m.proc.stuck = false + } + } + return m, nil + + case actionResultMsg: + // Action finished: clear the gate and surface the result. + m.proc.inFlightOp = "" + if msg.det != nil { + m.proc.fetchErr = "" + m.proc.fetchedAt = time.Now() + m.proc.detection = msg.det + m.rememberSpawnLogs(msg.det) + m.proc.lastOkAt = m.proc.fetchedAt + m.clampRunActionCursor() + if msg.det.Status != procctl.StatusRunning { + m.proc.stuck = false + } + } + if msg.result != "ok" { + m.proc.opErr = msg.err + } + if msg.stderr != "" { + m.proc.startFailStderr = msg.stderr + m.mithrilLines = dashboardSpawnTextLines(msg.stderr, 50) + } + if msg.spawnLogs.stdoutPath != "" || msg.spawnLogs.stderrPath != "" { + m.lastSpawnLogs = msg.spawnLogs + } + // Final progress line so the outcome shows even if the user navigates away. + switch msg.result { + case "ok": + m.proc.progressLines = append(m.proc.progressLines, "Done.") + // Process cleared; clear Stuck if set from a prior timeout. + if msg.op == opStop || msg.op == opForceStop || msg.op == opRestart { + m.proc.stuck = false + } + if msg.op == opStop || msg.op == opForceStop { + m.runActionIdx = 0 + } + if msg.op == opStart || msg.op == opRestart { + m.cancelStartFlow() + m.openLogs(false) + } + case "timeout": + m.proc.progressLines = append(m.proc.progressLines, "Timed out waiting for clean exit.") + // Only Stop/Restart can time out; set Stuck to surface [f] Force Stop. + if msg.op == opStop || msg.op == opRestart { + m.proc.stuck = true + } + default: + m.proc.progressLines = append(m.proc.progressLines, "Action failed: "+msg.err) + } + // Re-detect now so the badge updates without waiting for the next tick. + // On successful start/restart also refresh logs (the view switches there). + cmds := []tea.Cmd{} + if procFetch := m.triggerProcessFetch(); procFetch != nil { + cmds = append(cmds, procFetch) + } + if msg.result == "ok" && (msg.op == opStart || msg.op == opRestart) { + cmds = append(cmds, m.fetchDataCmd()) + } + return m, tea.Batch(cmds...) + + case configFixResultMsg: + if msg.err != "" { + // Use the tick-proof field so the next data refresh can't wipe it. + m.proc.preflightInfo = "" + m.proc.configFixErr = msg.err + return m, nil + } + m.proc.configFixErr = "" + m.proc.preflightInfo = msg.summary + cmds := []tea.Cmd{m.fetchDataCmd(), fetchDiskCmd(m.cfg)} + if procFetch := m.triggerProcessFetch(); procFetch != nil { + cmds = append(cmds, procFetch) + } + return m, tea.Batch(cmds...) + case childExitMsg: m.mode = modeDashboard m.childM = nil return m, tea.Batch( - fetchDataCmd(m.configFile), + m.fetchDataCmd(), fetchDiskCmd(m.cfg), ) case tickMsg: - return m, tea.Batch(tickCmd(), fetchDataCmd(m.configFile)) + // Gate the process fetch so a slow Detect can't stack across ticks. + batch := []tea.Cmd{tickCmd(), m.fetchDataCmd()} + if procFetch := m.triggerProcessFetch(); procFetch != nil { + batch = append(batch, procFetch) + } + return m, tea.Batch(batch...) case slowTickMsg: return m, tea.Batch(slowTickCmd(), fetchDiskCmd(m.cfg)) case tea.KeyMsg: + if m.startFlow.active { + switch msg.String() { + case "enter": + if cmd := m.advanceStartFlow(); cmd != nil { + return m, cmd + } + return m, nil + case "esc", "ctrl+c": + m.cancelStartFlow() + return m, nil + default: + return m, nil + } + } + // Modal takes priority: only y/n/esc act, other keys are no-ops so a + // stray 'q' can't fall through to the underlying view and quit. + if m.confirmActive { + switch msg.String() { + case "y", "Y", "enter": + onYes := m.confirmOnYes + m.confirmActive = false + m.confirmOnYes = nil + if onYes != nil { + return m, onYes(&m) + } + return m, nil + case "n", "N", "esc", "ctrl+c": + m.confirmActive = false + m.confirmOnYes = nil + return m, nil + default: + return m, nil + } + } + if m.editMode == editText { + // KeyRunes covers typed chars and pastes. Sanitize to strip + // control chars/newlines/ESC so a paste can't inject TOML or escapes. + if msg.Type == tea.KeyRunes && len(msg.Runes) > 0 { + text := config.SanitizeUserInput(string(msg.Runes)) + if text != "" { + m.editValue = m.editValue[:m.editCursor] + text + m.editValue[m.editCursor:] + m.editCursor += len(text) + } + return m, nil + } + switch msg.String() { + case "enter": + m.applyEditField() + return m, nil + case "esc": + m.editMode = editNone + return m, nil + case "ctrl+c": + return m, tea.Quit + case "backspace": + if m.editCursor > 0 { + _, size := utf8.DecodeLastRuneInString(m.editValue[:m.editCursor]) + m.editValue = m.editValue[:m.editCursor-size] + m.editValue[m.editCursor:] + m.editCursor -= size + } + return m, nil + case "left": + if m.editCursor > 0 { + _, size := utf8.DecodeLastRuneInString(m.editValue[:m.editCursor]) + m.editCursor -= size + } + return m, nil + case "right": + if m.editCursor < len(m.editValue) { + _, size := utf8.DecodeRuneInString(m.editValue[m.editCursor:]) + m.editCursor += size + } + return m, nil + default: + ch := msg.String() + if len(ch) == 1 && ch[0] >= 32 { + m.editValue = m.editValue[:m.editCursor] + ch + m.editValue[m.editCursor:] + m.editCursor++ + } + return m, nil + } + } switch msg.String() { case "q": if m.editMode == editNone && !m.logFocused { return m, tea.Quit } - // In text edit mode, insert 'q' as a character - if m.editMode == editText { - m.editValue = m.editValue[:m.editCursor] + "q" + m.editValue[m.editCursor:] - m.editCursor++ - return m, nil - } // In editMenu or logFocused: ignore q (use esc to exit first) case "ctrl+c": return m, tea.Quit case "up", "k": + if m.fullWidthRawLogs() { + maxScroll := m.maxLogScroll() + if m.logScroll < maxScroll { + m.logScroll++ + } + return m, nil + } if m.logFocused { - m.logScroll-- - if m.logScroll < 0 { - m.logScroll = 0 + maxScroll := m.maxLogScroll() + if m.logScroll < maxScroll { + m.logScroll++ } return m, nil } + if m.screen == screenProcess && m.runFocused { + m.moveRunAction(-1) + return m, nil + } if m.screen == screenEdit && m.editMode == editMenu { m.editOptCursor-- if m.editOptCursor < 0 { @@ -382,17 +735,22 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case "down", "j": - if m.logFocused { - // Cap scroll — use a generous limit since wrapped lines expand count - maxScroll := len(m.mithrilLines) * 3 // approximate: up to 3x after wrapping - if m.logPane == logPaneLightbringer { - maxScroll = len(m.lbLines) * 3 + if m.fullWidthRawLogs() { + if m.logScroll > 0 { + m.logScroll-- } - if m.logScroll < maxScroll { - m.logScroll++ + return m, nil + } + if m.logFocused { + if m.logScroll > 0 { + m.logScroll-- } return m, nil } + if m.screen == screenProcess && m.runFocused { + m.moveRunAction(1) + return m, nil + } if m.screen == screenEdit && m.editMode == editMenu { m.editOptCursor++ if m.editOptCursor >= len(m.editOptions) { @@ -405,27 +763,44 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case "enter": - if m.screen == screenEdit && m.editMode == editNone { - m.startEditField() + if m.fullWidthRawLogs() { return m, nil } - if m.screen == screenEdit && m.editMode == editText { - m.applyEditField() + if m.screen == screenEdit && m.editMode == editNone { + m.startEditField() return m, nil } if m.screen == screenEdit && m.editMode == editMenu { m.applyMenuSelection() return m, nil } + if m.screen == screenProcess && m.runFocused { + if cmd := m.activateRunAction(); cmd != nil { + return m, cmd + } + return m, nil + } if cmd := m.selectCurrent(); cmd != nil { return m, cmd } case "esc": + if m.fullWidthRawLogs() { + m.screen = screenProcess + m.runFocused = true + m.logFocused = false + m.logScroll = 0 + m.setMenuCursor("process") + return m, nil + } if m.logFocused { m.logFocused = false return m, nil } + if m.screen == screenProcess && m.runFocused { + m.runFocused = false + return m, nil + } if m.editMode != editNone { m.editMode = editNone return m, nil @@ -436,11 +811,57 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case "r": + // Action shortcuts only fire with action-list focus; otherwise r refreshes. + if m.screen == screenProcess && m.runFocused { + if cmd := m.handleRestartKey(); cmd != nil { + return m, cmd + } + return m, nil + } return m, tea.Batch( - fetchDataCmd(m.configFile), + m.fetchDataCmd(), fetchDiskCmd(m.cfg), ) + case "t": + if m.screen == screenLogs { + // Toggle full-width logs; the menu is always recoverable. + m.logRawMode = !m.logRawMode + m.logScroll = 0 + m.logFocused = false + return m, nil + } + + case "s": + if m.screen == screenProcess && m.runFocused { + if cmd := m.handleStartKey(); cmd != nil { + return m, cmd + } + return m, nil + } + + case "x": + if m.screen == screenLogs { + if cmd := m.handleStopFromLogsKey(); cmd != nil { + return m, cmd + } + return m, nil + } + if m.screen == screenProcess && m.runFocused { + if cmd := m.handleStopKey(); cmd != nil { + return m, cmd + } + return m, nil + } + + case "f": + if m.screen == screenProcess && m.runFocused { + if cmd := m.handleForceStopKey(); cmd != nil { + return m, cmd + } + return m, nil + } + case "e": if m.hasConfig && (m.screen == screenConfig || m.screen == screenOverview) { m.screen = screenEdit @@ -455,32 +876,32 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } } - case "backspace": - if m.editMode == editText && m.editCursor > 0 { - m.editValue = m.editValue[:m.editCursor-1] + m.editValue[m.editCursor:] - m.editCursor-- - return m, nil - } - case "left": if m.logFocused { + if m.logRawMode { + return m, nil + } m.logPane = logPaneMithril m.logScroll = 0 return m, nil } - if m.editMode == editText && m.editCursor > 0 { - m.editCursor-- + if m.screen == screenProcess && m.runFocused { + m.runFocused = false return m, nil } case "right": if m.logFocused { + if m.logRawMode { + return m, nil + } m.logPane = logPaneLightbringer m.logScroll = 0 return m, nil } - if m.editMode == editText && m.editCursor < len(m.editValue) { - m.editCursor++ + if m.screen == screenProcess { + m.runFocused = true + m.clampRunActionCursor() return m, nil } @@ -495,16 +916,6 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.rightScroll = 0 } - default: - // Text input for inline editing - if m.editMode == editText { - ch := msg.String() - if len(ch) == 1 && ch[0] >= 32 { - m.editValue = m.editValue[:m.editCursor] + ch + m.editValue[m.editCursor:] - m.editCursor++ - return m, nil - } - } } } return m, nil @@ -534,11 +945,25 @@ func (m *model) selectCurrent() tea.Cmd { item := m.items[m.cursor] m.rightScroll = 0 m.logFocused = false + m.runFocused = false + m.cancelStartFlow() m.logScroll = 0 m.editMode = editNone + // With no config, only "Create Config" is actionable; keep others on the + // overview helper so arrow keys still navigate the menu. + if !m.hasConfig && item.value != "setup" { + m.screen = screenOverview + return nil + } switch item.value { case "overview": m.screen = screenOverview + case "process": + m.screen = screenProcess + m.runFocused = true + m.runActionIdx = 0 + // Detect now (via the inflight gate) so the view isn't blank for up to 2s. + return m.triggerProcessFetch() case "config": m.screen = screenConfig case "doctor": @@ -549,7 +974,7 @@ func (m *model) selectCurrent() tea.Cmd { m.logFocused = !m.logFocused return nil } - m.screen = screenLogs + m.openLogs(false) case "disk": m.screen = screenDisk // Fetch disk data immediately when navigating to Disk screen @@ -568,6 +993,33 @@ func (m *model) selectCurrent() tea.Cmd { return nil } +func (m *model) openLogs(raw bool) { + m.screen = screenLogs + m.runFocused = false + m.logFocused = false + // Default keeps the menu visible; full-width logs are opt-in (raw / "t" toggle). + m.logRawMode = raw + m.logPane = logPaneMithril + m.logScroll = 0 + m.rightScroll = 0 + m.setMenuCursor("logs") +} + +func (m model) logsStopShortcutAvailable() bool { + return m.proc.detection != nil && + m.proc.detection.Status == procctl.StatusRunning && + m.proc.inFlightOp == "" +} + +func (m *model) setMenuCursor(value string) { + for i, item := range m.items { + if item.value == value { + m.cursor = i + return + } + } +} + // ── View ──────────────────────────────────────────────────────────────── func (m model) View() string { @@ -587,11 +1039,14 @@ func (m model) View() string { sbCfg.slot = m.state.LastSlot sbCfg.epoch = m.state.LastEpoch } - for _, svc := range m.services { - if svc.up { - sbCfg.online = true - break - } + // State file slot is frozen at bootstrap; prefer the live slot from the log tail. + if s, e, ok := m.liveNodeSlot(); ok { + sbCfg.slot = s + sbCfg.epoch = e + } + // Header status uses the same process source as the body so they can't contradict. + if m.proc.detection != nil { + sbCfg.runStatus = m.proc.detection.Status } statusBar := renderStatusBar(sbCfg, m.width) @@ -606,15 +1061,42 @@ func (m model) View() string { contentHeight = 6 } + if m.fullWidthRawLogs() { + logRows := contentHeight - 5 + if logRows < 5 { + logRows = 5 + } + content := renderSinglePane(singlePaneConfig{ + title: m.rightPaneTitle(), + content: m.renderRawLogsViewWith(m.fullPaneContentWidth(), logRows), + focus: true, + }, m.width, contentHeight) + help := renderHelpBar(m.helpItems(), m.width) + footer := renderFooter(footerConfig{configFile: m.configFile}, m.width) + return fitTerminalFrame(lipgloss.JoinVertical(lipgloss.Left, + logo, + statusBar, + "", + content, + help, + footer, + ), m.width, m.height) + } + // Left pane: menu (pass width for full-row highlight) leftPaneWidth := (m.width - 3) * 22 / 100 leftContent := renderLeftMenu(m.items, m.cursor, leftPaneWidth) - // Right pane: child TUI (setup) or screen-specific content + // Right pane: confirm modal (highest priority), child TUI, or screen content. var rightContent string - if m.mode != modeDashboard && m.childM != nil { + switch { + case m.startFlow.active: + rightContent = m.renderStartFlow() + case m.confirmActive: + rightContent = renderConfirmModal(m.confirmTitle, m.confirmBody, m.rightPaneContentWidth()) + case m.mode != modeDashboard && m.childM != nil: rightContent = m.childM.View() - } else { + default: rightContent = m.renderRightPane() } @@ -635,6 +1117,7 @@ func (m model) View() string { // Add scroll indicators when content overflows scrollHint := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) if len(rightLines) > contentHeight && contentHeight > 2 { + rightLines = rightLines[:contentHeight] rightLines[contentHeight-1] = scrollHint.Render(" ▼ pgdn for more") } if m.rightScroll > 0 && len(rightLines) > 0 { @@ -649,7 +1132,7 @@ func (m model) View() string { leftContent: leftContent, rightTitle: m.rightPaneTitle(), rightContent: rightContent, - focusLeft: true, + focusLeft: !m.rightPaneFocused(), } content := renderSplitView(splitCfg, m.width, contentHeight) @@ -658,20 +1141,16 @@ func (m model) View() string { help := renderHelpBar(helpItems, m.width) // Footer - fCfg := footerConfig{ - version: version.Version, - configFile: m.configFile, - } - footer := renderFooter(fCfg, m.width) + footer := renderFooter(footerConfig{configFile: m.configFile}, m.width) - return lipgloss.JoinVertical(lipgloss.Left, + return fitTerminalFrame(lipgloss.JoinVertical(lipgloss.Left, logo, statusBar, "", content, help, footer, - ) + ), m.width, m.height) } // moveEditCursor moves the edit field cursor, skipping separators. @@ -729,6 +1208,8 @@ func (m model) getFieldValue(f editFieldDef) string { return m.cfg.turbineAdvertisedIP case "turbine.shred_version": return m.cfg.turbineShredVersion + case "block.lightbringer_endpoint": + return m.cfg.lbExternalEndpoint case "block.max_rps": return m.cfg.blockMaxRPS case "block.max_inflight": @@ -738,8 +1219,18 @@ func (m model) getFieldValue(f editFieldDef) string { return "true" } return "false" + case "lightbringer.binary_path": + return m.cfg.lbBinaryPath + case "lightbringer.config_dir": + return m.cfg.lbConfigDir case "lightbringer.gossip_entrypoint": return m.cfg.lbGossip + case "lightbringer.gossip_port": + return m.cfg.lbGossipPort + case "lightbringer.port_range_start": + return m.cfg.lbPortRangeStart + case "lightbringer.port_range_end": + return m.cfg.lbPortRangeEnd case "lightbringer.grpc_addr": return m.cfg.lbGrpcAddr case "lightbringer.rpc_addr": @@ -861,13 +1352,17 @@ func (m *model) applyMenuSelection() { if hasExternalEndpoint { _ = saveConfigValue(m.configFile, "block", "lightbringer_endpoint", "") } - } else if !hasExternalEndpoint { - // Only force rpc when no external endpoint + } else if !hasExternalEndpoint && m.cfg != nil && m.cfg.blockSource == "lightbringer" { + // Leaving lightbringer mode → rpc; never clobber a turbine/other source. _ = saveConfigValue(m.configFile, "block", "source", "rpc") } } else if fullKey == "block.source" { if value == "lightbringer" && !hasExternalEndpoint { _ = saveConfigValue(m.configFile, "lightbringer", "enabled", "true") + } else if value == "rpc" || value == "turbine" { + // Any non-lightbringer source disables the managed sidecar so it + // doesn't spawn (and open public UDP ports) unused. + _ = saveConfigValue(m.configFile, "lightbringer", "enabled", "false") } } @@ -943,13 +1438,31 @@ func (m *model) applyEditField() { m.editErr = "Must be a port number (0-65535)" return } - case f.section == "storage": + case key == "lightbringer.gossip_port" || key == "lightbringer.port_range_start" || + key == "lightbringer.port_range_end": + gossipPort := m.cfg.lbGossipPort + rangeStart := m.cfg.lbPortRangeStart + rangeEnd := m.cfg.lbPortRangeEnd + switch key { + case "lightbringer.gossip_port": + gossipPort = value + case "lightbringer.port_range_start": + rangeStart = value + case "lightbringer.port_range_end": + rangeEnd = value + } + if _, _, _, err := parseLightbringerGossipPorts(gossipPort, rangeStart, rangeEnd); err != nil { + m.editErr = err.Error() + return + } + case f.section == "storage" || key == "lightbringer.binary_path" || key == "lightbringer.config_dir": if value == "" { m.editErr = "Path is required" return } value = filepath.Clean(value) - case key == "lightbringer.gossip_entrypoint" || key == "lightbringer.grpc_addr" || key == "lightbringer.rpc_addr": + case key == "lightbringer.gossip_entrypoint" || key == "lightbringer.grpc_addr" || + key == "lightbringer.rpc_addr" || key == "block.lightbringer_endpoint": if value != "" { host, portStr, err := net.SplitHostPort(value) if err != nil || host == "" { @@ -977,20 +1490,50 @@ func (m *model) applyEditField() { m.cfg = readConfig(m.configFile) return } + if (key == "lightbringer.gossip_port" || key == "lightbringer.port_range_start" || + key == "lightbringer.port_range_end") && value == "" { + _ = removeConfigKey(m.configFile, f.section, f.key) + m.editMode = editNone + m.cfg = readConfig(m.configFile) + return + } if err := saveConfigValue(m.configFile, f.section, f.key, value); err != nil { m.editErr = "Save failed: " + err.Error() return } + if key == "storage.accounts" { + _ = removeConfigKey(m.configFile, "ledger", "accounts_path") + } else if key == "storage.snapshots" { + _ = removeConfigKey(m.configFile, "snapshot", "download_path") + } else if key == "storage.shredstore" { + _ = removeConfigKey(m.configFile, "storage", "blockstore") + _ = removeConfigKey(m.configFile, "ledger", "path") + _ = removeConfigKey(m.configFile, "lightbringer", "storage") + } + if key == "block.lightbringer_endpoint" { + if value != "" { + _ = saveConfigValue(m.configFile, "block", "source", "lightbringer") + _ = saveConfigValue(m.configFile, "lightbringer", "enabled", "false") + } else if m.cfg != nil && !m.cfg.lbEnabled && m.cfg.blockSource == "lightbringer" { + // Leaving lightbringer mode → rpc; never clobber a turbine/other source. + _ = saveConfigValue(m.configFile, "block", "source", "rpc") + } + } m.editMode = editNone m.cfg = readConfig(m.configFile) } func (m model) rightPaneTitle() string { + if m.mode == modeSetup { + return "Create Config" + } switch m.screen { case screenOverview: return "Overview" + case screenProcess: + return "Run Node" case screenConfig: return "Configuration" case screenEdit: @@ -1001,6 +1544,9 @@ func (m model) rightPaneTitle() string { case screenDoctor: return "Health Check" case screenLogs: + if m.logRawMode { + return "Terminal Logs" + } return "Logs" case screenDisk: return "Disk Usage" @@ -1008,7 +1554,35 @@ func (m model) rightPaneTitle() string { return "" } +func (m model) rightPaneFocused() bool { + if m.confirmActive { + return true + } + if m.startFlow.active { + return true + } + if m.screen == screenProcess && m.runFocused { + return true + } + if m.screen == screenLogs && m.logFocused { + return true + } + return false +} + func (m model) helpItems() []helpItem { + if m.mode == modeSetup { + // The embedded setup TUI draws its own help; no footer needed. + return nil + } + if m.startFlow.active { + // Single confirm card — one Enter starts. + return []helpItem{ + {key: "⏎", desc: "start"}, + {key: "esc", desc: "cancel"}, + } + } + base := []helpItem{ {key: "↑↓", desc: "navigate"}, {key: "⏎", desc: "select"}, @@ -1016,20 +1590,71 @@ func (m model) helpItems() []helpItem { } switch m.screen { + case screenProcess: + if m.runFocused { + return []helpItem{ + {key: "↑↓", desc: "choose"}, + {key: "⏎", desc: "run action"}, + {key: "esc", desc: "menu"}, + {key: "q", desc: "quit"}, + } + } + return []helpItem{ + {key: "→", desc: "actions"}, + {key: "⏎", desc: "select menu"}, + {key: "r", desc: "refresh"}, + {key: "q", desc: "quit"}, + } case screenLogs: + if m.fullWidthRawLogs() { + items := []helpItem{ + {key: "↑↓", desc: "scroll"}, + {key: "esc", desc: "run node"}, + {key: "r", desc: "refresh"}, + } + if m.hasLightbringerLogPane() { + items = append(items, helpItem{key: "t", desc: "split"}) + } + if m.logsStopShortcutAvailable() { + items = append(items, helpItem{key: "x", desc: "stop safely"}) + } + items = append(items, helpItem{key: "q", desc: "quit"}) + return items + } if m.logFocused { pane := "mithril" - if m.logPane == logPaneLightbringer { + if m.logRawMode { + pane = "raw" + } else if m.logPane == logPaneLightbringer { pane = "lightbringer" } - return []helpItem{ - {key: "↑↓", desc: "scroll"}, - {key: "←→", desc: "switch pane"}, - {key: "esc", desc: "back"}, - {key: "", desc: "(" + pane + ")"}, + items := []helpItem{{key: "↑↓", desc: "scroll"}} + if m.hasLightbringerLogPane() { + if m.logRawMode { + items = append(items, helpItem{key: "t", desc: "split"}) + } else { + items = append(items, + helpItem{key: "t", desc: "terminal"}, + helpItem{key: "←→", desc: "switch pane"}, + ) + } } + items = append(items, + helpItem{key: "esc", desc: "back"}, + helpItem{key: "mode", desc: pane}, + ) + if m.logsStopShortcutAvailable() { + items = append(items, helpItem{key: "x", desc: "stop safely"}) + } + return items } base = append(base, helpItem{key: "⏎", desc: "scroll logs"}) + if m.hasLightbringerLogPane() { + base = append(base, helpItem{key: "t", desc: "toggle logs"}) + } + if m.logsStopShortcutAvailable() { + base = append(base, helpItem{key: "x", desc: "stop safely"}) + } case screenConfig: base = append(base, helpItem{key: "e", desc: "edit"}, helpItem{key: "pgdn", desc: "scroll"}) case screenOverview: diff --git a/cmd/mithril/dashboardcmd/data.go b/cmd/mithril/dashboardcmd/data.go index b8e549fc6..4472a80f3 100644 --- a/cmd/mithril/dashboardcmd/data.go +++ b/cmd/mithril/dashboardcmd/data.go @@ -1,6 +1,7 @@ package dashboardcmd import ( + "context" "encoding/json" "errors" "fmt" @@ -16,6 +17,8 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/progress" "github.com/Overclock-Validator/mithril/pkg/tui" "github.com/spf13/viper" ) @@ -36,6 +39,56 @@ type nodeState struct { Cluster string `json:"cluster"` } +const maxDashboardStateFileBytes int64 = 64 << 20 + +// slotsPerEpoch is the standard Solana epoch length, for deriving epoch from slot. +const slotsPerEpoch = 432000 + +// latestReplaySlot returns the slot from the newest per-slot replay log line, +// e.g. "(+1m2s) slot 426087845 | leader: ...". The trailing "|" requirement +// skips bootstrap lines like "snapshot slot N". +func latestReplaySlot(lines []string) (uint64, bool) { + for i := len(lines) - 1; i >= 0; i-- { + idx := strings.Index(lines[i], "slot ") + if idx < 0 { + continue + } + rest := lines[i][idx+len("slot "):] + end := 0 + for end < len(rest) && rest[end] >= '0' && rest[end] <= '9' { + end++ + } + if end == 0 { + continue + } + if after := strings.TrimLeft(rest[end:], " "); !strings.HasPrefix(after, "|") { + continue // not a per-slot replay line (e.g. "snapshot slot N") + } + if n, err := strconv.ParseUint(rest[:end], 10, 64); err == nil { + return n, true + } + } + return 0, false +} + +// isNodeRunning reports whether the managed mithril process is currently alive. +func (m model) isNodeRunning() bool { + return m.proc.detection != nil && m.proc.detection.Status == procctl.StatusRunning +} + +// liveNodeSlot returns the live replay slot/epoch from the log tail while the +// node runs. ok=false (caller falls back to state) when stopped or no slot yet. +func (m model) liveNodeSlot() (slot, epoch uint64, ok bool) { + if m.proc.detection == nil || m.proc.detection.Status != procctl.StatusRunning { + return 0, 0, false + } + s, found := latestReplaySlot(m.mithrilLines) + if !found { + return 0, 0, false + } + return s, s / slotsPerEpoch, true +} + func readState(accountsPath string) *nodeState { stateFile := filepath.Join(accountsPath, "mithril_state.json") f, err := os.Open(stateFile) @@ -44,15 +97,13 @@ func readState(accountsPath string) *nodeState { } defer f.Close() - // Cap at 1MB to prevent OOM from corrupted state files - data := make([]byte, 1<<20) - n, err := f.Read(data) - if err != nil && n == 0 { + if info, err := f.Stat(); err == nil && info.Size() > maxDashboardStateFileBytes { return nil } var s nodeState - if err := json.Unmarshal(data[:n], &s); err != nil { + decoder := json.NewDecoder(io.LimitReader(f, maxDashboardStateFileBytes)) + if err := decoder.Decode(&s); err != nil { return nil } return &s @@ -68,8 +119,12 @@ type configData struct { lbGossip string lbGrpcAddr string lbRpcAddr string + lbGossipPort string + lbPortRangeStart string + lbPortRangeEnd string lbExternalEndpoint string // block.lightbringer_endpoint for external LB mode lbBinaryPath string + lbConfigDir string lbQuiet bool turbineBindAddr string turbineGossip string @@ -105,36 +160,60 @@ func readConfig(configFile string) *configData { if txpar == "" { txpar = v.GetString("replay.txpar") } - // Don't fill in a computed default — let the UI show "auto" when empty + // Don't fill in a computed default; empty means sequential runtime mode. - logsPath := v.GetString("storage.logs") - if logsPath == "" { + logsPath := "" + if v.IsSet("storage.logs") { + logsPath = v.GetString("storage.logs") + } else if v.IsSet("log.dir") { logsPath = v.GetString("log.dir") + } else { + logsPath = "/mnt/mithril-logs" } turbineBindAddr := v.GetString("block.turbine_bind_addr") if turbineBindAddr == "" { turbineBindAddr = v.GetString("turbine.bind_addr") } + lbBinaryPath := v.GetString("lightbringer.binary_path") + if lbBinaryPath == "" { + lbBinaryPath = "./lightbringer" + } + lbConfigDir := v.GetString("lightbringer.config_dir") + if lbConfigDir == "" { + lbConfigDir = "." + } + rpcEndpoints := v.GetStringSlice("network.rpc") + if len(rpcEndpoints) == 0 { + rpcEndpoints = v.GetStringSlice("rpc.rpc") + } + accountsPath := firstNonEmptyConfigString(v, "storage.accounts", "ledger.accounts_path") + snapshotsPath := firstNonEmptyConfigString(v, "snapshot.download_path", "storage.snapshots") + shredstorePath := firstNonEmptyConfigString(v, "storage.shredstore", "storage.blockstore", "ledger.path", "lightbringer.storage") + return &configData{ cluster: cluster, - rpcEndpoints: v.GetStringSlice("network.rpc"), + rpcEndpoints: rpcEndpoints, blockSource: v.GetString("block.source"), lbEnabled: v.GetBool("lightbringer.enabled"), lbGossip: v.GetString("lightbringer.gossip_entrypoint"), lbGrpcAddr: v.GetString("lightbringer.grpc_addr"), lbRpcAddr: v.GetString("lightbringer.rpc_addr"), + lbGossipPort: v.GetString("lightbringer.gossip_port"), + lbPortRangeStart: v.GetString("lightbringer.port_range_start"), + lbPortRangeEnd: v.GetString("lightbringer.port_range_end"), lbQuiet: v.GetBool("lightbringer.quiet"), lbExternalEndpoint: v.GetString("block.lightbringer_endpoint"), - lbBinaryPath: v.GetString("lightbringer.binary_path"), + lbBinaryPath: lbBinaryPath, + lbConfigDir: lbConfigDir, turbineBindAddr: turbineBindAddr, turbineGossip: v.GetString("turbine.gossip_entrypoint"), turbineGossipBind: v.GetString("turbine.gossip_bind_addr"), turbineAdvertisedIP: v.GetString("turbine.advertised_ip"), turbineShredVersion: v.GetString("turbine.shred_version"), - accountsPath: v.GetString("storage.accounts"), - snapshotsPath: v.GetString("storage.snapshots"), - shredstorePath: v.GetString("storage.shredstore"), + accountsPath: accountsPath, + snapshotsPath: snapshotsPath, + shredstorePath: shredstorePath, logsPath: logsPath, txpar: txpar, blockMaxRPS: v.GetString("block.max_rps"), @@ -145,6 +224,33 @@ func readConfig(configFile string) *configData { } } +func firstNonEmptyConfigString(v *viper.Viper, keys ...string) string { + for _, key := range keys { + if key == "" { + continue + } + if value := v.GetString(key); value != "" { + return value + } + } + return "" +} + +func usesLightbringerBlocks(cfg *configData) bool { + if cfg == nil { + return false + } + return cfg.lbEnabled || (cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "") +} + +func isMainnetPublicRPC(cfg *configData, endpoint string) bool { + if cfg == nil || cfg.cluster != "mainnet-beta" { + return false + } + return strings.Contains(endpoint, "api.mainnet-beta.solana.com") || + strings.Contains(endpoint, "api.mainnet.solana.com") +} + // ── Service probing ───────────────────────────────────────────────────── type serviceStatus struct { @@ -182,12 +288,12 @@ func probeServices(cfg *configData) []serviceStatus { if cfg == nil || cfg.rpcPort != "0" { services = append(services, serviceStatus{name: "Mithril RPC", addr: "127.0.0.1:" + rpcPort}) } - // Probe lightbringer services based on the same mode matrix as runtime: - // - blockSource=lightbringer + enabled: managed sidecar → probe grpc + http - // - blockSource=lightbringer + endpoint: external → probe endpoint only - // - blockSource=rpc: no LB probes regardless of stale endpoint - if cfg != nil && cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" && !cfg.lbEnabled { + // Mirror runtime's LB mode matrix: external endpoint, managed sidecar, or none. + if cfg != nil && cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { services = append(services, serviceStatus{name: "LB External", addr: cfg.lbExternalEndpoint}) + if cfg.lbEnabled { + services = append(services, serviceStatus{name: "LB HTTP", addr: httpAddr}) + } } else if cfg != nil && cfg.lbEnabled { services = append(services, serviceStatus{name: "LB gRPC", addr: grpcAddr}, @@ -260,10 +366,17 @@ func getDiskUsage(cfg *configData) []diskUsage { }(i, p.label, p.path) } + // Bound the gather so a wedged mount cannot freeze the refresh; stragglers drain into the buffered channel. collected := make([]*diskUsage, len(paths)) + timeout := time.After(5 * time.Second) +gather: for range count { - r := <-ch - collected[r.idx] = r.du + select { + case r := <-ch: + collected[r.idx] = r.du + case <-timeout: + break gather + } } var results []diskUsage @@ -275,16 +388,30 @@ func getDiskUsage(cfg *configData) []diskUsage { return results } +// runDF runs `df -- path` with a 2s deadline so a hung mount can't +// block the dashboard. Each call gets its own context so the macOS -g fallback +// isn't starved by a slow -BG attempt. +func runDF(sizeFlag, path string) ([]byte, bool) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "df", sizeFlag, "--", path).Output() + return out, err == nil +} + func getDiskUsageForPath(label, path string) *diskUsage { if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { return nil } + probePath := existingDiskProbePath(path) + if probePath == "" { + return nil + } - out, err := exec.Command("df", "-BG", "--", path).Output() - if err != nil { - // Try without -BG for macOS - out, err = exec.Command("df", "-g", "--", path).Output() - if err != nil { + out, ok := runDF("-BG", probePath) + if !ok { + // -BG is GNU-only; macOS df uses -g. + out, ok = runDF("-g", probePath) + if !ok { return nil } } @@ -315,6 +442,25 @@ func getDiskUsageForPath(label, path string) *diskUsage { } } +func existingDiskProbePath(path string) string { + if strings.TrimSpace(path) == "" { + return "" + } + probePath := filepath.Clean(path) + for { + if _, err := os.Stat(probePath); err == nil { + return probePath + } else if !os.IsNotExist(err) { + return "" + } + parent := filepath.Dir(probePath) + if parent == probePath { + return "" + } + probePath = parent + } +} + func parseGB(s string) uint64 { s = strings.TrimSuffix(s, "G") s = strings.TrimSpace(s) @@ -354,7 +500,14 @@ func runDoctorChecks(configFile string, cfg *configData) []checkResult { // RPC if len(cfg.rpcEndpoints) > 0 { - results = append(results, checkResult{"RPC endpoint", "pass", cfg.rpcEndpoints[0]}) + results = append(results, checkResult{"RPC endpoint", "pass", config.RedactEndpointForDisplay(cfg.rpcEndpoints[0])}) + if isMainnetPublicRPC(cfg, cfg.rpcEndpoints[0]) { + results = append(results, checkResult{ + "RPC capacity", + "warn", + "public mainnet RPC is shared/rate-limited; use private RPC for long catchup", + }) + } } else { results = append(results, checkResult{"RPC endpoint", "fail", "no RPC endpoints configured"}) } @@ -394,6 +547,12 @@ func runDoctorChecks(configFile string, cfg *configData) []checkResult { results = append(results, checkResult{"Gossip entrypoint", "fail", "not set"}) } + if msg, err := describeLightbringerGossipPorts(cfg.lbGossipPort, cfg.lbPortRangeStart, cfg.lbPortRangeEnd); err != nil { + results = append(results, checkResult{"Lightbringer UDP ports", "fail", err.Error()}) + } else { + results = append(results, checkResult{"Lightbringer UDP ports", "warn", msg}) + } + // Quiet mode (informational) if cfg.lbQuiet { results = append(results, checkResult{"Lightbringer logs", "pass", "quiet (warn/error only)"}) @@ -402,7 +561,7 @@ func runDoctorChecks(configFile string, cfg *configData) []checkResult { } } else if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { // External Lightbringer mode — sidecar disabled but endpoint configured - results = append(results, checkResult{"Lightbringer", "pass", "external at " + cfg.lbExternalEndpoint}) + results = append(results, checkResult{"Lightbringer", "pass", "external at " + config.RedactEndpointForDisplay(cfg.lbExternalEndpoint)}) } else if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint == "" { // Invalid: source=lightbringer but no sidecar and no endpoint results = append(results, checkResult{"Lightbringer", "fail", "block.source=lightbringer requires enabled sidecar or endpoint"}) @@ -442,8 +601,60 @@ func runDoctorChecks(configFile string, cfg *configData) []checkResult { return results } +func describeLightbringerGossipPorts(gossipPortRaw, rangeStartRaw, rangeEndRaw string) (string, error) { + gossipPort, rangeStart, rangeEnd, err := parseLightbringerGossipPorts(gossipPortRaw, rangeStartRaw, rangeEndRaw) + if err != nil { + return "", err + } + return fmt.Sprintf("opens public Solana UDP gossip/repair sockets: gossip=%d range=%d-%d", gossipPort, rangeStart, rangeEnd), nil +} + +func parseLightbringerGossipPorts(gossipPortRaw, rangeStartRaw, rangeEndRaw string) (int, int, int, error) { + gossipPort, err := parseLightbringerPort(gossipPortRaw, "gossip_port", 65400) + if err != nil { + return 0, 0, 0, err + } + rangeStart, err := parseLightbringerPort(rangeStartRaw, "port_range_start", 65401) + if err != nil { + return 0, 0, 0, err + } + rangeEnd, err := parseLightbringerPort(rangeEndRaw, "port_range_end", 65500) + if err != nil { + return 0, 0, 0, err + } + if rangeStart > rangeEnd { + return 0, 0, 0, fmt.Errorf("port_range_start must be <= port_range_end") + } + if rangeEnd-rangeStart < 25 { + return 0, 0, 0, fmt.Errorf("port range must be at least 25 ports wide") + } + if rangeEnd+6 > 65535 { + return 0, 0, 0, fmt.Errorf("port_range_end must be <= 65529") + } + if gossipPort >= rangeStart && gossipPort <= rangeEnd { + return 0, 0, 0, fmt.Errorf("gossip_port must not overlap port_range_start..port_range_end") + } + return gossipPort, rangeStart, rangeEnd, nil +} + +func parseLightbringerPort(raw, field string, fallback int) (int, error) { + if strings.TrimSpace(raw) == "" { + return fallback, nil + } + value, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || value < 1 || value > 65535 { + return 0, fmt.Errorf("%s must be 1-65535", field) + } + return value, nil +} + // ── Log tailing ───────────────────────────────────────────────────────── +type dashboardSpawnLogs struct { + stdoutPath string + stderrPath string +} + func readLogTail(logsPath string, filename string, maxLines int) []string { if logsPath == "" { return []string{"(no log directory configured)"} @@ -473,10 +684,20 @@ func readLogTail(logsPath string, filename string, maxLines int) []string { return []string{"(log path escapes directory)"} } - // Read only the tail of the file to avoid loading huge logs into memory + return readPathLogTail(logFile, maxLines) +} + +func readPathLogTail(logFile string, maxLines int) []string { + if maxLines <= 0 { + return nil + } + f, err := os.Open(logFile) if err != nil { - return []string{"(could not read " + logFile + ")"} + if os.IsNotExist(err) { + return []string{"(" + filepath.Base(logFile) + " is not available yet)"} + } + return []string{"(could not read " + logFile + ": " + err.Error() + ")"} } defer f.Close() @@ -498,13 +719,645 @@ func readLogTail(logsPath string, filename string, maxLines int) []string { return []string{"(read error: " + err.Error() + ")"} } - lines := strings.Split(string(buf), "\n") + lines := strings.Split(normalizeTerminalLogText(string(buf)), "\n") + if len(lines) > maxLines { + lines = lines[len(lines)-maxLines:] + } + return lines +} + +func mithrilLogLines(logsPath string, maxLines int, spawnLogs dashboardSpawnLogs) []string { + lines := compactVolatileLogLines(cleanDashboardLogLines(trimTrailingEmptyLogLines(readMithrilLogTail(logsPath, maxLines))), maxLines) + spawnLines, spawnActive := dashboardSpawnOutputLines(maxLines, spawnLogs) + if !logTailUnavailable(lines) { + if spawnActive && len(spawnLines) > 0 { + return mergeMithrilLogLines(lines, spawnLines, maxLines) + } + return lines + } + if len(spawnLines) > 0 { + return spawnLines + } + if spawnActive { + return []string{"(Mithril is starting; logs are not available yet)"} + } + return lines +} + +func readMithrilLogTail(logsPath string, maxLines int) []string { + if info, err := procctl.ReadPidFile(procctl.DefaultPidFile()); err == nil && info != nil && info.LogDir != "" { + if ok, _ := procctl.Matches(info.Pid, info); ok { + if logFile := safeConfiguredLogFile(logsPath, info.LogDir, "mithril.log"); logFile != "" { + lines := readPathLogTail(logFile, maxLines) + if !logTailUnavailable(lines) { + return lines + } + } + } + } + return readLogTail(logsPath, "mithril.log", maxLines) +} + +func activeDashboardPidInfo() *procctl.PidInfo { + info, err := procctl.ReadPidFile(procctl.DefaultPidFile()) + if err != nil || info == nil || info.SpawnedBy != "dashboard" { + return nil + } + if ok, _ := procctl.Matches(info.Pid, info); !ok { + return nil + } + return info +} + +func safeConfiguredLogFile(logsPath, runDir, filename string) string { + if logsPath == "" || runDir == "" || filename == "" || filename != filepath.Base(filename) { + return "" + } + cleanLogs := filepath.Clean(logsPath) + cleanRunDir := filepath.Clean(runDir) + if !filepath.IsAbs(cleanRunDir) { + cleanRunDir = filepath.Join(cleanLogs, cleanRunDir) + } + cleanLogsWithSep := cleanLogs + string(os.PathSeparator) + if cleanRunDir != cleanLogs && !strings.HasPrefix(cleanRunDir+string(os.PathSeparator), cleanLogsWithSep) { + return "" + } + return filepath.Join(cleanRunDir, filename) +} + +func dashboardSpawnOutputLines(maxLines int, fallback dashboardSpawnLogs) ([]string, bool) { + spawnLogs := fallback + active := false + if info := activeDashboardPidInfo(); info != nil { + spawnLogs.stdoutPath = info.StdoutPath + spawnLogs.stderrPath = info.StderrPath + active = true + } + if spawnLogs.stdoutPath == "" && spawnLogs.stderrPath == "" { + return nil, active + } + + var lines []string + if safeDashboardSpawnLogPath(spawnLogs.stderrPath) { + lines = append(lines, trimTrailingEmptyLogLines(readDashboardSpawnLogTail(spawnLogs.stderrPath, maxLines))...) + } + return dashboardSpawnLines(lines, maxLines), active +} + +func dashboardSpawnTextLines(text string, maxLines int) []string { + lines := strings.Split(normalizeTerminalLogText(text), "\n") + return dashboardSpawnLines(lines, maxLines) +} + +func dashboardSpawnLines(lines []string, maxLines int) []string { + lines = cleanDashboardLogLines(filterUnavailableLogLines(lines)) + lines = compactVolatileLogLines(lines, maxLines-1) + if len(lines) == 0 { + return nil + } if len(lines) > maxLines { lines = lines[len(lines)-maxLines:] } return lines } +func cleanDashboardLogLines(lines []string) []string { + if len(lines) == 0 { + return nil + } + cleaned := make([]string, 0, len(lines)) + for _, line := range lines { + line = compactVerboseDashboardLogLine(line) + if isDashboardLogNoiseLine(line) { + continue + } + cleaned = append(cleaned, line) + } + return cleaned +} + +func compactVerboseDashboardLogLine(line string) string { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + return line + } + if idx := strings.Index(line, "Will save full snapshot to "); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Saving full snapshot while streaming" + } + if idx := strings.Index(line, "Cleaning up previous AccountsDB artifacts in "); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Cleaning up previous AccountsDB artifacts" + } + if idx := strings.Index(line, "Cleaning up existing snapshot files in "); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Cleaning up existing snapshot files" + } + if idx := strings.Index(line, "Cleaning up partial download:"); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Cleaning up partial snapshot download" + } + if idx := strings.Index(line, "Snapshot unpack stopped during shutdown:"); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Snapshot unpack canceled during shutdown" + } + if idx := strings.Index(strings.ToLower(line), "snapshot bootstrap cancelled during shutdown:"); idx >= 0 { + return dashboardLogPrefix(line[:idx]) + "Snapshot bootstrap canceled during shutdown" + } + return line +} + +func dashboardLogPrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return "" + } + return prefix + " " +} + +func isDashboardLogNoiseLine(line string) bool { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + return true + } + switch trimmed { + case "x", "lay": + return true + } + if strings.HasSuffix(trimmed, ")") { + if _, err := strconv.Atoi(strings.TrimSuffix(trimmed, ")")); err == nil { + return true + } + } + if strings.Contains(trimmed, "Search Time:") { + return true + } + if strings.Contains(trimmed, "→") && + (strings.Contains(trimmed, "Snapshot") || + strings.Contains(trimmed, "Incremental") || + strings.Contains(trimmed, "Fetch Blocks") || + strings.Contains(trimmed, "Replay")) { + return true + } + return isBoxDrawingOnlyLine(trimmed) +} + +func isBoxDrawingOnlyLine(line string) bool { + for _, r := range line { + switch r { + case ' ', '\t', '─', '━', '│', '┃', '┌', '┐', '└', '┘', '├', '┤', '┬', '┴', '┼', '╭', '╮', '╰', '╯': + continue + default: + return false + } + } + return true +} + +func mergeMithrilLogLines(primary, live []string, maxLines int) []string { + if len(live) == 0 { + return primary + } + merged := append([]string{}, primary...) + seen := make(map[string]struct{}, len(primary)) + for _, line := range primary { + seen[strings.TrimSpace(line)] = struct{}{} + } + for _, line := range live { + key := strings.TrimSpace(line) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + merged = append(merged, line) + } + if maxLines > 0 && len(merged) > maxLines { + merged = merged[len(merged)-maxLines:] + } + return merged +} + +func safeDashboardSpawnLogPath(path string) bool { + if path == "" { + return false + } + clean := filepath.Clean(path) + base := filepath.Base(clean) + if !strings.HasPrefix(base, "mithril-dashboard-spawn-") || !strings.HasSuffix(base, ".log") { + return false + } + dir, err := filepath.EvalSymlinks(filepath.Dir(clean)) + if err != nil { + return false + } + tempDir, err := filepath.EvalSymlinks(os.TempDir()) + if err != nil { + return false + } + return dir == tempDir +} + +func readDashboardSpawnLogTail(path string, maxLines int) []string { + if !safeDashboardSpawnLogPath(path) { + return nil + } + info, err := os.Lstat(path) + if err != nil { + return nil + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return nil + } + return readPathLogTail(path, maxLines) +} + +func logTailUnavailable(lines []string) bool { + if len(lines) == 0 { + return true + } + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + return strings.HasPrefix(trimmed, "(no latest run found") || + strings.HasPrefix(trimmed, "(could not read ") || + strings.Contains(trimmed, " is not available yet)") || + strings.HasPrefix(trimmed, "(no log directory configured)") + } + return true +} + +func filterUnavailableLogLines(lines []string) []string { + filtered := lines[:0] + for _, line := range lines { + if logTailUnavailable([]string{line}) { + continue + } + filtered = append(filtered, line) + } + return filtered +} + +func normalizeTerminalLogText(text string) string { + if text == "" { + return "" + } + text = strings.ReplaceAll(text, "\r\n", "\n") + text = strings.ReplaceAll(text, "\r", "\n") + lines := strings.Split(text, "\n") + for i, line := range lines { + lines[i] = sanitizeTerminalLogLine(line) + } + return strings.Join(lines, "\n") +} + +func sanitizeTerminalLogLine(line string) string { + if line == "" { + return "" + } + var b strings.Builder + inEscape := false + inCSI := false + inOSC := false + for _, r := range line { + if inOSC { + if r == '\a' { + inOSC = false + } + continue + } + if inEscape { + if inCSI { + if r >= 0x40 && r <= 0x7e { + inEscape = false + inCSI = false + } + continue + } + switch r { + case '[': + inCSI = true + case ']': + inEscape = false + inOSC = true + default: + inEscape = false + } + continue + } + if r == 0x1b { + inEscape = true + continue + } + if r == '\t' || (r >= 0x20 && r != 0x7f) { + b.WriteRune(r) + } + } + return b.String() +} + +func compactVolatileLogLines(lines []string, maxLines int) []string { + if len(lines) == 0 { + return nil + } + var compacted []string + pending := make(map[string]string) + var order []string + flushProgress := func() { + if len(order) == 0 { + return + } + for _, key := range order { + compacted = append(compacted, pending[key]) + } + pending = make(map[string]string) + order = order[:0] + } + + for _, line := range lines { + if key, ok := volatileProgressLogKey(line); ok { + if _, exists := pending[key]; !exists { + order = append(order, key) + } + pending[key] = compactVolatileProgressLogLine(key, line) + continue + } + flushProgress() + compacted = append(compacted, line) + } + flushProgress() + + if maxLines > 0 && len(compacted) > maxLines { + compacted = compacted[len(compacted)-maxLines:] + } + return compacted +} + +func volatileProgressLogKey(line string) (string, bool) { + trimmed := strings.TrimSpace(line) + switch { + case strings.Contains(trimmed, "Snapshot Read"): + return "snapshot-read", true + case strings.Contains(trimmed, "Extract (AppendVecs)"): + return "extract-appendvecs", true + case strings.Contains(trimmed, "AppendVec") && strings.Contains(trimmed, "ETA"): + return "extract-appendvecs", true + case strings.Contains(trimmed, "Flush (shard logs)"): + return "flush-shard-logs", true + default: + return "", false + } +} + +func compactVolatileProgressLogLine(key, line string) string { + trimmed := strings.Join(strings.Fields(strings.TrimSpace(line)), " ") + label := strings.TrimSpace(line) + switch key { + case "snapshot-read": + label = "Snapshot" + case "extract-appendvecs": + label = "Extract" + case "flush-shard-logs": + label = "Flush" + } + barStart := strings.Index(trimmed, "[") + barEnd := strings.LastIndex(trimmed, "]") + if barStart >= 0 && barEnd > barStart { + tail := strings.TrimSpace(trimmed[barEnd+1:]) + if tail != "" { + return label + ": " + tail + } + } + return trimmed +} + +func lightbringerLogLines(cfg *configData, maxLines int) []string { + if cfg == nil { + return nil + } + if cfg.lbEnabled { + lines := trimTrailingEmptyLogLines(readLogTail(cfg.logsPath, "lightbringer.log", maxLines)) + if cfg.lbQuiet { + hint := "(quiet mode: only warnings/errors are shown; silence can be normal)" + if maxLines == 1 { + return []string{hint} + } + if maxLines > 1 && len(lines) >= maxLines { + lines = lines[len(lines)-(maxLines-1):] + } + lines = append([]string{hint}, lines...) + } + return lines + } + if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { + return []string{ + "(external Lightbringer endpoint: " + config.RedactEndpointForDisplay(cfg.lbExternalEndpoint) + ")", + "(no local Lightbringer log is managed by this dashboard)", + } + } + return nil +} + +func trimTrailingEmptyLogLines(lines []string) []string { + for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { + lines = lines[:len(lines)-1] + } + return lines +} + +type progressEvent struct { + TS time.Time + Phase string + Status string + Message string + Fields map[string]any +} + +type snapshotActivity struct { + Path string + Name string + Bytes int64 + Partial bool + ModTime time.Time + Observed time.Time +} + +type accountsActivity struct { + Path string + Name string + ModTime time.Time + Observed time.Time +} + +func (s snapshotActivity) active() bool { + return s.Path != "" && s.Bytes > 0 +} + +func (a accountsActivity) active() bool { + return a.Path != "" && !a.ModTime.IsZero() +} + +func (a accountsActivity) hasData(root string) bool { + if !a.active() { + return false + } + if root == "" { + return true + } + return filepath.Clean(a.Path) != filepath.Clean(root) +} + +func readProgressEvents(logsPath string, maxEvents int) []progressEvent { + if maxEvents <= 0 { + return nil + } + lines := readLogTail(logsPath, progress.JSONLFileName, maxEvents*2) + events := make([]progressEvent, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || !strings.HasPrefix(line, "{") { + continue + } + ev, ok := parseProgressEvent(line) + if ok { + events = append(events, ev) + } + } + if len(events) > maxEvents { + events = events[len(events)-maxEvents:] + } + return events +} + +func parseProgressEvent(line string) (progressEvent, bool) { + var raw map[string]any + if err := json.Unmarshal([]byte(line), &raw); err != nil { + return progressEvent{}, false + } + phase, _ := raw["phase"].(string) + if phase == "" { + return progressEvent{}, false + } + status, _ := raw["status"].(string) + message, _ := raw["message"].(string) + var ts time.Time + if tsRaw, ok := raw["ts"].(string); ok { + ts, _ = time.Parse(time.RFC3339Nano, tsRaw) + } + fields := make(map[string]any, len(raw)) + for k, v := range raw { + switch k { + case "phase", "status", "message", "ts": + continue + default: + if s, ok := v.(string); ok { + fields[k] = config.RedactSecretsInText(s) + } else { + fields[k] = v + } + } + } + return progressEvent{ + TS: ts, + Phase: phase, + Status: status, + Message: config.RedactSecretsInText(message), + Fields: fields, + }, true +} + +func readSnapshotActivity(snapshotDir string) snapshotActivity { + if snapshotDir == "" { + return snapshotActivity{} + } + if info, err := os.Stat(snapshotDir); err == nil && !info.IsDir() { + name := filepath.Base(snapshotDir) + if !isSnapshotArtifactName(name) { + return snapshotActivity{} + } + return snapshotActivity{ + Path: snapshotDir, + Name: name, + Bytes: info.Size(), + Partial: strings.HasSuffix(name, ".partial"), + ModTime: info.ModTime(), + Observed: time.Now(), + } + } + entries, err := os.ReadDir(snapshotDir) + if err != nil { + return snapshotActivity{} + } + + var best snapshotActivity + now := time.Now() + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !isSnapshotArtifactName(name) { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + candidate := snapshotActivity{ + Path: filepath.Join(snapshotDir, name), + Name: name, + Bytes: info.Size(), + Partial: strings.HasSuffix(name, ".partial"), + ModTime: info.ModTime(), + Observed: now, + } + if best.Path == "" || candidate.ModTime.After(best.ModTime) { + best = candidate + } + } + return best +} + +func readAccountsActivity(accountsPath string) accountsActivity { + if accountsPath == "" { + return accountsActivity{} + } + candidates := []string{ + accountsPath, + filepath.Join(accountsPath, "accounts"), + filepath.Join(accountsPath, "mithril_db_log_shards"), + filepath.Join(accountsPath, "mithril_db"), + filepath.Join(accountsPath, "bankhash_db"), + filepath.Join(accountsPath, "largest_file_id"), + filepath.Join(accountsPath, "bank_hash"), + filepath.Join(accountsPath, "manifest"), + filepath.Join(accountsPath, "mithril_state.json"), + } + + var best accountsActivity + now := time.Now() + for _, path := range candidates { + info, err := os.Stat(path) + if err != nil { + continue + } + candidate := accountsActivity{ + Path: path, + Name: filepath.Base(path), + ModTime: info.ModTime(), + Observed: now, + } + if best.Path == "" || candidate.ModTime.After(best.ModTime) { + best = candidate + } + } + return best +} + +func isSnapshotArtifactName(name string) bool { + if strings.HasSuffix(name, ".partial") { + name = strings.TrimSuffix(name, ".partial") + } + return (strings.HasPrefix(name, "snapshot-") || strings.HasPrefix(name, "incremental-snapshot-")) && + strings.HasSuffix(name, ".tar.zst") +} + // ── Config saving ─────────────────────────────────────────────────────── // saveConfigValue writes a single config field to the TOML file. @@ -522,7 +1375,10 @@ func saveConfigValue(configFile, section, key, value string) error { switch { case fullKey == "block.max_rps" || fullKey == "block.max_inflight" || fullKey == "tuning.txpar" || fullKey == "rpc.port" || - fullKey == "turbine.shred_version": + fullKey == "turbine.shred_version" || + fullKey == "lightbringer.gossip_port" || + fullKey == "lightbringer.port_range_start" || + fullKey == "lightbringer.port_range_end": tomlValue = value // numeric — no quoting case fullKey == "lightbringer.enabled" || fullKey == "lightbringer.quiet": tomlValue = value // boolean — no quoting @@ -533,6 +1389,9 @@ func saveConfigValue(configFile, section, key, value string) error { v.SetConfigFile(configFile) if err := v.ReadInConfig(); err == nil { existing := v.GetStringSlice("network.rpc") + if len(existing) == 0 { + existing = v.GetStringSlice("rpc.rpc") + } if len(existing) > 1 { existing[0] = value var parts []string @@ -555,7 +1414,6 @@ func saveConfigValue(configFile, section, key, value string) error { return nil } -// setTomlValueInline replaces a value in a TOML file, preserving structure. // removeConfigKey removes a key from a TOML file (comments it out). func removeConfigKey(configFile, section, key string) error { data, err := os.ReadFile(configFile) @@ -567,8 +1425,7 @@ func removeConfigKey(configFile, section, key string) error { inSection := false for i, line := range lines { trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "[[") { - sectionName := strings.Trim(trimmed, "[] ") + if sectionName, ok := tomlSectionName(trimmed); ok { inSection = sectionName == section continue } @@ -581,13 +1438,14 @@ func removeConfigKey(configFile, section, key string) error { return nil // key not found, nothing to remove } +// setTomlValueInline replaces a value in a TOML file, preserving structure. func setTomlValueInline(content, section, key, value string) string { lines := strings.Split(content, "\n") inSection := false sectionFound := false for i, line := range lines { trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "[[") { + if sectionName, ok := tomlSectionName(trimmed); ok { if inSection { // Section found but key missing — insert before next section header result := make([]string, 0, len(lines)+1) @@ -596,7 +1454,6 @@ func setTomlValueInline(content, section, key, value string) string { result = append(result, lines[i:]...) return strings.Join(result, "\n") } - sectionName := strings.Trim(trimmed, "[] ") inSection = sectionName == section if inSection { sectionFound = true @@ -628,3 +1485,19 @@ func setTomlValueInline(content, section, key, value string) string { } return content } + +func tomlSectionName(line string) (string, bool) { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || !strings.HasPrefix(trimmed, "[") || strings.HasPrefix(trimmed, "[[") { + return "", false + } + end := strings.Index(trimmed, "]") + if end <= 1 { + return "", false + } + tail := strings.TrimSpace(trimmed[end+1:]) + if tail != "" && !strings.HasPrefix(tail, "#") { + return "", false + } + return strings.TrimSpace(trimmed[1:end]), true +} diff --git a/cmd/mithril/dashboardcmd/data_test.go b/cmd/mithril/dashboardcmd/data_test.go new file mode 100644 index 000000000..533392490 --- /dev/null +++ b/cmd/mithril/dashboardcmd/data_test.go @@ -0,0 +1,1137 @@ +package dashboardcmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/progress" + "github.com/charmbracelet/lipgloss" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadConfig_DefaultsRuntimeOwnedFields(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" +rpc = ["https://example.invalid"] +`), 0600)) + + cfg := readConfig(path) + require.NotNil(t, cfg) + assert.Equal(t, "/mnt/mithril-logs", cfg.logsPath) + assert.Equal(t, "./lightbringer", cfg.lbBinaryPath) + assert.Equal(t, ".", cfg.lbConfigDir) +} + +func TestReadStateHandlesLargeMainnetStateFile(t *testing.T) { + dir := t.TempDir() + body := fmt.Sprintf(`{ + "last_slot": 425544687, + "last_epoch": 985, + "stage": "ready", + "last_shutdown_reason": "graceful shutdown (Ctrl+C)", + "manifest_accts_lt_hash": %q +}`, strings.Repeat("a", 2<<20)) + require.Greater(t, len(body), 1<<20, "test must exceed the historical 1 MiB reader cap") + require.NoError(t, os.WriteFile(filepath.Join(dir, "mithril_state.json"), []byte(body), 0600)) + + st := readState(dir) + require.NotNil(t, st) + assert.Equal(t, uint64(425544687), st.LastSlot) + assert.Equal(t, uint64(985), st.LastEpoch) + assert.Equal(t, "ready", st.Stage) + assert.Equal(t, "graceful shutdown (Ctrl+C)", st.LastShutdownReason) +} + +func TestMithrilLogLinesFallsBackToDashboardSpawnOutput(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + stdoutF, err := os.CreateTemp("", "mithril-dashboard-spawn-stdout-*.log") + require.NoError(t, err) + defer os.Remove(stdoutF.Name()) + defer stdoutF.Close() + _, err = stdoutF.WriteString(" [1] Snapshot Download + Extract AppendVecs → [2] Flush Index\n│ Search Time: 53s\n 1)\nstdout progress should stay hidden\n") + require.NoError(t, err) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString("Snapshot Read (.tar.zst) 17.0%\nExtract (AppendVecs) 60.0 GB\n") + require.NoError(t, err) + + writeCurrentDashboardPidFile(t, pidPath, func(info *procctl.PidInfo) { + info.StdoutPath = stdoutF.Name() + info.StderrPath = stderrF.Name() + }) + + lines := mithrilLogLines(filepath.Join(t.TempDir(), "missing-logs"), 20, dashboardSpawnLogs{}) + joined := strings.Join(lines, "\n") + assert.Contains(t, joined, "Snapshot Read") + assert.Contains(t, joined, "Extract (AppendVecs)") + assert.NotContains(t, joined, "stdout progress should stay hidden") + assert.NotContains(t, joined, "Flush Index") + assert.NotContains(t, joined, "Search Time") + assert.NotContains(t, joined, " 1)") +} + +func TestMithrilLogLinesUsesRememberedSpawnOutputAfterPidFileGone(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString("Snapshot Read (.tar.zst) 8.7%\n") + require.NoError(t, err) + + lines := mithrilLogLines(filepath.Join(t.TempDir(), "missing-logs"), 20, dashboardSpawnLogs{ + stderrPath: stderrF.Name(), + }) + joined := strings.Join(lines, "\n") + assert.Contains(t, joined, "Snapshot Read") +} + +func TestMithrilLogLinesMergesActiveDashboardSpawnOutput(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + logsDir := t.TempDir() + runDir := filepath.Join(logsDir, "run-1") + require.NoError(t, os.MkdirAll(runDir, 0700)) + require.NoError(t, os.Symlink("run-1", filepath.Join(logsDir, "latest"))) + require.NoError(t, os.WriteFile(filepath.Join(runDir, "mithril.log"), []byte("mode=auto: No existing AccountsDB\nWill save full snapshot\n"), 0600)) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString("Snapshot Read (.tar.zst) 21.0%\nExtract (AppendVecs) 74.0/352.8 GB\n") + require.NoError(t, err) + + writeCurrentDashboardPidFile(t, pidPath, func(info *procctl.PidInfo) { + info.StderrPath = stderrF.Name() + }) + + lines := mithrilLogLines(logsDir, 20, dashboardSpawnLogs{}) + joined := strings.Join(lines, "\n") + assert.Contains(t, joined, "No existing AccountsDB") + assert.Contains(t, joined, "Snapshot Read") + assert.Contains(t, joined, "Extract (AppendVecs)") +} + +func TestMithrilLogLinesDeduplicatesSpawnOutputAlreadyInLog(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + logsDir := t.TempDir() + runDir := filepath.Join(logsDir, "run-1") + duplicate := "(+ 3s) mode=auto: Resuming from existing AccountsDB at slot 425546069" + require.NoError(t, os.MkdirAll(runDir, 0700)) + require.NoError(t, os.Symlink("run-1", filepath.Join(logsDir, "latest"))) + require.NoError(t, os.WriteFile(filepath.Join(runDir, "mithril.log"), []byte(duplicate+"\n"), 0600)) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString(duplicate + "\n") + require.NoError(t, err) + + writeCurrentDashboardPidFile(t, pidPath, func(info *procctl.PidInfo) { + info.StderrPath = stderrF.Name() + }) + + lines := mithrilLogLines(logsDir, 20, dashboardSpawnLogs{}) + assert.Equal(t, 1, strings.Count(strings.Join(lines, "\n"), duplicate)) +} + +func TestMithrilLogLinesPrefersActivePidLogDirOverStaleLatest(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + logsDir := t.TempDir() + staleDir := filepath.Join(logsDir, "stale-run") + activeDir := filepath.Join(logsDir, "active-run") + require.NoError(t, os.MkdirAll(staleDir, 0700)) + require.NoError(t, os.MkdirAll(activeDir, 0700)) + require.NoError(t, os.Symlink("stale-run", filepath.Join(logsDir, "latest"))) + require.NoError(t, os.WriteFile(filepath.Join(activeDir, "mithril.log"), []byte("current run line\n"), 0600)) + writeCurrentDashboardPidFile(t, pidPath, func(info *procctl.PidInfo) { + info.LogDir = activeDir + }) + + lines := mithrilLogLines(logsDir, 20, dashboardSpawnLogs{}) + + assert.Contains(t, strings.Join(lines, "\n"), "current run line") +} + +func TestMithrilLogLinesUsesCleanMessageWhenLatestLogMissing(t *testing.T) { + t.Setenv("MITHRIL_PID_FILE", filepath.Join(t.TempDir(), "missing.pid")) + + logsDir := t.TempDir() + runDir := filepath.Join(logsDir, "run-without-main-log") + require.NoError(t, os.MkdirAll(runDir, 0700)) + require.NoError(t, os.Symlink("run-without-main-log", filepath.Join(logsDir, "latest"))) + + lines := mithrilLogLines(logsDir, 20, dashboardSpawnLogs{}) + joined := strings.Join(lines, "\n") + + assert.Contains(t, joined, "mithril.log is not available yet") + assert.NotContains(t, joined, runDir) +} + +func TestMithrilLogLinesIgnoresStaleDashboardPidFile(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString("stale crash output\n") + require.NoError(t, err) + + logsDir := t.TempDir() + runDir := filepath.Join(logsDir, "run-without-main-log") + require.NoError(t, os.MkdirAll(runDir, 0700)) + require.NoError(t, os.Symlink("run-without-main-log", filepath.Join(logsDir, "latest"))) + require.NoError(t, procctl.WritePidFile(pidPath, &procctl.PidInfo{ + Pid: 99999999, + SpawnedBy: "dashboard", + StderrPath: stderrF.Name(), + })) + + lines := mithrilLogLines(logsDir, 20, dashboardSpawnLogs{}) + joined := strings.Join(lines, "\n") + + assert.Contains(t, joined, "mithril.log is not available yet") + assert.NotContains(t, joined, "stale crash output") + assert.NotContains(t, joined, "Mithril is starting") +} + +func writeCurrentDashboardPidFile(t *testing.T, pidPath string, mutate func(*procctl.PidInfo)) { + t.Helper() + id, err := procctl.ReadIdentity(os.Getpid()) + require.NoError(t, err) + exe, err := os.Executable() + require.NoError(t, err) + info := &procctl.PidInfo{ + Pid: os.Getpid(), + StartTimeTicks: id.StartTimeTicks, + ExeInode: id.ExeInode, + BinaryPath: exe, + RunID: "test-run", + SpawnedBy: "dashboard", + } + if mutate != nil { + mutate(info) + } + require.NoError(t, procctl.WritePidFile(pidPath, info)) +} + +func TestReadPathLogTailNormalizesTerminalControlCharacters(t *testing.T) { + path := filepath.Join(t.TempDir(), "mithril.log") + require.NoError(t, os.WriteFile(path, []byte("first\rsecond\x1b[31m red\x1b[0m\nthird\x07\n"), 0600)) + + lines := readPathLogTail(path, 10) + + assert.Equal(t, []string{"first", "second red", "third"}, trimTrailingEmptyLogLines(lines)) +} + +func TestCompactVolatileLogLinesKeepsLatestProgressRows(t *testing.T) { + lines := []string{ + "startup", + "Snapshot Read (.tar.zst) 0.1% 0.1/45.4 GB ETA 7m30s", + "Extract (AppendVecs) 0.2% 0.5/340.0 GB ETA 7m10s", + "Snapshot Read (.tar.zst) 0.8% 0.4/45.4 GB ETA 7m29s", + "Extract (AppendVecs) 1.0% 3.5/339.7 GB ETA 7m05s", + "ready", + } + + compacted := compactVolatileLogLines(lines, 20) + + assert.Equal(t, []string{ + "startup", + "Snapshot Read (.tar.zst) 0.8% 0.4/45.4 GB ETA 7m29s", + "Extract (AppendVecs) 1.0% 3.5/339.7 GB ETA 7m05s", + "ready", + }, compacted) +} + +func TestCompactVolatileLogLinesShortensProgressBars(t *testing.T) { + lines := []string{ + "Snapshot Read (.tar.zst) [██░░░░░░░░░░░░░░░░░░░░] 3.5% 3.8/108.1 GB 110.6 MB/s ETA 16m06s", + "Extract (AppendVecs) [█░░░░░░░░░░░░░░░░░░░░░] 2.7% 10.4/385.4 GB 377.8 MB/s ETA 16m56s", + "Flush (shard logs) [███████████████████████░░░░░░░░░░░░░░░░░] 58.6% 150/256 shards ETA 1m21s", + } + + compacted := compactVolatileLogLines(lines, 20) + + assert.Equal(t, []string{ + "Snapshot: 3.5% 3.8/108.1 GB 110.6 MB/s ETA 16m06s", + "Extract: 2.7% 10.4/385.4 GB 377.8 MB/s ETA 16m56s", + "Flush: 58.6% 150/256 shards ETA 1m21s", + }, compacted) +} + +func TestCleanDashboardLogLinesRemovesTerminalProgressFragments(t *testing.T) { + lines := []string{ + " [1] Snapshot Download + Extract AppendVecs → [2] Flush Index", + " x", + " 1)", + "│ Search Time: 53s", + " │", + "(+ 0s) Cleaning up previous AccountsDB artifacts in /very/long/accounts/path", + "(+ 0s) Cleaning up existing snapshot files in /very/long/snapshot/path (keeping 1)", + "(+ 1m03s) Will save full snapshot to /very/long/path/snapshot.tar.zst while streaming", + "(+ 1m30s) Cleaning up partial download: /very/long/path/snapshot.tar.zst.partial", + "(+20m46s) Snapshot unpack stopped during shutdown: context canceled", + "(+20m49s) snapshot bootstrap cancelled during shutdown: failed to build AccountsDB from snapshot: processing full snapshot: context canceled", + "Snapshot Read (.tar.zst) [██░░] 3.5%", + } + + cleaned := cleanDashboardLogLines(lines) + + assert.Equal(t, []string{ + "(+ 0s) Cleaning up previous AccountsDB artifacts", + "(+ 0s) Cleaning up existing snapshot files", + "(+ 1m03s) Saving full snapshot while streaming", + "(+ 1m30s) Cleaning up partial snapshot download", + "(+20m46s) Snapshot unpack canceled during shutdown", + "(+20m49s) Snapshot bootstrap canceled during shutdown", + "Snapshot Read (.tar.zst) [██░░] 3.5%", + }, cleaned) +} + +func TestMithrilLogLinesCompactsStartupCleanupPathTails(t *testing.T) { + pidPath := filepath.Join(t.TempDir(), "mithril.pid") + t.Setenv("MITHRIL_PID_FILE", pidPath) + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + _, err = stderrF.WriteString(strings.Join([]string{ + "(+ 0s) Cleaning up previous AccountsDB artifacts in /home/ubuntu/mithril-data/mainnet-live/accounts", + "(+ 0s) Cleaning up existing snapshot files in /home/ubuntu/mithril-data/mainnet-live/snapshots (keeping 1)", + "(+ 0s) Probing 318 nodes for snapshot availability...", + }, "\n")) + require.NoError(t, err) + + lines := mithrilLogLines(filepath.Join(t.TempDir(), "missing-logs"), 20, dashboardSpawnLogs{ + stderrPath: stderrF.Name(), + }) + joined := strings.Join(wrapLogLines(lines, 58), "\n") + + assert.Contains(t, joined, "Cleaning up previous AccountsDB artifacts") + assert.Contains(t, joined, "Cleaning up existing snapshot files") + assert.NotContains(t, joined, "mainnet-live") + assert.NotContains(t, joined, " 1)") +} + +func TestReadDashboardSpawnLogTailRejectsSymlink(t *testing.T) { + target, err := os.CreateTemp("", "mithril-dashboard-target-*.log") + require.NoError(t, err) + defer os.Remove(target.Name()) + defer target.Close() + _, err = target.WriteString("secret\n") + require.NoError(t, err) + + link := filepath.Join(os.TempDir(), "mithril-dashboard-spawn-stderr-symlink-test.log") + _ = os.Remove(link) + if err := os.Symlink(target.Name(), link); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + defer os.Remove(link) + + assert.Empty(t, readDashboardSpawnLogTail(link, 10)) +} + +func TestReadConfig_FallsBackToLegacyRPCList(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" + +[rpc] +rpc = ["https://legacy-rpc.example.invalid"] +`), 0600)) + + cfg := readConfig(path) + require.NotNil(t, cfg) + assert.Equal(t, []string{"https://legacy-rpc.example.invalid"}, cfg.rpcEndpoints) +} + +func TestReadConfig_FallsBackToRuntimeStoragePaths(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" +rpc = ["https://example.invalid"] + +[ledger] +accounts_path = "/legacy/accounts" +path = "/legacy/shredstore" + +[snapshot] +download_path = "/legacy/snapshots" + +[log] +dir = "/legacy/logs" +`), 0600)) + + cfg := readConfig(path) + require.NotNil(t, cfg) + assert.Equal(t, "/legacy/accounts", cfg.accountsPath) + assert.Equal(t, "/legacy/snapshots", cfg.snapshotsPath) + assert.Equal(t, "/legacy/shredstore", cfg.shredstorePath) + assert.Equal(t, "/legacy/logs", cfg.logsPath) +} + +func TestSaveConfigValue_PreservesLegacyRPCFailovers(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" + +[rpc] +rpc = ["https://old-primary.example.invalid", "https://backup.example.invalid"] +`), 0600)) + + require.NoError(t, saveConfigValue(path, "network", "rpc", "https://new-primary.example.invalid")) + + cfg := readConfig(path) + require.NotNil(t, cfg) + assert.Equal(t, []string{ + "https://new-primary.example.invalid", + "https://backup.example.invalid", + }, cfg.rpcEndpoints) +} + +func TestSaveConfigValue_UpdatesSectionWithInlineComment(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[lightbringer] # managed sidecar +enabled = false +`), 0600)) + + require.NoError(t, saveConfigValue(path, "lightbringer", "enabled", "true")) + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `[lightbringer] # managed sidecar`) + assert.Contains(t, body, `enabled = true`) + assert.Equal(t, 1, strings.Count(body, "[lightbringer]")) +} + +func TestRenderConfigView_UsesInlineCommentSectionName(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[lightbringer] # managed sidecar +enabled = false +`), 0600)) + + m := newModel(path) + m.width = 100 + m.height = 40 + + out := m.renderConfigView() + assert.Contains(t, out, "lightbringer") + assert.NotContains(t, out, "lightbringer] # managed sidecar") +} + +func TestReadConfig_ExplicitEmptyLogsDisablesDefault(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" +rpc = ["https://example.invalid"] + +[storage] +logs = "" +`), 0600)) + + cfg := readConfig(path) + require.NotNil(t, cfg) + assert.Equal(t, "", cfg.logsPath) +} + +func TestLightbringerLogLines_DisabledStaysSilent(t *testing.T) { + lines := lightbringerLogLines(&configData{blockSource: "rpc"}, 50) + + assert.Empty(t, lines) +} + +func TestLightbringerLogLines_ExternalExplainsNoManagedLog(t *testing.T) { + lines := lightbringerLogLines(&configData{ + blockSource: "lightbringer", + lbExternalEndpoint: "127.0.0.1:3001", + }, 50) + + require.Len(t, lines, 2) + assert.Contains(t, lines[0], "127.0.0.1:3001") + assert.Contains(t, lines[1], "no local Lightbringer log") +} + +func TestLightbringerLogLines_ManagedReadsTail(t *testing.T) { + dir := t.TempDir() + runDir := filepath.Join(dir, "run-1") + require.NoError(t, os.Mkdir(runDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(runDir, "lightbringer.log"), []byte("ready\nserving\n"), 0600)) + require.NoError(t, os.Symlink("run-1", filepath.Join(dir, "latest"))) + + lines := lightbringerLogLines(&configData{logsPath: dir, lbEnabled: true}, 50) + + assert.Contains(t, lines, "ready") + assert.Contains(t, lines, "serving") +} + +func TestLightbringerLogLines_QuietModeKeepsHelpfulHint(t *testing.T) { + dir := t.TempDir() + runDir := filepath.Join(dir, "run-1") + require.NoError(t, os.Mkdir(runDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(runDir, "lightbringer.log"), []byte("line 1\nline 2\nline 3\n"), 0600)) + require.NoError(t, os.Symlink("run-1", filepath.Join(dir, "latest"))) + + lines := lightbringerLogLines(&configData{logsPath: dir, lbEnabled: true, lbQuiet: true}, 3) + + require.Len(t, lines, 3) + assert.Contains(t, lines[0], "quiet mode") + assert.NotContains(t, lines[0], "line 1") + assert.Contains(t, lines[1], "line 2") + assert.Contains(t, lines[2], "line 3") +} + +func TestReadProgressEventsReadsLatestRun(t *testing.T) { + dir := t.TempDir() + runDir := filepath.Join(dir, "run-1") + require.NoError(t, os.Mkdir(runDir, 0700)) + body := strings.Join([]string{ + `{"phase":"starting","status":"running","message":"Mithril process started","ts":"2026-06-01T00:00:00Z"}`, + `not-json`, + `{"phase":"bootstrap_snapshot","status":"running","message":"Downloading snapshot","slot":123,"endpoint":"https://rpc.invalid/?api-key=test-key-00000000"}`, + }, "\n") + require.NoError(t, os.WriteFile(filepath.Join(runDir, progress.JSONLFileName), []byte(body), 0600)) + require.NoError(t, os.Symlink("run-1", filepath.Join(dir, "latest"))) + + events := readProgressEvents(dir, 10) + + require.Len(t, events, 2) + assert.Equal(t, "starting", events[0].Phase) + assert.Equal(t, "bootstrap_snapshot", events[1].Phase) + assert.Equal(t, float64(123), events[1].Fields["slot"]) + assert.Contains(t, events[1].Fields["endpoint"], "api-key=REDACTED") + assert.NotContains(t, events[1].Fields["endpoint"], "test-key-00000000") +} + +func TestReadSnapshotActivity_ChoosesNewestSnapshotArtifact(t *testing.T) { + dir := t.TempDir() + oldPath := filepath.Join(dir, "snapshot-100-old.tar.zst") + newPath := filepath.Join(dir, "snapshot-200-new.tar.zst.partial") + require.NoError(t, os.WriteFile(oldPath, []byte("old"), 0600)) + require.NoError(t, os.WriteFile(newPath, []byte("newer-data"), 0600)) + require.NoError(t, os.Chtimes(oldPath, time.Now().Add(-time.Hour), time.Now().Add(-time.Hour))) + + activity := readSnapshotActivity(dir) + + assert.Equal(t, newPath, activity.Path) + assert.Equal(t, "snapshot-200-new.tar.zst.partial", activity.Name) + assert.True(t, activity.Partial) + assert.Equal(t, int64(len("newer-data")), activity.Bytes) +} + +func TestReadSnapshotActivity_AcceptsExplicitSnapshotFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "snapshot-300-test.tar.zst") + require.NoError(t, os.WriteFile(path, []byte("snapshot-data"), 0600)) + + activity := readSnapshotActivity(path) + + assert.Equal(t, path, activity.Path) + assert.Equal(t, "snapshot-300-test.tar.zst", activity.Name) + assert.False(t, activity.Partial) + assert.Equal(t, int64(len("snapshot-data")), activity.Bytes) +} + +func TestReadAccountsActivity_ChoosesNewestKnownArtifact(t *testing.T) { + dir := t.TempDir() + accountsDir := filepath.Join(dir, "accounts") + shardsDir := filepath.Join(dir, "mithril_db_log_shards") + require.NoError(t, os.Mkdir(accountsDir, 0700)) + require.NoError(t, os.Mkdir(shardsDir, 0700)) + oldTime := time.Now().Add(-2 * time.Hour) + newTime := time.Now().Add(-time.Hour) + require.NoError(t, os.Chtimes(dir, oldTime, oldTime)) + require.NoError(t, os.Chtimes(accountsDir, oldTime, oldTime)) + require.NoError(t, os.Chtimes(shardsDir, newTime, newTime)) + + activity := readAccountsActivity(dir) + + assert.Equal(t, shardsDir, activity.Path) + assert.Equal(t, "mithril_db_log_shards", activity.Name) +} + +func TestExistingDiskProbePath_UsesNearestExistingParent(t *testing.T) { + dir := t.TempDir() + missingChild := filepath.Join(dir, "not-yet-created", "accounts") + + assert.Equal(t, dir, existingDiskProbePath(missingChild)) +} + +func TestDescribeLightbringerGossipPorts_DefaultsAndValidation(t *testing.T) { + msg, err := describeLightbringerGossipPorts("", "", "") + require.NoError(t, err) + assert.Contains(t, msg, "gossip=65400") + assert.Contains(t, msg, "65401-65500") + + _, err = describeLightbringerGossipPorts("55010", "55001", "55100") + assert.ErrorContains(t, err, "must not overlap") +} + +func TestDisplayConfigValue_RedactsRPCSecrets(t *testing.T) { + got := displayConfigValue( + "network", + "rpc", + "https://rpc.example.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000, https://example.invalid/rpc?network=mainnet", + ) + + assert.Contains(t, got, "api-key=REDACTED") + assert.NotContains(t, got, "test-key-00000000") + assert.Contains(t, got, "network=mainnet") +} + +func TestRenderEditList_RedactsRPCSecretsInPassiveView(t *testing.T) { + m := newModel("config.toml") + m.height = 40 + m.cfg = &configData{ + rpcEndpoints: []string{"https://rpc.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000"}, + } + + out := m.renderEditList() + assert.Contains(t, out, "api-key=RED") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestRenderEditFocused_RedactsRPCSecretsWhileEditing(t *testing.T) { + m := newModel("config.toml") + m.cfg = &configData{ + rpcEndpoints: []string{"https://rpc.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000"}, + } + for i, field := range m.editFields { + if field.section == "network" && field.key == "rpc" { + m.editIdx = i + break + } + } + m.editMode = editText + m.editValue = m.cfg.rpcEndpoints[0] + m.editCursor = len(m.editValue) + + out := m.renderEditFocused() + assert.Contains(t, out, "api-key=REDACTED") + assert.Contains(t, out, "Sensitive URL values are hidden") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestColorLogLine_RedactsEndpointSecrets(t *testing.T) { + out := colorLogLine("Reference slot from https://rpc.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000") + + assert.Contains(t, out, "api-key=REDACTED") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestRenderLogsView_RedactsBeforeWrapping(t *testing.T) { + secretURL := "https://rpc.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000" + m := newModel("config.toml") + m.hasConfig = true + m.width = 120 + m.height = 30 + m.screen = screenLogs + m.mithrilLines = []string{"Reference slot from " + secretURL} + + out := m.renderLogsView() + assert.Contains(t, out, "REDACTED") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestRenderRawLogsView_RedactsBeforeWrapping(t *testing.T) { + secretURL := "https://rpc.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000" + m := newModel("config.toml") + m.hasConfig = true + m.width = 120 + m.height = 30 + m.screen = screenLogs + m.logRawMode = true + m.mithrilLines = []string{"Mithril repair RPC " + secretURL} + + out := m.renderRawLogsView() + assert.Contains(t, out, "REDACTED") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestMithrilOnlyLogsUseTerminalModeWithoutLightbringerNoise(t *testing.T) { + m := newModel("config.toml") + m.hasConfig = true + m.cfg = &configData{blockSource: "rpc"} + m.width = 100 + m.height = 30 + m.screen = screenLogs + m.mithrilLines = []string{"INFO replay running"} + + out := m.renderLogsView() + + assert.Contains(t, out, "terminal logs") + assert.Contains(t, out, "Mithril live tail") + assert.Contains(t, out, "INFO replay running") + assert.NotContains(t, out, "Lightbringer disabled") + assert.NotContains(t, out, "[lightbringer]") + assert.NotContains(t, out, "split view") +} + +func TestViewFitsTerminalFrameWithRawProgressLogs(t *testing.T) { + m := newModel("config.toml") + m.hasConfig = true + m.width = 100 + m.height = 24 + m.screen = screenLogs + m.logRawMode = true + m.mithrilLines = []string{ + "Snapshot Read (.tar.zst) 0.1% 0.1/45.4 GB ETA 7m30s\rSnapshot Read (.tar.zst) 0.8% 0.4/45.4 GB ETA 7m29s", + "Extract (AppendVecs) 0.2% 0.5/340.0 GB ETA 7m10s\rExtract (AppendVecs) 1.0% 3.5/339.7 GB ETA 7m05s", + } + + out := m.View() + + assert.NotContains(t, out, "\r") + assert.LessOrEqual(t, lipgloss.Height(out), m.height) + for _, line := range strings.Split(out, "\n") { + assert.LessOrEqual(t, lipgloss.Width(line), m.width) + } +} + +func TestFullWidthRawLogsHideMenuInView(t *testing.T) { + m := newModel("config.toml") + m.hasConfig = true + m.cfg = &configData{blockSource: "rpc"} + m.width = 120 + m.height = 30 + m.screen = screenLogs + m.logRawMode = true + m.mithrilLines = []string{"slot 425544688 | leader: sample-validator | txns: v:708 nv:384 | exec: 0.486s"} + + out := m.View() + + assert.Contains(t, out, "terminal logs") + assert.Contains(t, out, "slot 425544688") + assert.NotContains(t, out, "Run Node") + assert.NotContains(t, out, "Edit Config") + for _, line := range strings.Split(out, "\n") { + assert.LessOrEqual(t, lipgloss.Width(line), m.width) + } +} + +func TestDashboardRawLogLinesCompactsReplaySlotRows(t *testing.T) { + raw := "(+ 45.607s) slot 425556344 | leader: PUmpKiNnSVAZ3w4KaFX6jKSjXUNHFShGkXbERo54xjb | txns: v:712 nv:1402 | cu: 50580776 | exec: 0.608s | wait: 0.000s | total: 0.608s" + + lines := dashboardRawLogLines([]string{raw}, 78) + + require.Len(t, lines, 1) + assert.Contains(t, lines[0], "slot 425,556,344") + assert.Contains(t, lines[0], "txns v712/nv1402") + assert.Contains(t, lines[0], "exec 0.608s") + assert.Contains(t, lines[0], "cu 50.6M") + assert.NotContains(t, lines[0], ") | slot") + assert.NotContains(t, lines[0], "leader:") + assert.LessOrEqual(t, lipgloss.Width(lines[0]), 78) +} + +func TestDashboardRawLogLinesKeepsNonSlotErrorsVerbose(t *testing.T) { + raw := "ERROR: Replay stopped before persisting the first post-start slot: state file missing manifest_epoch_authorized_voters - delete AccountsDB and rebuild from snapshot" + + lines := dashboardRawLogLines([]string{raw}, 60) + joined := strings.Join(lines, "\n") + + assert.Contains(t, joined, "ERROR: Replay stopped") + assert.Contains(t, joined, "delete AccountsDB") + assert.Greater(t, len(lines), 1) +} + +func TestVisibleLogWindowDefaultsToNewestLines(t *testing.T) { + lines := []string{"line-00", "line-01", "line-02", "line-03", "line-04", "line-05"} + + assert.Equal(t, []string{"line-03", "line-04", "line-05"}, visibleLogWindow(lines, 3, 0)) + assert.Equal(t, []string{"line-01", "line-02", "line-03"}, visibleLogWindow(lines, 3, 2)) + assert.Equal(t, []string{"line-00", "line-01", "line-02"}, visibleLogWindow(lines, 3, 99)) +} + +func TestRenderRawLogsViewShowsNewestLinesByDefault(t *testing.T) { + m := newModel("config.toml") + m.hasConfig = true + m.width = 100 + m.height = 28 + m.screen = screenLogs + m.logRawMode = true + for i := 0; i < 20; i++ { + m.mithrilLines = append(m.mithrilLines, fmt.Sprintf("line-%02d", i)) + } + + out := m.renderRawLogsView() + + assert.Contains(t, out, "line-19") + assert.NotContains(t, out, "line-00") +} + +func TestCombinedRawLogLinesInterleavesSourcesFromTail(t *testing.T) { + m := newModel("config.toml") + m.cfg = &configData{lbEnabled: true} + for i := 0; i < 12; i++ { + m.lbLines = append(m.lbLines, fmt.Sprintf("lb-%02d", i)) + } + m.mithrilLines = []string{"mithril-00", "mithril-01"} + + lines := m.combinedRawLogLines() + tail := strings.Join(lines[len(lines)-6:], "\n") + + assert.Contains(t, tail, "[mithril] mithril-00") + assert.Contains(t, tail, "[mithril] mithril-01") + assert.Contains(t, tail, "[lightbringer] lb-11") +} + +func TestDataRefreshAnchorsFocusedLogScrollback(t *testing.T) { + m := newModel("config.toml") + m.width = 100 + m.height = 28 + m.screen = screenLogs + m.logFocused = true + m.logScroll = 2 + for i := 0; i < 10; i++ { + m.mithrilLines = append(m.mithrilLines, fmt.Sprintf("old-%02d", i)) + } + + var nextLines []string + for i := 0; i < 13; i++ { + nextLines = append(nextLines, fmt.Sprintf("new-%02d", i)) + } + next, _ := m.Update(dataRefreshedMsg{mithrilLines: nextLines}) + updated := next.(model) + + assert.Equal(t, 5, updated.logScroll) +} + +func TestWrapLogLinesPreservesUnicodeProgressGlyphs(t *testing.T) { + lines := wrapLogLines([]string{ + "Snapshot Read (.tar.zst) [██░░░░░░░░░░░░░░░░░░░░] 2.9% 3.2/108.1 GB 112.3 MB/s ETA 17m14s", + }, 36) + + joined := strings.Join(lines, "\n") + assert.Contains(t, joined, "██") + assert.Contains(t, joined, "░░") + assert.NotContains(t, joined, "�") +} + +func TestWrapLogLinesPrefersWordBoundaries(t *testing.T) { + lines := wrapLogLines([]string{ + "snapshot bootstrap cancelled during shutdown: failed to build AccountsDB from snapshot: processing full snapshot: context canceled", + }, 48) + + joined := strings.Join(lines, "\n") + assert.NotContains(t, joined, "fai\n led") + assert.NotContains(t, joined, "snap\n shot") +} + +func TestViewFitsTerminalFrameAcrossLogResize(t *testing.T) { + for _, size := range []struct { + width int + height int + }{ + {width: 72, height: 22}, + {width: 100, height: 24}, + {width: 150, height: 36}, + } { + m := newModel("config.toml") + m.hasConfig = true + m.cfg = &configData{blockSource: "rpc"} + m.width = size.width + m.height = size.height + m.screen = screenLogs + m.mithrilLines = []string{ + "INFO " + strings.Repeat("long-log-field ", 20), + "ERROR " + strings.Repeat("stall-heartbeat ", 18), + } + + out := m.View() + + assert.LessOrEqual(t, lipgloss.Height(out), size.height, "height=%dx%d", size.width, size.height) + for _, line := range strings.Split(out, "\n") { + assert.LessOrEqual(t, lipgloss.Width(line), size.width, "width=%dx%d line=%q", size.width, size.height, line) + } + } +} + +func TestApplyEditField_ClearingExternalLightbringerEndpointReturnsToRPC(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[block] +source = "lightbringer" +lightbringer_endpoint = "127.0.0.1:3001" + +[lightbringer] +enabled = false +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, field := range m.editFields { + if field.section == "block" && field.key == "lightbringer_endpoint" { + m.editIdx = i + break + } + } + require.Equal(t, "block", m.editFields[m.editIdx].section) + require.Equal(t, "lightbringer_endpoint", m.editFields[m.editIdx].key) + + m.editValue = "" + m.applyEditField() + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `lightbringer_endpoint = ""`) + assert.Contains(t, body, `source = "rpc"`) + assert.Contains(t, body, `enabled = false`) + assert.False(t, strings.Contains(body, `source = "lightbringer"`)) +} + +// Negative path: clearing the LB endpoint must NOT rewrite a turbine source to rpc. +func TestApplyEditField_ClearingEndpointPreservesTurbineSource(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[block] +source = "turbine" +lightbringer_endpoint = "127.0.0.1:3001" + +[lightbringer] +enabled = false +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, field := range m.editFields { + if field.section == "block" && field.key == "lightbringer_endpoint" { + m.editIdx = i + break + } + } + m.editValue = "" + m.applyEditField() + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `source = "turbine"`, "turbine source must survive clearing the LB endpoint") + assert.False(t, strings.Contains(body, `source = "rpc"`)) +} + +// Selecting block.source=turbine must disable the managed Lightbringer so an +// unused sidecar doesn't spawn and open public UDP ports. +func TestApplyMenuSelection_TurbineSourceDisablesLightbringer(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[block] +source = "lightbringer" + +[lightbringer] +enabled = true +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, f := range m.editFields { + if f.section == "block" && f.key == "source" { + m.editIdx = i + break + } + } + m.editOptions = menuOptionsFor("block", "source") + for i, o := range m.editOptions { + if o.value == "turbine" { + m.editOptCursor = i + break + } + } + m.applyMenuSelection() + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `source = "turbine"`) + assert.Contains(t, body, `enabled = false`, "selecting turbine must disable the managed Lightbringer") +} + +// Disabling the managed Lightbringer via the enabled toggle must not clobber a turbine source. +func TestApplyMenuSelection_DisablingLBPreservesTurbineSource(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[block] +source = "turbine" + +[lightbringer] +enabled = true +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, f := range m.editFields { + if f.section == "lightbringer" && f.key == "enabled" { + m.editIdx = i + break + } + } + m.editOptions = menuOptionsFor("lightbringer", "enabled") + for i, o := range m.editOptions { + if o.value == "false" { + m.editOptCursor = i + break + } + } + m.applyMenuSelection() + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `source = "turbine"`, "disabling LB must not clobber a turbine source") + assert.False(t, strings.Contains(body, `source = "rpc"`)) +} + +// Positive path: disabling LB while source was "lightbringer" must fall back to rpc. +func TestApplyMenuSelection_DisablingLBFromLightbringerSetsRPC(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[block] +source = "lightbringer" + +[lightbringer] +enabled = true +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, f := range m.editFields { + if f.section == "lightbringer" && f.key == "enabled" { + m.editIdx = i + break + } + } + m.editOptions = menuOptionsFor("lightbringer", "enabled") + for i, o := range m.editOptions { + if o.value == "false" { + m.editOptCursor = i + break + } + } + m.applyMenuSelection() + + content, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(content), `source = "rpc"`, "disabling LB from lightbringer mode must fall back to rpc") +} + +func TestApplyEditField_SnapshotsPathClearsShadowingDownloadPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" +rpc = ["https://example.invalid"] + +[storage] +snapshots = "/old/storage-snapshots" + +[snapshot] +download_path = "/old/download-path" +`), 0600)) + + m := newModel(path) + m.cfg = readConfig(path) + for i, field := range m.editFields { + if field.section == "storage" && field.key == "snapshots" { + m.editIdx = i + break + } + } + m.editValue = "/new/snapshots" + m.applyEditField() + + content, err := os.ReadFile(path) + require.NoError(t, err) + body := string(content) + assert.Contains(t, body, `snapshots = "/new/snapshots"`) + assert.Contains(t, body, `# download_path = "/old/download-path"`) +} + +func TestRunDoctorChecks_WarnsOnPublicMainnetRPC(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(`[network]`), 0600)) + + for _, endpoint := range []string{ + "https://api.mainnet-beta.solana.com", + "https://api.mainnet.solana.com", + } { + checks := runDoctorChecks(path, &configData{ + cluster: "mainnet-beta", + rpcEndpoints: []string{endpoint}, + accountsPath: "/tmp/mithril-accounts", + blockSource: "rpc", + }) + + found := false + for _, check := range checks { + if check.name == "RPC capacity" { + found = true + assert.Equal(t, "warn", check.status) + assert.Contains(t, check.msg, "private RPC") + } + } + assert.Truef(t, found, "doctor should warn before long mainnet runs use public RPC: %s", endpoint) + } +} + +func TestLatestReplaySlot(t *testing.T) { + cases := []struct { + name string + lines []string + want uint64 + ok bool + }{ + {"per-slot replay line", []string{"(+ 3m24.830s) slot 426087845 | leader: X | txns: v:716"}, 426087845, true}, + {"picks the latest (newest last)", []string{ + "(+1s) slot 100 | leader: A", + "(+2s) slot 200 | leader: B", + }, 200, true}, + {"ignores snapshot slot (no pipe)", []string{"building from snapshot slot 426004874"}, 0, false}, + {"no slot at all", []string{"Probing nodes for snapshot availability..."}, 0, false}, + {"empty", nil, 0, false}, + {"skips trailing non-slot line back to last replay line", []string{ + "(+1s) slot 999 | leader: A", + "AccountsDB is ready", + }, 999, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok := latestReplaySlot(c.lines) + if ok != c.ok || got != c.want { + t.Errorf("latestReplaySlot = (%d,%v), want (%d,%v)", got, ok, c.want, c.ok) + } + }) + } +} diff --git a/cmd/mithril/dashboardcmd/download_bar.go b/cmd/mithril/dashboardcmd/download_bar.go new file mode 100644 index 000000000..f1e207fc1 --- /dev/null +++ b/cmd/mithril/dashboardcmd/download_bar.go @@ -0,0 +1,219 @@ +package dashboardcmd + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/Overclock-Validator/mithril/pkg/tui" + "github.com/charmbracelet/lipgloss" +) + +// downloadProgress is one parsed snapshot/extract progress sample, e.g. +// "Snapshot: 8.3% 9.0/107.5 GB 89.6 MB/s ETA 18m47s". Rendered as a progress bar. +type downloadProgress struct { + label string // "Snapshot" or "Extract" + percent float64 // 0..100 + current string // "9.0" + total string // "107.5" + unit string // "GB" + rate string // "89.6 MB/s" + eta string // "18m47s" +} + +// Matches a progress line anywhere in a log line (tolerates a leading +// "(+ 1m03s) " elapsed prefix). +var downloadProgressRe = regexp.MustCompile( + `(?i)(Snapshot|Extract):\s+([0-9]+(?:\.[0-9]+)?)%\s+` + + `([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)\s*([KMGT]?i?B)\s+` + + `([0-9]+(?:\.[0-9]+)?\s*[KMGT]?i?B/s)\s+ETA\s+([0-9hms]+)`) + +// parseDownloadProgress extracts a progress sample from a single log line. +func parseDownloadProgress(line string) (downloadProgress, bool) { + m := downloadProgressRe.FindStringSubmatch(line) + if m == nil { + return downloadProgress{}, false + } + pct, err := strconv.ParseFloat(m[2], 64) + if err != nil { + return downloadProgress{}, false + } + if pct < 0 { + pct = 0 + } + if pct > 100 { + pct = 100 + } + label := "Extract" + if strings.EqualFold(m[1], "snapshot") { + label = "Snapshot" + } + return downloadProgress{ + label: label, + percent: pct, + current: m[3], + total: m[4], + unit: m[5], + rate: strings.Join(strings.Fields(m[6]), " "), + eta: m[7], + }, true +} + +// isDownloadProgressLine reports whether a log line is a progress sample. +func isDownloadProgressLine(line string) bool { + return downloadProgressRe.MatchString(line) +} + +// latestDownloadProgress returns the newest Snapshot and Extract samples +// (Snapshot first), omitting labels that don't appear. +func latestDownloadProgress(lines []string) []downloadProgress { + var snap, ext *downloadProgress + for i := len(lines) - 1; i >= 0; i-- { + p, ok := parseDownloadProgress(lines[i]) + if !ok { + continue + } + switch { + case strings.EqualFold(p.label, "Snapshot") && snap == nil: + c := p + snap = &c + case strings.EqualFold(p.label, "Extract") && ext == nil: + c := p + ext = &c + } + if snap != nil && ext != nil { + break + } + } + out := make([]downloadProgress, 0, 2) + if snap != nil { + out = append(out, *snap) + } + if ext != nil { + out = append(out, *ext) + } + return out +} + +// filterDownloadProgressLines drops progress samples from display lines; they +// show as a bar instead, so keeping them would duplicate and flood scrollback. +func filterDownloadProgressLines(lines []string) []string { + out := lines[:0:0] // new backing array, never mutate caller's slice + for _, l := range lines { + if isDownloadProgressLine(l) { + continue + } + out = append(out, l) + } + return out +} + +const downloadBarLabelWidth = 10 // fits "Snapshot" and "AccountsDB" + +// maxDownloadBarWidth caps the filled bar so it stays tidy on wide terminals. +const maxDownloadBarWidth = 48 + +// renderDownloadBar renders one progress bar within width columns. As width +// shrinks it drops rate/ETA, then byte counts, then the bar itself. +func renderDownloadBar(p downloadProgress, width int) string { + teal := lipgloss.NewStyle().Foreground(tui.MithrilTeal) + empty := lipgloss.NewStyle().Foreground(tui.ColorBorder) + labelSt := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary).Bold(true) + pctSt := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + statSt := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + + name := "Snapshot" + if p.label == "Extract" { + name = "AccountsDB" + } + label := padRightPlain(name, downloadBarLabelWidth) + pct := fmt.Sprintf("%5.1f%%", p.percent) + stats := fmt.Sprintf("%s/%s %s", p.current, p.total, p.unit) + rateEta := fmt.Sprintf("%s · ETA %s", p.rate, p.eta) + + const indent = 2 + // Mandatory: indent + label + " " + bar + " " + pct. + fixed := indent + lipgloss.Width(label) + 1 + 1 + lipgloss.Width(pct) + + // Trailing detail, only when there's room (most-droppable last). + suffix := "" + if width-fixed-2-lipgloss.Width(stats) >= 6 { + suffix = " " + stats + if width-fixed-lipgloss.Width(suffix)-2-lipgloss.Width(rateEta) >= 0 { + suffix += " " + rateEta + } + } + + barWidth := width - fixed - lipgloss.Width(suffix) + if barWidth > maxDownloadBarWidth { + barWidth = maxDownloadBarWidth + } + if barWidth < 4 { + // Too narrow for a bar — compact "label pct", truncated to fit. + plain := name + " " + strings.TrimSpace(pct) + avail := width - indent + if avail < 0 { + avail = 0 + } + plain = truncatePlain(plain, avail) + return strings.Repeat(" ", indent) + labelSt.Render(plain) + } + + filled := int(float64(barWidth)*p.percent/100.0 + 0.5) + if filled > barWidth { + filled = barWidth + } + if filled < 0 { + filled = 0 + } + bar := teal.Render(strings.Repeat("█", filled)) + empty.Render(strings.Repeat("░", barWidth-filled)) + + return strings.Repeat(" ", indent) + + labelSt.Render(label) + " " + + bar + " " + + pctSt.Render(pct) + + statSt.Render(suffix) +} + +// renderDownloadProgress returns a progress block for the latest snapshot +// activity, or "" if none is active. +func renderDownloadProgress(lines []string, width int) string { + bars := latestDownloadProgress(lines) + if len(bars) == 0 { + return "" + } + if width < 12 { + width = 12 + } + head := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + var b strings.Builder + b.WriteString(" " + head.Render("Snapshot bootstrap — downloading") + "\n") + for _, p := range bars { + b.WriteString(renderDownloadBar(p, width) + "\n") + } + return b.String() +} + +// truncatePlain truncates an unstyled string to w columns with an ellipsis. +func truncatePlain(s string, w int) string { + if w <= 0 { + return "" + } + r := []rune(s) + if len(r) <= w { + return s + } + if w == 1 { + return string(r[:1]) + } + return string(r[:w-1]) + "…" +} + +// padRightPlain pads s with spaces to at least n display columns. +func padRightPlain(s string, n int) string { + if w := lipgloss.Width(s); w < n { + return s + strings.Repeat(" ", n-w) + } + return s +} diff --git a/cmd/mithril/dashboardcmd/download_bar_test.go b/cmd/mithril/dashboardcmd/download_bar_test.go new file mode 100644 index 000000000..6955328d7 --- /dev/null +++ b/cmd/mithril/dashboardcmd/download_bar_test.go @@ -0,0 +1,163 @@ +package dashboardcmd + +import ( + "strings" + "testing" + + "github.com/charmbracelet/lipgloss" +) + +func TestParseDownloadProgress(t *testing.T) { + cases := []struct { + name string + line string + ok bool + label string + percent float64 + current string + total string + unit string + rate string + eta string + }{ + { + name: "snapshot line", + line: "Snapshot: 8.3% 9.0/107.5 GB 89.6 MB/s ETA 18m47s", + ok: true, + label: "Snapshot", + percent: 8.3, current: "9.0", total: "107.5", unit: "GB", rate: "89.6 MB/s", eta: "18m47s", + }, + { + name: "extract line", + line: "Extract: 5.7% 24.2/424.6 GB 393.5 MB/s ETA 17m22s", + ok: true, + label: "Extract", + percent: 5.7, current: "24.2", total: "424.6", unit: "GB", rate: "393.5 MB/s", eta: "17m22s", + }, + { + name: "with elapsed prefix", + line: "(+ 1m03s) Snapshot: 100.0% 107.5/107.5 GB 110.0 MB/s ETA 0s", + ok: true, + label: "Snapshot", + percent: 100, current: "107.5", total: "107.5", unit: "GB", rate: "110.0 MB/s", eta: "0s", + }, + {name: "not a progress line", line: "(+ 0s) Probing 321 nodes for snapshot availability...", ok: false}, + {name: "build-phase appendvec line is not download", line: "Extract (AppendVecs) [##### ] 50%", ok: false}, + {name: "empty", line: "", ok: false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p, ok := parseDownloadProgress(c.line) + if ok != c.ok { + t.Fatalf("ok=%v want %v (line=%q)", ok, c.ok, c.line) + } + if !c.ok { + return + } + if p.label != c.label || p.percent != c.percent || p.current != c.current || p.total != c.total || + p.unit != c.unit || p.rate != c.rate || p.eta != c.eta { + t.Fatalf("parsed %+v, want label=%s pct=%v current=%s total=%s unit=%s rate=%s eta=%s", + p, c.label, c.percent, c.current, c.total, c.unit, c.rate, c.eta) + } + }) + } +} + +func TestParseDownloadProgressClampsPercent(t *testing.T) { + p, ok := parseDownloadProgress("Snapshot: 250.0% 9/9 GB 1 MB/s ETA 0s") + if !ok || p.percent != 100 { + t.Fatalf("expected clamp to 100, got %v ok=%v", p.percent, ok) + } +} + +func TestLatestDownloadProgress_NewestWins(t *testing.T) { + lines := []string{ + "Snapshot: 1.0% 1/107 GB 90 MB/s ETA 20m", + "Extract: 1.0% 4/424 GB 390 MB/s ETA 20m", + "Snapshot: 8.3% 9.0/107.5 GB 89.6 MB/s ETA 18m47s", + "Extract: 8.3% 35/424 GB 393 MB/s ETA 17m", + "some other log line", + } + got := latestDownloadProgress(lines) + if len(got) != 2 { + t.Fatalf("expected 2 bars, got %d", len(got)) + } + if got[0].label != "Snapshot" || got[0].percent != 8.3 { + t.Fatalf("snapshot bar wrong: %+v", got[0]) + } + if got[1].label != "Extract" || got[1].percent != 8.3 { + t.Fatalf("extract bar wrong: %+v", got[1]) + } +} + +func TestLatestDownloadProgress_None(t *testing.T) { + if got := latestDownloadProgress([]string{"a", "b", "c"}); len(got) != 0 { + t.Fatalf("expected none, got %d", len(got)) + } +} + +func TestFilterDownloadProgressLines(t *testing.T) { + in := []string{ + "(+ 0s) starting", + "Snapshot: 8.3% 9.0/107.5 GB 89.6 MB/s ETA 18m47s", + "Extract: 8.3% 35/424 GB 393 MB/s ETA 17m", + "(+ 1m) building", + } + out := filterDownloadProgressLines(in) + if len(out) != 2 || out[0] != "(+ 0s) starting" || out[1] != "(+ 1m) building" { + t.Fatalf("filter wrong: %#v", out) + } + // caller slice must be untouched + if len(in) != 4 { + t.Fatalf("filter mutated input: %#v", in) + } +} + +// Resize guard: the rendered bar never exceeds its width budget. +func TestRenderDownloadBar_NeverOverflows(t *testing.T) { + p := downloadProgress{label: "Snapshot", percent: 63.4, current: "68.1", total: "107.5", unit: "GB", rate: "112.0 MB/s", eta: "5m58s"} + for w := 8; w <= 200; w++ { + got := renderDownloadBar(p, w) + if width := lipgloss.Width(got); width > w { + t.Fatalf("width %d: rendered display width %d exceeds budget\n%q", w, width, got) + } + } +} + +func TestRenderDownloadBar_DegradesNarrow(t *testing.T) { + p := downloadProgress{label: "Extract", percent: 50, current: "1", total: "2", unit: "GB", rate: "1 MB/s", eta: "1m"} + // wide: should include the bar blocks and the ETA detail + wide := renderDownloadBar(p, 120) + if !strings.Contains(wide, "█") || !strings.Contains(wide, "ETA") { + t.Fatalf("wide bar missing fill/eta: %q", wide) + } + // narrow: still renders the percent + narrow := renderDownloadBar(p, 18) + if !strings.Contains(narrow, "50.0%") && !strings.Contains(narrow, "50") { + t.Fatalf("narrow bar missing percent: %q", narrow) + } +} + +func TestRenderDownloadProgress_EmptyWhenInactive(t *testing.T) { + if s := renderDownloadProgress([]string{"nothing here"}, 100); s != "" { + t.Fatalf("expected empty, got %q", s) + } +} + +func TestRenderDownloadProgress_ShowsBothBars(t *testing.T) { + lines := []string{ + "Snapshot: 8.3% 9.0/107.5 GB 89.6 MB/s ETA 18m47s", + "Extract: 8.3% 35/424 GB 393 MB/s ETA 17m", + } + out := renderDownloadProgress(lines, 100) + // 18m47s == Snapshot bar, AccountsDB == Extract bar (header word "Snapshot" would false-match) + if !strings.Contains(out, "18m47s") || !strings.Contains(out, "AccountsDB") { + t.Fatalf("expected both bars, got:\n%s", out) + } + // every rendered row must be within the width budget + for _, row := range strings.Split(strings.TrimRight(out, "\n"), "\n") { + if w := lipgloss.Width(row); w > 100 { + t.Fatalf("row exceeds width: %d\n%q", w, row) + } + } +} diff --git a/cmd/mithril/dashboardcmd/field_help_test.go b/cmd/mithril/dashboardcmd/field_help_test.go new file mode 100644 index 000000000..dbba43810 --- /dev/null +++ b/cmd/mithril/dashboardcmd/field_help_test.go @@ -0,0 +1,28 @@ +package dashboardcmd + +import ( + "strings" + "testing" +) + +// Every non-separator editable field has plain-language help. +func TestFieldHelp_CoversEveryEditableField(t *testing.T) { + m := newModel("config.toml") // same field list the UI uses + for _, f := range m.editFields { + if f.isSep { + continue + } + if h := fieldHelp(f.section, f.key); strings.TrimSpace(h) == "" { + t.Errorf("no plain-language help for %s.%s (%q)", f.section, f.key, f.label) + } + } +} + +func TestFieldHelp_FlagsFirewallSensitiveFields(t *testing.T) { + // RPC port and Lightbringer UDP ports have firewall implications. + for _, kv := range [][2]string{{"rpc", "port"}, {"lightbringer", "gossip_port"}, {"lightbringer", "port_range_start"}} { + if !strings.Contains(strings.ToLower(fieldHelp(kv[0], kv[1])), "firewall") { + t.Errorf("%s.%s help should mention firewall", kv[0], kv[1]) + } + } +} diff --git a/cmd/mithril/dashboardcmd/platform_linux.go b/cmd/mithril/dashboardcmd/platform_linux.go new file mode 100644 index 000000000..20267499e --- /dev/null +++ b/cmd/mithril/dashboardcmd/platform_linux.go @@ -0,0 +1,27 @@ +//go:build linux + +package dashboardcmd + +import "syscall" + +// cleanupOrphanLightbringer is a no-op on Linux: Pdeathsig makes the kernel +// SIGTERM the child when mithril is force-killed. +func cleanupOrphanLightbringer() {} + +// spawnSysProcAttr detaches the `mithril run` child via Setsid (new session) so +// it outlives the dashboard. Pdeathsig is left unset. +func spawnSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setsid: true} +} + +// statFSType returns the filesystem's f_type magic number (e.g. NFS_SUPER_MAGIC), +// used to refuse PID dirs on network filesystems where flock is unreliable. +func statFSType(path string) (int64, bool) { + var s syscall.Statfs_t + if err := syscall.Statfs(path, &s); err != nil { + return 0, false + } + // uint32 cast keeps high-bit magics (CIFS, SMB2) positive; a direct int64 + // would sign-extend them to negative on 32-bit Linux. + return int64(uint32(s.Type)), true +} diff --git a/cmd/mithril/dashboardcmd/platform_other.go b/cmd/mithril/dashboardcmd/platform_other.go new file mode 100644 index 000000000..b0842519d --- /dev/null +++ b/cmd/mithril/dashboardcmd/platform_other.go @@ -0,0 +1,20 @@ +//go:build !linux + +package dashboardcmd + +import "syscall" + +// cleanupOrphanLightbringer is a no-op off Linux: no safe PID identity to prove +// which orphan is ours. Cleanup is left to an explicit operator step. +func cleanupOrphanLightbringer() {} + +// spawnSysProcAttr detaches the child via Setsid (supported on macOS/BSD too). +func spawnSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setsid: true} +} + +// statFSType: no-op off Linux (no portable filesystem-magic access). +func statFSType(path string) (int64, bool) { + _ = path + return 0, false +} diff --git a/cmd/mithril/dashboardcmd/preflight.go b/cmd/mithril/dashboardcmd/preflight.go new file mode 100644 index 000000000..feb32444a --- /dev/null +++ b/cmd/mithril/dashboardcmd/preflight.go @@ -0,0 +1,333 @@ +// Pre-flight checks run before spawning mithril: systemd conflicts, network +// filesystems where flock is unreliable, and root/non-root ownership mistakes. + +package dashboardcmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" +) + +// preflightCheck returns "" if Start is allowed, else an operator-friendly +// refusal shown verbatim in the Process view. +func preflightCheck(cfg *configData, accountsDir string) string { + var reasons []string + if reason := checkSystemd(); reason != "" { + reasons = append(reasons, reason) + } + if reason := checkStateDirFilesystem(); reason != "" { + reasons = append(reasons, reason) + } + if reason := checkUIDMatch(accountsDir); reason != "" { + reasons = append(reasons, reason) + } + if cfg != nil { + if reason := checkLogPathWritable(cfg.logsPath); reason != "" { + reasons = append(reasons, reason) + } + if reason := checkAccountsStateCompatible(cfg, accountsDir); reason != "" { + reasons = append(reasons, reason) + } + } + return strings.Join(reasons, "\n\n") +} + +// checkSystemd refuses only when a running mithril is supervised by systemd +// (per its cgroup); a fresh start is always allowed. +func checkSystemd() string { + info, err := procctl.ReadPidFile(procctl.DefaultPidFile()) + if err != nil { + return "" // no PID file → no supervisor + } + cgroupPath := fmt.Sprintf("/proc/%d/cgroup", info.Pid) + data, err := os.ReadFile(cgroupPath) + if err != nil { + return "" // process gone or no /proc (macOS) — nothing to block + } + if info.SpawnedBy == "dashboard" { + return "" + } + if _, ok := mithrilSystemdUnit(string(data)); ok { + return strings.Join([]string{ + "Mithril is managed by systemd. The dashboard would fight", + "systemd's restart loop. To control it, use:", + "", + " sudo systemctl stop mithril", + " sudo systemctl restart mithril", + "", + "Stop the systemd unit first, then return to this dashboard.", + }, "\n") + } + return "" +} + +func mithrilSystemdUnit(cgroup string) (string, bool) { + for _, line := range strings.Split(cgroup, "\n") { + if line == "" { + continue + } + path := line + if idx := strings.LastIndex(line, ":"); idx >= 0 { + path = line[idx+1:] + } + unit := filepath.Base(path) + lower := strings.ToLower(unit) + if !strings.Contains(lower, "mithril") { + continue + } + if strings.HasSuffix(lower, ".service") || strings.HasSuffix(lower, ".scope") { + return unit, true + } + } + return "", false +} + +// checkStateDirFilesystem refuses if the PID-file directory is on a network +// filesystem where flock is unreliable. macOS passes (dev-only, lower stakes). +func checkStateDirFilesystem() string { + pidDir := stateDirForPidFile() + if pidDir == "" { + return "" + } + statDir := nearestExistingDir(pidDir) + if statDir == "" { + return "" + } + fsType, ok := statFSType(statDir) + if !ok { + return "" // can't tell, don't refuse + } + switch fsType { + case 0x6969: // NFS_SUPER_MAGIC + return refusedByNetworkFS("NFS", pidDir) + case 0x517B: // SMB_SUPER_MAGIC + return refusedByNetworkFS("SMB", pidDir) + case 0xFF534D42, 0xFE534D42: // CIFS_MAGIC_NUMBER / SMB2_MAGIC_NUMBER + return refusedByNetworkFS("CIFS", pidDir) + } + return "" +} + +func refusedByNetworkFS(name, dir string) string { + return strings.Join([]string{ + fmt.Sprintf("PID-file directory is on %s — flock is unreliable.", name), + "Two dashboards on different hosts could both think they hold the lock,", + "which would lead to a corrupted AccountsDB.", + "", + "Move the PID file to a local filesystem by setting MITHRIL_PID_FILE,", + fmt.Sprintf("or relocate %s to a non-networked path.", dir), + }, "\n") +} + +// stateDirForPidFile returns the directory of DefaultPidFile(). +func stateDirForPidFile() string { + pidPath := procctl.DefaultPidFile() + // Root-level PID ("/mithril.pid") checks "/", not cwd. + if i := strings.LastIndex(pidPath, "/"); i > 0 { + return pidPath[:i] + } else if i == 0 { + return "/" + } + return "." +} + +func nearestExistingDir(dir string) string { + for dir != "" { + if info, err := os.Stat(dir); err == nil && info.IsDir() { + return dir + } + next := strings.TrimRight(dir, "/") + i := strings.LastIndex(next, "/") + if i <= 0 { + if !strings.HasPrefix(next, "/") { + return "." + } + if info, err := os.Stat("/"); err == nil && info.IsDir() { + return "/" + } + return "" + } + dir = next[:i] + } + return "" +} + +// checkUIDMatch refuses if the dashboard runs as root but the AccountsDB dir is +// owned by a non-root user — root-written files the real user couldn't read. +func checkUIDMatch(accountsDir string) string { + if os.Geteuid() != 0 { + return "" // not root, no mismatch possible + } + if accountsDir == "" { + return "" // no AccountsDB configured yet — fresh setup + } + info, err := os.Stat(accountsDir) + if err != nil { + return "" // not created yet — bootstrap makes it as root, consistent + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "" // non-Unix, no UID concept here + } + if stat.Uid == 0 { + return "" // AccountsDB is already root-owned; consistent + } + return strings.Join([]string{ + "Dashboard is running as root but AccountsDB at", + " " + accountsDir, + fmt.Sprintf("is owned by uid %d.", stat.Uid), + "", + "Starting Mithril as root would create files the original user cannot read.", + "Either:", + " - Run the dashboard as the correct user: sudo -u mithril dashboard", + " - Or fix ownership before starting Mithril from this dashboard.", + }, "\n") +} + +func checkLogPathWritable(logsPath string) string { + if strings.TrimSpace(logsPath) == "" { + return "" + } + clean := filepath.Clean(logsPath) + info, err := os.Stat(clean) + if err == nil { + if !info.IsDir() { + return strings.Join([]string{ + "Log path is not a directory:", + " " + clean, + "", + "Change storage.logs to a directory, or remove that file first.", + }, "\n") + } + if !dirWritableByCurrentUser(info) { + return unwritableLogPathReason(clean, clean, "write log files") + } + return "" + } + if !os.IsNotExist(err) { + return strings.Join([]string{ + "Cannot inspect log directory:", + " " + clean, + "", + "Error: " + err.Error(), + }, "\n") + } + + parent := nearestExistingDir(filepath.Dir(clean)) + if parent == "" { + return strings.Join([]string{ + "Cannot create log directory:", + " " + clean, + "", + "No existing parent directory was found.", + }, "\n") + } + parentInfo, err := os.Stat(parent) + if err != nil { + return "" + } + if !dirWritableByCurrentUser(parentInfo) { + return unwritableLogPathReason(clean, parent, "create the log directory") + } + return "" +} + +func unwritableLogPathReason(logsPath, checkedPath, action string) string { + return strings.Join([]string{ + "Mithril cannot write logs in the configured folder.", + " logs: " + logsPath, + " checked: " + checkedPath, + "", + "Recommended fix: choose Fix with safe folders.", + "That creates user-owned folders and keeps existing data untouched.", + "", + "Advanced fix: ask an administrator to create/chown the log folder.", + "Needed permission: " + action + ".", + }, "\n") +} + +func dirWritableByCurrentUser(info os.FileInfo) bool { + if os.Geteuid() == 0 { + return true + } + if !info.IsDir() { + return false + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return false + } + mode := int(info.Mode().Perm()) + perm := mode & 0007 + if uint32(os.Geteuid()) == stat.Uid { + perm = (mode >> 6) & 0007 + } else if currentUserInGroup(stat.Gid) { + perm = (mode >> 3) & 0007 + } + return perm&0003 == 0003 // write + execute/search +} + +func currentUserInGroup(gid uint32) bool { + if uint32(os.Getegid()) == gid { + return true + } + groups, err := os.Getgroups() + if err != nil { + return false + } + for _, group := range groups { + if uint32(group) == gid { + return true + } + } + return false +} + +func checkAccountsStateCompatible(cfg *configData, accountsDir string) string { + if strings.TrimSpace(accountsDir) == "" { + return "" + } + st, err := state.LoadState(accountsDir) + if err != nil { + return strings.Join([]string{ + "Existing AccountsDB state cannot be read.", + " " + filepath.Join(accountsDir, state.StateFileName), + "", + "Error: " + err.Error(), + "", + "Use a fresh AccountsDB path or rebuild from snapshot before starting.", + }, "\n") + } + if st == nil { + return "" + } + // "unknown" is the unset-cluster sentinel; treat it as no opinion, not a + // mismatch (matches data.go). The state file only stores real names. + if cfg != nil && cfg.cluster != "" && cfg.cluster != "unknown" && st.Cluster != "" && cfg.cluster != st.Cluster { + return strings.Join([]string{ + "Existing AccountsDB belongs to a different cluster.", + " config: " + cfg.cluster, + " state: " + st.Cluster, + "", + "Use a cluster-matched AccountsDB path or rebuild from snapshot.", + }, "\n") + } + if st.Stage == "ready" && len(st.ManifestEpochStakes) > 0 && len(st.ManifestEpochAuthorizedVoters) == 0 { + return strings.Join([]string{ + "Stored node data cannot be resumed by this Mithril version.", + " " + filepath.Join(accountsDir, state.StateFileName), + "", + "Recommended fix: choose Fix with safe folders.", + "That builds fresh local data and leaves the old folder untouched.", + "", + "Advanced detail: state file is missing manifest_epoch_authorized_voters.", + }, "\n") + } + return "" +} diff --git a/cmd/mithril/dashboardcmd/process_actions.go b/cmd/mithril/dashboardcmd/process_actions.go new file mode 100644 index 000000000..274448539 --- /dev/null +++ b/cmd/mithril/dashboardcmd/process_actions.go @@ -0,0 +1,863 @@ +// Start/Stop/Restart/Force-Stop wiring for the Run Node view (UI state, +// confirmations, startup watching; signal delivery lives in pkg/procctl). + +package dashboardcmd + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" + tea "github.com/charmbracelet/bubbletea" +) + +const ( + opStart = "starting" + opStop = "stopping" + opRestart = "restarting" + opForceStop = "force_stopping" +) + +// startupWatchWindow bounds how long after spawning a failure counts as +// "start failed" (vs a later crash) while we tail child stderr. +const startupWatchWindow = 30 * time.Second + +// startupSettleWindow is how long the child must outlive its PID file before +// Start counts as successful (surfaces early config/bootstrap failures). +const startupSettleWindow = 5 * time.Second + +// dashboardSpawnLogMaxAge bounds accumulation of dashboard-owned child log +// files; old ones are cleaned on next start. +const dashboardSpawnLogMaxAge = 24 * time.Hour + +// actionResultMsg is the terminal message for an in-flight action. result is +// "ok"/"failed"/"timeout"/"refused"; err is set when not "ok". +type actionResultMsg struct { + op string // matches procState.inFlightOp + result string + err string + stderr string // captured child stderr for Start failures + spawnLogs dashboardSpawnLogs + det *procctl.Detection +} + +// handleStartKey starts mithril if Stopped or Crashed. Runs preflight checks +// first; on refusal it stores the message in preflightErr and does not start. +func (m *model) handleStartKey() tea.Cmd { + if !m.canAct() { + return nil + } + if m.proc.detection == nil { + return nil + } + st := m.proc.detection.Status + if st != procctl.StatusStopped && st != procctl.StatusCrashed { + return nil // nothing to start; mithril already running + } + if reason := preflightCheck(m.cfg, m.procAccountsDir()); reason != "" { + m.proc.preflightErr = reason + m.proc.preflightInfo = "" + return nil + } + m.proc.preflightErr = "" // clear any prior refusal + m.proc.preflightInfo = "" + start := func(current *model) tea.Cmd { + return current.beginAction(opStart, spawnMithrilCmd(current.configFile, current.procAccountsDir())) + } + m.beginStartFlow(start) + return nil +} + +// handleStopKey confirms before sending SIGTERM. No-op if not running. +func (m *model) handleStopKey() tea.Cmd { + if !m.canAct() { + return nil + } + return m.openStopConfirmation() +} + +// handleStopFromLogsKey is the same shutdown path from the log screen; it +// allows log-scroll focus since the confirmation modal still gates the signal. +func (m *model) handleStopFromLogsKey() tea.Cmd { + if !m.canActAllowingLogFocus() { + return nil + } + return m.openStopConfirmation() +} + +func (m *model) openStopConfirmation() tea.Cmd { + if m.proc.detection == nil || m.proc.detection.Status != procctl.StatusRunning { + return nil + } + if reason := checkSystemd(); reason != "" { + m.proc.preflightErr = reason + return nil + } + m.proc.preflightErr = "" + m.confirmActive = true + m.rightScroll = 0 + m.confirmTitle = "Stop Mithril?" + m.confirmBody = "This sends SIGTERM and waits for clean exit. Up to 60 seconds during normal operation; longer during AccountsDB rebuild." + m.confirmOnYes = func(current *model) tea.Cmd { + return current.beginAction(opStop, stopMithrilCmd(current.procAccountsDir())) + } + return nil +} + +func (m *model) openSafeFoldersConfirmation() tea.Cmd { + m.proc.configFixErr = "" // clear any prior failure so a retry starts clean + if m.cfg == nil { + m.proc.preflightErr = "Config is still loading. Wait a moment, then try again." + m.proc.preflightInfo = "" + return nil + } + paths, err := safeStoragePathsForConfig(m.cfg) + if err != nil { + m.proc.preflightErr = "Could not choose safe folders.\n\nError: " + err.Error() + m.proc.preflightInfo = "" + return nil + } + m.confirmActive = true + m.rightScroll = 0 + m.confirmTitle = "Use safe folders?" + m.confirmBody = strings.Join([]string{ + "This updates the config to user-owned folders and keeps old data untouched.", + "", + "New data root:", + " " + paths.root, + "", + "Mithril will build fresh local data on the next Start.", + "Your old data folders are not deleted.", + }, "\n") + m.confirmOnYes = func(current *model) tea.Cmd { + if current.cfg == nil { + return func() tea.Msg { return configFixResultMsg{err: "config is still loading"} } + } + cfg := *current.cfg + return applySafeStoragePathsCmd(current.configFile, &cfg) + } + return nil +} + +// openRebuildInPlaceConfirmation rebuilds from snapshot into the EXISTING paths, +// deleting the current AccountsDB — the destructive fallback when no spare disk fits. +func (m *model) openRebuildInPlaceConfirmation() tea.Cmd { + m.proc.configFixErr = "" // clear any prior failure so a retry starts clean + if m.cfg == nil { + m.proc.preflightErr = "Config is still loading. Wait a moment, then try again." + m.proc.preflightInfo = "" + return nil + } + acc := strings.TrimSpace(m.cfg.accountsPath) + if acc == "" { + m.proc.preflightErr = "No accounts path is configured, so there is nothing to rebuild in place. Use Review config to set storage paths." + m.proc.preflightInfo = "" + return nil + } + + lines := []string{ + "This rebuilds from snapshot using your current storage paths.", + "", + "Rebuild on:", + " " + acc, + } + if config.HasExistingAccountsDb(acc) { + reclaim := config.ReclaimableDirBytes(acc) / (1 << 30) + lines = append(lines, "", fmt.Sprintf("The current AccountsDB (~%d GB) is DELETED first, then rebuilt.", reclaim)) + } else { + lines = append(lines, "", "Any existing AccountsDB here is deleted first, then rebuilt.") + } + + // Reclaim-aware disk readiness — the same check the build runs. + check := checkBuildSpaceFn(m.cfg.cluster, acc, m.cfg.snapshotsPath) + switch { + case !check.Determined: + lines = append(lines, "", "Free space could not be read here; the build verifies before downloading.") + case check.OK: + lines = append(lines, "", fmt.Sprintf("Disk: ~%d GB usable after reclaiming, ~%d GB needed — enough.", check.UsableGB, check.NeedGB)) + default: + lines = append(lines, "", "⚠ "+check.Reason) + } + + m.confirmActive = true + m.rightScroll = 0 + m.confirmTitle = "Rebuild in place — deletes current data" + m.confirmBody = strings.Join(lines, "\n") + m.confirmOnYes = func(current *model) tea.Cmd { + if current.cfg == nil { + return func() tea.Msg { return configFixResultMsg{err: "config is still loading"} } + } + cfg := *current.cfg + return applyRebuildInPlaceCmd(current.configFile, &cfg) + } + return nil +} + +func applyRebuildInPlaceCmd(configFile string, cfg *configData) tea.Cmd { + return func() tea.Msg { + summary, err := applyRebuildInPlace(configFile, cfg) + if err != nil { + return configFixResultMsg{err: err.Error()} + } + return configFixResultMsg{summary: summary} + } +} + +// applyRebuildInPlace sets bootstrap.mode=snapshot (deletion happens on next Start). +// Refuses if the disk can't fit even after reclaiming the old DB. +func applyRebuildInPlace(configFile string, cfg *configData) (string, error) { + if cfg == nil { + return "", fmt.Errorf("config is still loading") + } + acc := strings.TrimSpace(cfg.accountsPath) + if acc == "" { + return "", fmt.Errorf("no accounts path configured") + } + if check := checkBuildSpaceFn(cfg.cluster, acc, cfg.snapshotsPath); check.Determined && !check.OK { + return "", fmt.Errorf("%s", check.Reason) + } + if err := saveConfigValue(configFile, "bootstrap", "mode", "snapshot"); err != nil { + return "", fmt.Errorf("save bootstrap.mode: %w", err) + } + return fmt.Sprintf("Set to rebuild in place on %s. Press Start to rebuild — this replaces the current AccountsDB.", acc), nil +} + +// handleForceStopKey is valid only when proc.stuck is set. SIGKILL risks +// AccountsDB corruption, so it requires explicit confirmation. +func (m *model) handleForceStopKey() tea.Cmd { + if !m.canAct() { + return nil + } + if !m.proc.stuck { + return nil + } + if m.proc.detection == nil || m.proc.detection.Status != procctl.StatusRunning { + return nil + } + if reason := checkSystemd(); reason != "" { + m.proc.preflightErr = reason + return nil + } + m.proc.preflightErr = "" + m.confirmActive = true + m.rightScroll = 0 + m.confirmTitle = "Force Stop — Data Loss Risk" + m.confirmBody = strings.Join([]string{ + "This sends SIGKILL, which bypasses every shutdown safeguard.", + "AccountsDB may be left in an inconsistent state and require a rebuild from snapshot.", + "", + "Only proceed if SIGTERM has already been stuck for several minutes", + "AND the node is NOT currently rebuilding AccountsDB (which is normal and slow).", + }, "\n") + m.confirmOnYes = func(current *model) tea.Cmd { + return current.beginAction(opForceStop, forceKillMithrilCmd()) + } + return nil +} + +// handleRestartKey confirms, then chains Stop → Start. +func (m *model) handleRestartKey() tea.Cmd { + if !m.canAct() { + return nil + } + if m.proc.detection == nil || m.proc.detection.Status != procctl.StatusRunning { + return nil + } + if reason := checkSystemd(); reason != "" { + m.proc.preflightErr = reason + return nil + } + m.proc.preflightErr = "" + m.confirmActive = true + m.rightScroll = 0 + m.confirmTitle = "Restart Mithril?" + m.confirmBody = "Sends SIGTERM, waits for clean exit, then starts a new mithril process." + m.confirmOnYes = func(current *model) tea.Cmd { + return current.beginAction(opRestart, restartMithrilCmd(current.configFile, current.procAccountsDir())) + } + return nil +} + +// canAct reports whether a new action can start: not editing config text, not +// log-focused, none in flight. +func (m *model) canAct() bool { + if m.proc.inFlightOp != "" { + return false + } + if m.editMode == editText { + return false + } + if m.logFocused { + return false + } + return true +} + +func (m *model) canActAllowingLogFocus() bool { + if m.proc.inFlightOp != "" { + return false + } + if m.editMode == editText { + return false + } + return true +} + +// beginAction marks the op in-flight, clears prior error/progress, and returns +// the cmd that runs it. +func (m *model) beginAction(op string, cmd tea.Cmd) tea.Cmd { + m.proc.inFlightOp = op + m.proc.opStartedAt = time.Now() + m.proc.opErr = "" + m.proc.startFailStderr = "" + m.proc.progressLines = m.proc.progressLines[:0] + return cmd +} + +// spawnMithrilCmd forks `mithril run` detached, then polls the outcome: +// PID file + survives settle = ok; early exit or no PID file = failed. +func spawnMithrilCmd(configPath, accountsDir string) tea.Cmd { + return func() tea.Msg { + // Re-exec our own binary in `run` mode. + exe, err := os.Executable() + if err != nil { + return actionResultMsg{op: opStart, result: "failed", + err: fmt.Sprintf("cannot resolve own binary path: %v", err), + det: detectProcessForAction(accountsDir)} + } + + args := []string{"run"} + if configPath != "" { + args = append(args, "--config", configPath) + } + cmd := exec.Command(exe, args...) + cmd.Env = append(os.Environ(), procctl.SpawnedByEnv+"=dashboard") + // Setsid detaches the child so it survives dashboard exit; it acquires + // its own flock via procctl.AcquireForRun. + cmd.SysProcAttr = spawnSysProcAttr() + + // Redirect child stdout/stderr to per-run temp files + a stderr ring + // (for fast Start-failed surfacing before mithril's mlog inits). + errBuf := newRingBuffer(4096) + cleanupOldDashboardSpawnTempLogs(dashboardSpawnLogMaxAge) + stdoutF, stdoutPath, _ := createPrivateTempLogFile("mithril-dashboard-spawn-stdout-*.log") + stderrF, stderrPath, _ := createPrivateTempLogFile("mithril-dashboard-spawn-stderr-*.log") + if stdoutF != nil { + cmd.Stdout = stdoutF + } + if stderrF != nil { + // Use the *os.File directly: a non-*os.File stderr makes os/exec open + // a parent-side pipe that breaks when the dashboard exits. + cmd.Stderr = stderrF + } else { + cmd.Stderr = errBuf + } + + if err := cmd.Start(); err != nil { + if stdoutF != nil { + _ = stdoutF.Close() + _ = os.Remove(stdoutF.Name()) + } + if stderrF != nil { + _ = stderrF.Close() + _ = os.Remove(stderrF.Name()) + } + return actionResultMsg{op: opStart, result: "failed", + err: fmt.Sprintf("exec.Start: %v", err), + det: detectProcessForAction(accountsDir)} + } + childPid := cmd.Process.Pid + + // Reap in a goroutine so the child doesn't become a zombie; we don't + // wait on it here — it outlives us by design. + go func() { + _ = cmd.Wait() + if stdoutF != nil { + _ = stdoutF.Close() + } + if stderrF != nil { + _ = stderrF.Close() + } + }() + + // Poll for the outcome within startupWatchWindow. + deadline := time.Now().Add(startupWatchWindow) + var pidFileSeenAt time.Time + var childRunID string + for time.Now().Before(deadline) { + time.Sleep(500 * time.Millisecond) + + // Probe liveness via the os.Process handle, not the bare PID (a + // recycled PID could signal an unrelated process). + if err := cmd.Process.Signal(syscall.Signal(0)); err != nil { + if det, ok := cleanCompletionAfterStart(accountsDir, childRunID); ok { + cleanupSpawnTempFiles(stdoutPath, stderrPath) + return actionResultMsg{op: opStart, result: "ok", det: det} + } + stderr := errBuf.String() + if stderrF != nil { + stderr = readFileTail(stderrPath, 4096) + } + return actionResultMsg{op: opStart, result: "failed", + err: fmt.Sprintf("child mithril (pid %d) exited during startup", childPid), + stderr: stderr, + spawnLogs: dashboardSpawnLogs{ + stdoutPath: stdoutPath, + stderrPath: stderrPath, + }, + det: detectProcessForAction(accountsDir)} + } + + // Did the PID file appear (child acquired its lock)? + info, perr := procctl.ReadPidFile(procctl.DefaultPidFile()) + if perr == nil && info.Pid == childPid { + if pidFileSeenAt.IsZero() { + pidFileSeenAt = time.Now() + childRunID = info.RunID + _ = procctl.UpdatePidOutputPaths(procctl.DefaultPidFile(), childPid, stdoutPath, stderrPath) + continue + } + if time.Since(pidFileSeenAt) >= startupSettleWindow { + return actionResultMsg{op: opStart, result: "ok"} + } + continue + } + // Different PID in the file — fail only if Detect proves that process + // is live and holds the lock; otherwise keep waiting (stale file). + if perr == nil && info.Pid != childPid { + if pid, ok := pidFileDescribesOtherLiveLockedProcess( + procctl.DefaultPidFile(), + procctl.DefaultLockFile(), + accountsDir, + childPid, + ); ok { + stderr := errBuf.String() + if stderrF != nil { + stderr = readFileTail(stderrPath, 4096) + } + return actionResultMsg{op: opStart, result: "failed", + err: fmt.Sprintf("another mithril already holds the lock (pid %d)", pid), + stderr: stderr, + spawnLogs: dashboardSpawnLogs{ + stdoutPath: stdoutPath, + stderrPath: stderrPath, + }, + det: detectProcessForAction(accountsDir)} + } + } + } + _ = cmd.Process.Signal(syscall.SIGTERM) + stderr := errBuf.String() + if stderrF != nil { + stderr = readFileTail(stderrPath, 4096) + } + return actionResultMsg{op: opStart, result: "failed", + err: fmt.Sprintf("child mithril (pid %d) did not create pid file within %s; sent SIGTERM", childPid, startupWatchWindow), + stderr: stderr, + spawnLogs: dashboardSpawnLogs{ + stdoutPath: stdoutPath, + stderrPath: stderrPath, + }, + det: detectProcessForAction(accountsDir)} + } +} + +func cleanCompletionAfterStart(accountsDir, childRunID string) (*procctl.Detection, bool) { + if accountsDir == "" || childRunID == "" { + return nil, false + } + st, err := state.LoadState(accountsDir) + if err != nil || st == nil || st.CurrentRunID != childRunID { + return nil, false + } + clean, _ := state.WasCleanExit(st) + if !clean { + return nil, false + } + det := detectProcessForAction(accountsDir) + return det, det != nil && det.Status == procctl.StatusStopped +} + +func detectProcessForAction(accountsDir string) *procctl.Detection { + det, err := procctl.Detect(procctl.DefaultPidFile(), procctl.DefaultLockFile(), accountsDir) + if err != nil { + return nil + } + return det +} + +func readFileTail(path string, maxBytes int64) string { + if path == "" || maxBytes <= 0 { + return "" + } + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return "" + } + offset := info.Size() - maxBytes + if offset < 0 { + offset = 0 + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return "" + } + data, err := io.ReadAll(io.LimitReader(f, maxBytes)) + if err != nil { + return "" + } + return string(data) +} + +type safeStoragePaths struct { + root string + accounts string + snapshots string + shredstore string + logs string +} + +func applySafeStoragePathsCmd(configFile string, cfg *configData) tea.Cmd { + return func() tea.Msg { + summary, err := applySafeStoragePaths(configFile, cfg) + if err != nil { + return configFixResultMsg{err: err.Error()} + } + return configFixResultMsg{summary: summary} + } +} + +func applySafeStoragePaths(configFile string, cfg *configData) (string, error) { + paths, err := safeStoragePathsForConfig(cfg) + if err != nil { + return "", err + } + // Refuse to repoint storage onto a disk too small for the AccountsDB; + // otherwise the build fails mid-download with a cryptic "no space left". + need := estimatedAccountsDbGB(cfg) + if free, ok := diskFreeGBFn(paths.root); ok && free < need { + return "", fmt.Errorf( + "%s is on a disk with only ~%d GB free, but the %s AccountsDB needs ~%d GB. "+ + "Point storage at a larger disk first (Edit Config → storage paths), then use safe folders", + paths.root, free, clusterLabel(cfg), need) + } + for _, dir := range []string{paths.root, paths.accounts, paths.snapshots, paths.shredstore, paths.logs} { + if err := os.MkdirAll(dir, 0700); err != nil { + return "", fmt.Errorf("create %s: %w", dir, err) + } + } + updates := []struct { + section string + key string + value string + }{ + {"storage", "accounts", paths.accounts}, + {"storage", "snapshots", paths.snapshots}, + {"snapshot", "download_path", paths.snapshots}, + {"storage", "shredstore", paths.shredstore}, + {"storage", "logs", paths.logs}, + // Keep [log] dir in step with storage.logs (mlog writes there). + {"log", "dir", paths.logs}, + {"bootstrap", "mode", "auto"}, + } + for _, update := range updates { + if err := saveConfigValue(configFile, update.section, update.key, update.value); err != nil { + return "", fmt.Errorf("save %s.%s: %w", update.section, update.key, err) + } + } + return fmt.Sprintf("Updated config to use %s. Old data was not deleted. Press Start to build fresh local data.", paths.root), nil +} + +func safeStoragePathsForConfig(cfg *configData) (safeStoragePaths, error) { + root, err := safeStorageRoot(cfg) + if err != nil { + return safeStoragePaths{}, err + } + paths := safeStoragePaths{ + root: root, + accounts: filepath.Join(root, "accounts"), + snapshots: filepath.Join(root, "snapshots"), + shredstore: filepath.Join(root, "shredstore"), + logs: filepath.Join(root, "logs"), + } + if reason := checkAccountsStateCompatible(cfg, paths.accounts); reason != "" { + paths.accounts = filepath.Join(root, "accounts-"+time.Now().Format("20060102-150405")) + } + return paths, nil +} + +func safeStorageRootForDisplay(cfg *configData) string { + root, err := safeStorageRoot(cfg) + if err != nil { + return "" + } + return root +} + +// safeStorageRoot picks a user-owned root for fresh data: a "mithril-data" dir +// beside existing storage, else $HOME — first writable one with room. +func safeStorageRoot(cfg *configData) (string, error) { + cluster := "default" + rawCluster := "" + if cfg != nil && strings.TrimSpace(cfg.cluster) != "" { + cluster = sanitizePathComponent(cfg.cluster) + rawCluster = cfg.cluster + } + need := config.EstimatedBuildBytes(rawCluster) // accounts + snapshot share the safe-folders disk + + // Build candidate roots in preference order. + var candidates []string + add := func(dir string) { + if d := strings.TrimSpace(dir); d != "" { + candidates = append(candidates, filepath.Join(filepath.Dir(d), "mithril-data", cluster)) + } + } + if cfg != nil { + add(cfg.snapshotsPath) // big "ledger" disk on the standard layout + add(cfg.shredstorePath) // usually the same big disk + add(cfg.accountsPath) + add(cfg.logsPath) + } + home, homeErr := os.UserHomeDir() + if homeErr == nil && strings.TrimSpace(home) != "" { + candidates = append(candidates, filepath.Join(home, "mithril-data", cluster)) + } + + seen := map[string]bool{} + firstWritable := "" + for _, c := range candidates { + if seen[c] { + continue + } + seen[c] = true + if !config.WritableDir(filepath.Dir(c)) { // can we create the mithril-data dir here? + continue + } + if firstWritable == "" { + firstWritable = c + } + if free, ok := config.FreeDiskBytes(c); ok && free >= need { + return c, nil // writable AND has room — ideal + } + } + if firstWritable != "" { + return firstWritable, nil // none has room; the apply-guard/UI will warn + } + if homeErr == nil && strings.TrimSpace(home) != "" { + return filepath.Join(home, "mithril-data", cluster), nil + } + return "", fmt.Errorf("no writable storage location found") +} + +// clusterLabel is a human-friendly cluster name for messages. +func clusterLabel(cfg *configData) string { + if cfg != nil && strings.TrimSpace(cfg.cluster) != "" { + return cfg.cluster + } + return "this cluster's" +} + +// estimatedAccountsDbGB is the free space (GB) needed to build, by cluster — +// the full footprint (accounts + snapshots share the safe-folders disk). +func estimatedAccountsDbGB(cfg *configData) uint64 { + cluster := "" + if cfg != nil { + cluster = cfg.cluster + } + return config.EstimatedBuildBytes(cluster) / (1 << 30) +} + +// pathFreeGB reports free GB on the filesystem holding path (probing the +// nearest existing ancestor); ok is false when it can't be determined. +func pathFreeGB(path string) (uint64, bool) { + du := getDiskUsageForPath("", path) + if du == nil || du.total == 0 || du.used > du.total { + return 0, false + } + return du.total - du.used, true +} + +// diskFreeGBFn is the safe-folders free-space probe; a var so tests can stub it. +var diskFreeGBFn = pathFreeGB + +// checkBuildSpaceFn is the reclaim-aware build-space verdict for rebuild-in- +// place; a var so tests can stub disk readiness. +var checkBuildSpaceFn = config.CheckBuildSpace + +func sanitizePathComponent(value string) string { + value = strings.TrimSpace(strings.ToLower(value)) + if value == "" { + return "default" + } + var b strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-' || r == '_': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + out := strings.Trim(b.String(), "-_") + if out == "" { + return "default" + } + return out +} + +func createPrivateTempLogFile(pattern string) (*os.File, string, error) { + f, err := os.CreateTemp("", pattern) + if err != nil { + return nil, "", err + } + if err := f.Chmod(0600); err != nil { + _ = f.Close() + _ = os.Remove(f.Name()) + return nil, "", err + } + return f, f.Name(), nil +} + +func cleanupSpawnTempFiles(paths ...string) { + for _, path := range paths { + if path != "" { + _ = os.Remove(path) + } + } +} + +func cleanupOldDashboardSpawnTempLogs(maxAge time.Duration) { + if maxAge <= 0 { + return + } + entries, err := os.ReadDir(os.TempDir()) + if err != nil { + return + } + cutoff := time.Now().Add(-maxAge) + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, "mithril-dashboard-spawn-") || !strings.HasSuffix(name, ".log") { + continue + } + info, err := entry.Info() + if err != nil || !info.Mode().IsRegular() || info.ModTime().After(cutoff) { + continue + } + _ = os.Remove(filepath.Join(os.TempDir(), name)) + } +} + +// stopMithrilCmd sends SIGTERM and waits for clean exit, returning the result. +// No progress streaming — the dashboard's 2s tick refreshes detection. +func stopMithrilCmd(accountsDir string) tea.Cmd { + return func() tea.Msg { + dashPid := os.Getpid() + if err := procctl.SignalStop(procctl.DefaultPidFile(), + procctl.DefaultAuditLog(), dashPid); err != nil { + return actionResultMsg{op: opStop, result: "failed", + err: fmt.Sprintf("signal stop: %v", err)} + } + det, err := procctl.WaitStopped( + procctl.DefaultPidFile(), + procctl.DefaultLockFile(), + accountsDir, + 60*time.Second, + ) + if err != nil { + // Distinguish timeout from other errors so the UI can offer + // a Force Stop action. + result := "failed" + if err == procctl.ErrStopTimeout { + result = "timeout" + } + return actionResultMsg{op: opStop, result: result, err: err.Error()} + } + return actionResultMsg{op: opStop, result: "ok", det: det} + } +} + +// forceKillMithrilCmd is the confirmed SIGKILL escalation after a stuck stop. +// procctl.ForceKill re-verifies identity (PID-reuse guard) and audit-logs it. +func forceKillMithrilCmd() tea.Cmd { + return func() tea.Msg { + dashPid := os.Getpid() + err := procctl.ForceKill(procctl.DefaultPidFile(), + procctl.DefaultAuditLog(), dashPid) + if err != nil { + return actionResultMsg{op: opForceStop, result: "failed", + err: fmt.Sprintf("force kill: %v", err)} + } + // Platform hook: Linux relies on Pdeathsig; macOS skips name-based + // cleanup so it can't kill another operator's Lightbringer. + cleanupOrphanLightbringer() + return actionResultMsg{op: opForceStop, result: "ok"} + } +} + +// restartMithrilCmd chains stop → start in one goroutine, keeping inFlightOp at +// "restarting" throughout so the UI doesn't flash "Stopped" between phases. +func restartMithrilCmd(configPath, accountsDir string) tea.Cmd { + return func() tea.Msg { + // Stop phase. + dashPid := os.Getpid() + if err := procctl.SignalStop(procctl.DefaultPidFile(), + procctl.DefaultAuditLog(), dashPid); err != nil { + return actionResultMsg{op: opRestart, result: "failed", + err: fmt.Sprintf("restart: stop phase: %v", err)} + } + _, err := procctl.WaitStopped( + procctl.DefaultPidFile(), + procctl.DefaultLockFile(), + accountsDir, + 60*time.Second, + ) + if err != nil { + result := "failed" + if err == procctl.ErrStopTimeout { + result = "timeout" + } + return actionResultMsg{op: opRestart, result: result, + err: fmt.Sprintf("restart: stop phase: %v", err)} + } + + // Start phase. + spawn := spawnMithrilCmd(configPath, accountsDir) + msg := spawn() + // Relabel the result op so the dashboard clears the right inFlightOp. + if res, ok := msg.(actionResultMsg); ok { + res.op = opRestart + return res + } + return msg + } +} + +func pidFileDescribesOtherLiveLockedProcess(pidPath, lockPath, accountsDir string, childPid int) (int, bool) { + info, err := procctl.ReadPidFile(pidPath) + if err != nil || info == nil || info.Pid == childPid { + return 0, false + } + det, err := procctl.Detect(pidPath, lockPath, accountsDir) + if err != nil || det == nil { + return 0, false + } + return det.Pid, det.Status == procctl.StatusRunning && det.Pid == info.Pid && det.LockHeld +} diff --git a/cmd/mithril/dashboardcmd/process_actions_test.go b/cmd/mithril/dashboardcmd/process_actions_test.go new file mode 100644 index 000000000..be073595b --- /dev/null +++ b/cmd/mithril/dashboardcmd/process_actions_test.go @@ -0,0 +1,890 @@ +package dashboardcmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Action key handlers: dashboard decision logic only (cmd vs modal vs no-op). +// Real spawn/signal/wait paths are tested in pkg/procctl. + +// runningModel: mithril running (Stop and Restart valid, Start not). +func runningModel() *model { + return &model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } +} + +// stoppedModel mirrors runningModel for Stopped/Crashed branches. +func stoppedModel(status procctl.Status) *model { + return &model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: status, + }}, + } +} + +// Start while running is silently ignored — guards against double-spawn. +func TestHandleStartKey_NoOpWhenRunning(t *testing.T) { + m := runningModel() + cmd := m.handleStartKey() + assert.Nil(t, cmd, "Start while Running should be a no-op") + assert.False(t, m.confirmActive, "Start must not open a modal") + assert.Empty(t, m.proc.inFlightOp, "Start must not flip inFlightOp on no-op") +} + +// Start while Stopped opens the interactive flow before spawning. +func TestHandleStartKey_OpensStartFlowWhenStopped(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + cmd := m.handleStartKey() + assert.Nil(t, cmd) + assert.True(t, m.startFlow.active) + assert.False(t, m.confirmActive) + assert.Empty(t, m.proc.inFlightOp) + + // One advance (Enter) starts mithril. + cmd = m.advanceStartFlow() + assert.NotNil(t, cmd, "confirm starts mithril") + assert.Equal(t, opStart, m.proc.inFlightOp) +} + +func TestHandleStartKey_ManagedLightbringerUsesStartFlow(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.cfg = &configData{lbEnabled: true} + + cmd := m.handleStartKey() + assert.Nil(t, cmd, "Managed Lightbringer Start should wait for the start flow") + assert.True(t, m.startFlow.active) + assert.False(t, m.confirmActive) + assert.Contains(t, m.renderStartFlow(), "Mithril + Lightbringer") + assert.NotContains(t, m.renderStartFlow(), "Steps:") + assert.Empty(t, m.proc.inFlightOp, "no spawn until the operator confirms") + + // One advance (Enter) starts. + cmd = m.advanceStartFlow() + assert.NotNil(t, cmd) + assert.Equal(t, opStart, m.proc.inFlightOp) +} + +// Start opens the flow on Crashed (post-crash recovery), not refused. +func TestHandleStartKey_AlsoWorksOnCrashed(t *testing.T) { + m := stoppedModel(procctl.StatusCrashed) + cmd := m.handleStartKey() + assert.Nil(t, cmd, "Start should open the flow on Crashed") + assert.True(t, m.startFlow.active) +} + +// Stop opens a confirm modal; no signal until the user confirms. +func TestHandleStopKey_OpensConfirmModalWhenRunning(t *testing.T) { + m := runningModel() + cmd := m.handleStopKey() + assert.Nil(t, cmd, "Stop opens a modal; cmd fires only on Y") + assert.True(t, m.confirmActive, "Stop must open the confirmation modal") + assert.Contains(t, m.confirmTitle, "Stop", "modal title should announce Stop") + assert.NotNil(t, m.confirmOnYes, "modal must carry an onYes callback") + assert.Empty(t, m.proc.inFlightOp, "no action until user confirms") +} + +func TestConfirmStopViaUpdateSetsInFlightOnReturnedModel(t *testing.T) { + m := runningModel() + m.runFocused = true + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'x'}}) + res := updated.(model) + require.Nil(t, cmd) + require.True(t, res.confirmActive) + + updated, cmd = res.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res = updated.(model) + assert.NotNil(t, cmd) + assert.Equal(t, opStop, res.proc.inFlightOp) + assert.False(t, res.confirmActive) +} + +// Stop while already stopped is silent. +func TestHandleStopKey_NoOpWhenStopped(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + cmd := m.handleStopKey() + assert.Nil(t, cmd) + assert.False(t, m.confirmActive) +} + +// Restart gates behind confirmation (destructive — SIGTERMs the running node). +func TestHandleRestartKey_OpensConfirmModalWhenRunning(t *testing.T) { + m := runningModel() + cmd := m.handleRestartKey() + assert.Nil(t, cmd, "Restart opens a modal") + assert.True(t, m.confirmActive) + assert.Contains(t, m.confirmTitle, "Restart") + assert.NotNil(t, m.confirmOnYes) +} + +// Restart while Stopped is a no-op (use Start). +func TestHandleRestartKey_NoOpWhenStopped(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + cmd := m.handleRestartKey() + assert.Nil(t, cmd) + assert.False(t, m.confirmActive) +} + +// Rebuild in place: destructive recovery (reuses paths, wipes diverged +// AccountsDB, rebuilds via bootstrap=snapshot) offered alongside "Rebuild fresh". + +// Divergence crash offers the danger-flagged in-place rebuild after the safe one. +func TestRunActions_DivergenceOffersRebuildInPlace(t *testing.T) { + m := stoppedModel(procctl.StatusCrashed) + m.runFocused = true + m.proc.startFailStderr = "FATAL: replay divergence detected at slot 123" + require.True(t, m.hasReplayDivergenceCrash(), "test setup must trigger divergence path") + + actions := m.runActions() + require.NotEmpty(t, actions) + assert.Equal(t, runActionSafe, actions[0].id, "fresh, non-destructive recovery stays first") + + var inPlace *runAction + for i := range actions { + if actions[i].id == runActionInPlace { + inPlace = &actions[i] + } + } + require.NotNil(t, inPlace, "in-place rebuild must be offered on divergence") + assert.True(t, inPlace.danger, "in-place rebuild deletes data — must be danger-flagged") +} + +// Confirmation names the path and warns about deletion — never a silent wipe. +func TestOpenRebuildInPlaceConfirmation_DestructiveConfirm(t *testing.T) { + m := stoppedModel(procctl.StatusCrashed) + m.cfg = &configData{ + cluster: "mainnet-beta", + accountsPath: "/mnt/mithril-accounts", + snapshotsPath: "/mnt/mithril-ledger/snapshots", + } + + cmd := m.openRebuildInPlaceConfirmation() + assert.Nil(t, cmd, "confirmation opens a modal; the apply cmd fires only on Yes") + assert.True(t, m.confirmActive) + assert.NotNil(t, m.confirmOnYes, "modal must carry an onYes callback") + assert.Contains(t, strings.ToLower(m.confirmTitle), "delete", "title must warn it deletes data") + assert.Contains(t, m.confirmBody, "/mnt/mithril-accounts", "body must name the path being rebuilt") +} + +// No AccountsDB path → actionable message, not a destructive modal. +func TestOpenRebuildInPlaceConfirmation_NoOpWithoutAccountsPath(t *testing.T) { + m := stoppedModel(procctl.StatusCrashed) + m.cfg = &configData{cluster: "mainnet-beta"} // no accountsPath + + cmd := m.openRebuildInPlaceConfirmation() + assert.Nil(t, cmd) + assert.False(t, m.confirmActive, "no destructive modal without a path to rebuild") + assert.NotEmpty(t, m.proc.preflightErr) +} + +// Apply flips bootstrap mode to "snapshot" and leaves storage paths untouched. +// Disk readiness stubbed to "fits" so the config write runs regardless of disk. +func TestApplyRebuildInPlaceSetsSnapshotMode(t *testing.T) { + orig := checkBuildSpaceFn + checkBuildSpaceFn = func(string, string, string) config.BuildSpaceCheck { + return config.BuildSpaceCheck{Determined: true, OK: true, UsableGB: 900, NeedGB: 600} + } + t.Cleanup(func() { checkBuildSpaceFn = orig }) + + dir := t.TempDir() + accounts := filepath.Join(dir, "accounts") + snapshots := filepath.Join(dir, "snapshots") + configFile := filepath.Join(dir, "config.toml") + require.NoError(t, os.WriteFile(configFile, []byte(` +[network] +cluster = "mainnet-beta" +[storage] +accounts = "`+accounts+`" +snapshots = "`+snapshots+`" +[bootstrap] +mode = "auto" +`), 0600)) + + cfg := &configData{cluster: "mainnet-beta", accountsPath: accounts, snapshotsPath: snapshots} + summary, err := applyRebuildInPlace(configFile, cfg) + require.NoError(t, err) + assert.Contains(t, summary, accounts) + + got := readConfig(configFile) + require.NotNil(t, got) + assert.Equal(t, "snapshot", got.bootstrapMode, "bootstrap mode must be set to snapshot") + assert.Equal(t, accounts, got.accountsPath, "storage paths must be unchanged") + assert.Equal(t, snapshots, got.snapshotsPath, "storage paths must be unchanged") +} + +// Apply refuses and writes nothing when the disk still can't fit the rebuild. +func TestApplyRebuildInPlaceRefusesWhenDiskTooSmall(t *testing.T) { + orig := checkBuildSpaceFn + checkBuildSpaceFn = func(string, string, string) config.BuildSpaceCheck { + return config.BuildSpaceCheck{Determined: true, OK: false, Reason: "not enough disk space: needs ~600 GB; point storage at a larger disk"} + } + t.Cleanup(func() { checkBuildSpaceFn = orig }) + + dir := t.TempDir() + configFile := filepath.Join(dir, "config.toml") + require.NoError(t, os.WriteFile(configFile, []byte(` +[network] +cluster = "mainnet-beta" +[storage] +accounts = "/mnt/mithril-accounts" +[bootstrap] +mode = "auto" +`), 0600)) + + cfg := &configData{cluster: "mainnet-beta", accountsPath: "/mnt/mithril-accounts"} + _, err := applyRebuildInPlace(configFile, cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "larger disk") + + got := readConfig(configFile) + require.NotNil(t, got) + assert.Equal(t, "auto", got.bootstrapMode, "config must be untouched on refusal") +} + +// canAct — gate that serializes all action keys. + +// A second action is blocked while one is in flight (e.g. spamming 'x'). +func TestCanAct_BlocksDuringInFlightOp(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStop // stop already in progress + assert.False(t, m.canAct(), "must not allow new actions during stop") + + // Even Start is blocked: spawning during shutdown would race for the flock. + cmd := m.handleStartKey() + assert.Nil(t, cmd) +} + +// Action keys must not fire while typing in the editor (e.g. "stop" in a URL). +func TestCanAct_BlocksDuringTextEdit(t *testing.T) { + m := runningModel() + m.editMode = editText + assert.False(t, m.canAct()) +} + +// Stop must not fire while scrolling logs. +func TestCanAct_BlocksWhileLogFocused(t *testing.T) { + m := runningModel() + m.logFocused = true + assert.False(t, m.canAct()) +} + +// renderActionHints — secondary shortcuts below the Run Node action list. + +// Running shows exactly Stop and Restart, nothing else. +func TestRenderActionHints_RunningShowsStopAndRestart(t *testing.T) { + dim := dummyStyle() + out := renderActionHints(procctl.StatusRunning, "", false, nil, dim) + assert.Contains(t, out, "[x]") + assert.Contains(t, out, "Stop") + assert.Contains(t, out, "[r]") + assert.Contains(t, out, "Restart") + assert.NotContains(t, out, "[s]", "Running state must NOT advertise Start") + assert.NotContains(t, out, "[f]", "non-stuck Running must NOT advertise Force Stop") +} + +// Once stuck, [f] Force Stop joins the hint line. +func TestRenderActionHints_StuckShowsForceStop(t *testing.T) { + dim := dummyStyle() + out := renderActionHints(procctl.StatusRunning, "", true, nil, dim) + assert.Contains(t, out, "[f]", "stuck state must surface Force Stop") + assert.Contains(t, out, "Force Stop") + // Force Stop is additive — Stop/Restart still present. + assert.Contains(t, out, "[x]") + assert.Contains(t, out, "[r]") +} + +// Stopped only shows Start, never Stop/Restart. +func TestRenderActionHints_StoppedShowsStart(t *testing.T) { + dim := dummyStyle() + out := renderActionHints(procctl.StatusStopped, "", false, nil, dim) + assert.Contains(t, out, "[s]") + assert.Contains(t, out, "Start") + assert.NotContains(t, out, "Stop") + assert.NotContains(t, out, "Restart") +} + +// During an inFlightOp, hints explain keys are disabled rather than advertise them. +func TestRenderActionHints_InFlightDisablesAll(t *testing.T) { + dim := dummyStyle() + out := renderActionHints(procctl.StatusRunning, opStop, false, nil, dim) + assert.Contains(t, out, "disabled") + assert.NotContains(t, out, "[x]") +} + +// Confirmation modal rendering. + +// Smoke check: modal frame produces the expected user-visible content. +func TestRenderConfirmModal_IncludesTitleAndKeys(t *testing.T) { + out := renderConfirmModal("Stop Mithril?", "This sends SIGTERM.", 80) + assert.Contains(t, out, "Stop Mithril?", "title must appear") + assert.Contains(t, out, "SIGTERM", "body must appear") + assert.Contains(t, out, "[enter]", "Enter confirm key must be hinted") + assert.Contains(t, out, "[y]", "Yes key must be hinted") + assert.Contains(t, out, "[n]", "No key must be hinted") +} + +func TestRenderConfirmModal_WrapsBodyToPaneWidth(t *testing.T) { + out := renderConfirmModal( + "Use safe folders?", + "This updates the config to user-owned folders and keeps old data untouched.", + 46, + ) + + assert.Contains(t, out, "keeps old") + assert.Contains(t, out, "data untouched") + for _, line := range strings.Split(out, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + assert.LessOrEqual(t, lipgloss.Width(line), 46, "line should fit pane: %q", line) + } +} + +// Ring buffer + +// Bounded write: 13 bytes into a cap-8 buffer keeps only the last 8. +func TestRingBuffer_DropsOldest(t *testing.T) { + rb := newRingBuffer(8) + _, _ = rb.Write([]byte("01234")) + _, _ = rb.Write([]byte("56789ABC")) + got := rb.String() + assert.Equal(t, "56789ABC", got, + "ring buffer should retain only the last 8 bytes (got %q)", got) + assert.LessOrEqual(t, len(got), 8) +} + +// Write reports full input length even past cap (safe for io.MultiWriter). +func TestRingBuffer_NeverShortWrites(t *testing.T) { + rb := newRingBuffer(4) + n, err := rb.Write([]byte("hello-this-is-longer")) + assert.NoError(t, err) + assert.Equal(t, len("hello-this-is-longer"), n) +} + +// Startup failure reporting reads only the tail of the stderr log. +func TestReadFileTail(t *testing.T) { + path := filepath.Join(t.TempDir(), "stderr.log") + require.NoError(t, os.WriteFile(path, []byte("0123456789abcdef"), 0600)) + + assert.Equal(t, "cdef", readFileTail(path, 4)) + assert.Equal(t, "0123456789abcdef", readFileTail(path, 64)) +} + +func TestCreatePrivateTempLogFile_Uses0600(t *testing.T) { + f, path, err := createPrivateTempLogFile("mithril-dashboard-test-*.log") + require.NoError(t, err) + defer os.Remove(path) + require.NoError(t, f.Close()) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestCleanupOldDashboardSpawnTempLogs(t *testing.T) { + dir := t.TempDir() + t.Setenv("TMPDIR", dir) + + oldPath := filepath.Join(dir, "mithril-dashboard-spawn-stderr-old.log") + newPath := filepath.Join(dir, "mithril-dashboard-spawn-stderr-new.log") + otherPath := filepath.Join(dir, "other.log") + require.NoError(t, os.WriteFile(oldPath, []byte("old"), 0600)) + require.NoError(t, os.WriteFile(newPath, []byte("new"), 0600)) + require.NoError(t, os.WriteFile(otherPath, []byte("other"), 0600)) + oldTime := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(oldPath, oldTime, oldTime)) + + cleanupOldDashboardSpawnTempLogs(time.Hour) + + _, err := os.Stat(oldPath) + assert.True(t, os.IsNotExist(err), "old dashboard spawn log should be removed") + _, err = os.Stat(newPath) + assert.NoError(t, err, "fresh dashboard spawn log should stay") + _, err = os.Stat(otherPath) + assert.NoError(t, err, "unrelated temp file should stay") +} + +func TestCleanCompletionAfterStart_RequiresMatchingCleanRunID(t *testing.T) { + dir := t.TempDir() + t.Setenv("MITHRIL_PID_FILE", filepath.Join(dir, "mithril.pid")) + + st := state.NewReadyState(100, 1, "", "", 0, 0) + st.CurrentRunID = "child-run" + st.LastShutdownReason = state.ShutdownReasonCompleted + st.LastShutdownAt = st.CurrentSessionStartedAt.Add(time.Second) + require.NoError(t, st.Save(dir)) + + det, ok := cleanCompletionAfterStart(dir, "child-run") + require.True(t, ok) + require.NotNil(t, det) + assert.Equal(t, procctl.StatusStopped, det.Status) + assert.True(t, det.LastCleanExit) + + _, ok = cleanCompletionAfterStart(dir, "other-run") + assert.False(t, ok) +} + +func TestPidFileDescribesOtherLiveLockedProcess_IgnoresStalePidFile(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + require.NoError(t, procctl.WritePidFile(pidPath, &procctl.PidInfo{ + Pid: 9999999, + StartTimeTicks: 1, + ExeInode: 1, + RunID: "stale", + SpawnedBy: "dashboard", + })) + + pid, ok := pidFileDescribesOtherLiveLockedProcess(pidPath, lockPath, dir, os.Getpid()) + assert.False(t, ok) + assert.Zero(t, pid) +} + +func TestPidFileDescribesOtherLiveLockedProcess_RecognizesLiveLockedProcess(t *testing.T) { + id, err := procctl.ReadIdentity(os.Getpid()) + if err != nil { + t.Skipf("process identity unavailable in this sandbox: %v", err) + } + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, procctl.WritePidFile(pidPath, &procctl.PidInfo{ + Pid: os.Getpid(), + StartTimeTicks: id.StartTimeTicks, + ExeInode: id.ExeInode, + BinaryPath: id.ExePath, + RunID: "live", + SpawnedBy: "dashboard", + })) + lh, err := procctl.AcquireLock(lockPath) + require.NoError(t, err) + defer lh.Release() + + pid, ok := pidFileDescribesOtherLiveLockedProcess(pidPath, lockPath, dir, os.Getpid()+1) + assert.True(t, ok) + assert.Equal(t, os.Getpid(), pid) +} + +// Update message flow — result handler clears state and re-triggers detection. + +// Successful action result resets state so the next key press is accepted. +func TestActionResultMsg_ClearsInFlightOp(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStop + m.proc.opStartedAt = time.Now().Add(-2 * time.Second) + + updated, _ := m.Update(actionResultMsg{op: opStop, result: "ok"}) + res := updated.(model) + assert.Empty(t, res.proc.inFlightOp, "successful result must clear inFlightOp") + // Last progress line is the "Done." marker. + if assert.NotEmpty(t, res.proc.progressLines) { + assert.Equal(t, "Done.", res.proc.progressLines[len(res.proc.progressLines)-1]) + } +} + +// Stop flips the view to Stopped immediately from the final detection. +func TestActionResultMsg_AppliesFinalDetection(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStop + + updated, _ := m.Update(actionResultMsg{ + op: opStop, + result: "ok", + det: &procctl.Detection{Status: procctl.StatusStopped}, + }) + res := updated.(model) + require.NotNil(t, res.proc.detection) + assert.Equal(t, procctl.StatusStopped, res.proc.detection.Status) + assert.Empty(t, res.proc.fetchErr) + assert.False(t, res.proc.fetchedAt.IsZero()) + assert.False(t, res.proc.lastOkAt.IsZero()) +} + +func TestActionResultMsg_StopSuccessSelectsStartAction(t *testing.T) { + m := runningModel() + m.screen = screenProcess + m.runFocused = true + m.runActionIdx = 1 // Stop safely while running + m.proc.inFlightOp = opStop + + updated, _ := m.Update(actionResultMsg{ + op: opStop, + result: "ok", + det: &procctl.Detection{Status: procctl.StatusStopped}, + }) + res := updated.(model) + + assert.Equal(t, 0, res.runActionIdx) + actions := res.runActions() + require.NotEmpty(t, actions) + assert.Equal(t, runActionStart, actions[res.runActionIdx].id) +} + +func TestActionResultMsg_StartSuccessSwitchesToLogs(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.proc.inFlightOp = opStart + m.logRawMode = true + m.startFlow = startFlowState{active: true, step: 2} + m.items = []menuItem{ + {label: "Run Node", value: "process"}, + {label: "Logs", value: "logs"}, + } + + updated, _ := m.Update(actionResultMsg{op: opStart, result: "ok"}) + res := updated.(model) + assert.Equal(t, screenLogs, res.screen) + assert.Equal(t, 1, res.cursor, "left menu should follow the automatic Logs handoff") + assert.False(t, res.logFocused, "logs should open readable, not trap q until esc") + assert.False(t, res.logRawMode, "Mithril-only start keeps the menu visible — no full-width takeover (use t/Terminal logs to opt in)") + assert.False(t, res.startFlow.active, "start success must clear the guided overlay") +} + +func TestRunNodeRawLogsActionUsesExistingLogView(t *testing.T) { + m := stoppedModel(procctl.StatusRunning) + m.runFocused = true + m.runActionIdx = 1 // Terminal logs + m.items = []menuItem{ + {label: "Run Node", value: "process"}, + {label: "Logs", value: "logs"}, + } + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res := updated.(model) + + assert.NotNil(t, cmd, "opening logs refreshes tails from disk") + assert.Equal(t, screenLogs, res.screen) + assert.True(t, res.logRawMode) + assert.False(t, res.runFocused) + assert.Empty(t, res.proc.inFlightOp, "raw logs must not start or stop a process") +} + +func TestLogsToggleSwitchesRawModeWithoutChangingProcess(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenLogs, + cfg: &configData{lbEnabled: true}, + mithrilLines: []string{"INFO mithril started"}, + lbLines: []string{"WARN repair peer missing"}, + proc: procState{inFlightOp: ""}, + } + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'t'}}) + res := updated.(model) + assert.Nil(t, cmd) + assert.True(t, res.logRawMode) + assert.Empty(t, res.proc.inFlightOp) + assert.Contains(t, res.renderLogsView(), "terminal logs") + + updated, _ = res.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'t'}}) + res = updated.(model) + assert.False(t, res.logRawMode) +} + +func TestRawLogsEmptyStateStillUsesTerminalView(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenLogs, + logRawMode: true, + width: 120, + height: 40, + } + + out := m.renderLogsView() + assert.Contains(t, out, "terminal logs") + assert.Contains(t, out, "(no log lines yet)") + assert.NotContains(t, out, "Open Run Node, choose Start") +} + +func TestFullWidthRawLogsEscReturnsToRunNode(t *testing.T) { + m := model{ + mode: modeDashboard, + hasConfig: true, + screen: screenLogs, + logRawMode: true, + width: 120, + height: 30, + items: []menuItem{ + {label: "Run Node", value: "process"}, + {label: "Logs", value: "logs"}, + }, + } + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyEsc}) + res := updated.(model) + + assert.Nil(t, cmd) + assert.Equal(t, screenProcess, res.screen) + assert.True(t, res.runFocused) + assert.False(t, res.logFocused) + assert.Equal(t, 0, res.cursor) +} + +func TestFocusedSplitLogsRedactSecrets(t *testing.T) { + secretURL := "https://rpc.example.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000" + m := model{ + hasConfig: true, + screen: screenLogs, + logFocused: true, + logPane: logPaneMithril, + cfg: &configData{lbEnabled: true}, + width: 180, + height: 40, + mithrilLines: []string{"INFO preferred rpc " + secretURL}, + lbLines: []string{"INFO repair rpc " + secretURL}, + } + + out := m.renderLogsView() + assert.NotContains(t, out, "test-key-00000000") + assert.Contains(t, out, "api-key=REDACTED") + + m.logPane = logPaneLightbringer + out = m.renderLogsView() + assert.NotContains(t, out, "test-key-00000000") + assert.Contains(t, out, "api-key=REDACTED") +} + +func TestLogsStopShortcutOpensConfirmWhileFocused(t *testing.T) { + m := runningModel() + m.screen = screenLogs + m.logFocused = true + m.logRawMode = true + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'x'}}) + res := updated.(model) + + assert.Nil(t, cmd, "stop from logs must still wait for confirmation") + assert.True(t, res.confirmActive) + assert.Contains(t, res.confirmTitle, "Stop") + assert.NotNil(t, res.confirmOnYes) + assert.Empty(t, res.proc.inFlightOp, "no signal until the confirmation is accepted") +} + +func TestLogsStopShortcutConfirmSetsInFlight(t *testing.T) { + m := runningModel() + m.screen = screenLogs + m.logFocused = true + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'x'}}) + res := updated.(model) + require.Nil(t, cmd) + require.True(t, res.confirmActive) + + updated, cmd = res.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res = updated.(model) + assert.NotNil(t, cmd) + assert.Equal(t, opStop, res.proc.inFlightOp) + + updated, cmd = res.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'x'}}) + res = updated.(model) + assert.Nil(t, cmd) + assert.False(t, res.confirmActive, "second stop must be blocked while stop is in flight") +} + +func TestLogsHelpShowsStopShortcutWhenRunning(t *testing.T) { + m := runningModel() + m.screen = screenLogs + m.logFocused = true + + var found bool + for _, item := range m.helpItems() { + if item.key == "x" && item.desc == "stop safely" { + found = true + break + } + } + assert.True(t, found, "running logs view should expose the safe stop shortcut") +} + +func TestStartFlowFinalEnterViaUpdateSetsInFlightOnReturnedModel(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.runFocused = true + m.cfg = &configData{} + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res := updated.(model) + require.Nil(t, cmd) + require.True(t, res.startFlow.active) + + for i := 0; i < 5 && res.startFlow.active; i++ { + updated, cmd = res.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res = updated.(model) + } + assert.NotNil(t, cmd) + assert.Equal(t, opStart, res.proc.inFlightOp) + assert.False(t, res.startFlow.active) +} + +func TestRunNodeEnterActivatesSelectedStartAction(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.runFocused = true + m.cfg = &configData{lbEnabled: true} + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res := updated.(model) + assert.Nil(t, cmd, "start opens the guided flow before spawning") + assert.True(t, res.startFlow.active) + assert.False(t, res.confirmActive) + assert.Contains(t, res.renderStartFlow(), "Mithril + Lightbringer") +} + +func TestRunNodeActionNavigationOpensDoctor(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.runFocused = true + + updated, _ := m.Update(tea.KeyMsg{Type: tea.KeyDown}) + afterDown := updated.(model) + assert.Equal(t, 1, afterDown.runActionIdx, "down should move from Start to Check readiness") + + updated, cmd := afterDown.Update(tea.KeyMsg{Type: tea.KeyEnter}) + res := updated.(model) + assert.NotNil(t, cmd, "Doctor navigation refreshes dashboard data") + assert.Equal(t, screenDoctor, res.screen) + assert.False(t, res.runFocused) +} + +func TestRunNodeShortcutKeysRequireActionFocus(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.runFocused = false + + updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'s'}}) + res := updated.(model) + assert.Nil(t, cmd) + assert.Empty(t, res.proc.inFlightOp) + assert.False(t, res.confirmActive) + + m = stoppedModel(procctl.StatusRunning) + m.runFocused = false + updated, _ = m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'r'}}) + res = updated.(model) + assert.False(t, res.confirmActive, "r should refresh with menu focus, not open Restart") + assert.Empty(t, res.proc.inFlightOp) +} + +// Timeout shows a plain "Timed out" line, not a raw error trace. +func TestActionResultMsg_TimeoutSurfacesPlainMessage(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStop + + updated, _ := m.Update(actionResultMsg{ + op: opStop, + result: "timeout", + err: procctl.ErrStopTimeout.Error(), + }) + res := updated.(model) + assert.Empty(t, res.proc.inFlightOp) + last := res.proc.progressLines[len(res.proc.progressLines)-1] + assert.True(t, strings.Contains(last, "Timed out") || strings.Contains(last, "timeout"), + "last progress line should describe the timeout in plain English: %q", last) +} + +// startFailStderr is populated when a Start fails fast. +func TestActionResultMsg_FailedCapturesStderrForStart(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStart + + updated, _ := m.Update(actionResultMsg{ + op: opStart, + result: "failed", + err: "exec failed", + stderr: "E_CONFIG_MISSING: file not found at /etc/mithril.toml", + }) + res := updated.(model) + assert.Contains(t, res.proc.startFailStderr, "E_CONFIG_MISSING", + "failure stderr must be preserved for the Process view to surface") +} + +func TestActionResultMsg_FailedStartRemembersSpawnLogsForLogsView(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStart + + stderrF, err := os.CreateTemp("", "mithril-dashboard-spawn-stderr-*.log") + require.NoError(t, err) + defer os.Remove(stderrF.Name()) + defer stderrF.Close() + + updated, _ := m.Update(actionResultMsg{ + op: opStart, + result: "failed", + err: "child exited during startup", + stderr: "panic: replay divergence", + spawnLogs: dashboardSpawnLogs{ + stderrPath: stderrF.Name(), + }, + }) + res := updated.(model) + + assert.Equal(t, stderrF.Name(), res.lastSpawnLogs.stderrPath) + assert.Contains(t, strings.Join(res.mithrilLines, "\n"), "panic: replay divergence") +} + +// "Start failed" must not render with a stale Running badge. +func TestActionResultMsg_FailedStartAppliesFinalDetection(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStart + + updated, _ := m.Update(actionResultMsg{ + op: opStart, + result: "failed", + err: "child exited during startup", + stderr: "mode=accountsdb requires existing AccountsDB", + det: &procctl.Detection{Status: procctl.StatusStopped}, + }) + res := updated.(model) + + require.NotNil(t, res.proc.detection) + assert.Equal(t, procctl.StatusStopped, res.proc.detection.Status) + out := res.renderProcessView() + assert.Contains(t, out, "Start failed") + assert.Contains(t, out, "Stopped") + assert.NotContains(t, out, "Running") +} + +// Text fields own printable keys (r/s/e/f/x), not global shortcuts. +func TestTextEditOwnsPrintableShortcutKeys(t *testing.T) { + m := model{screen: screenEdit, editMode: editText} + for _, r := range "private/lightbringer-rpc" { + updated, _ := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{r}}) + m = updated.(model) + } + assert.Equal(t, "private/lightbringer-rpc", m.editValue) +} + +// Bubble Tea can deliver multiple runes (paste) in one KeyMsg. +func TestTextEditAcceptsPastedRunes(t *testing.T) { + m := model{screen: screenEdit, editMode: editText} + updated, _ := m.Update(tea.KeyMsg{ + Type: tea.KeyRunes, + Runes: []rune("127.0.0.1:3001"), + }) + res := updated.(model) + assert.Equal(t, "127.0.0.1:3001", res.editValue) + assert.Equal(t, len("127.0.0.1:3001"), res.editCursor) +} + +// dummyStyle returns a no-op style; only content assertions matter in tests. +func dummyStyle() lipgloss.Style { + return lipgloss.NewStyle() +} diff --git a/cmd/mithril/dashboardcmd/process_escalation_test.go b/cmd/mithril/dashboardcmd/process_escalation_test.go new file mode 100644 index 000000000..2691652af --- /dev/null +++ b/cmd/mithril/dashboardcmd/process_escalation_test.go @@ -0,0 +1,348 @@ +package dashboardcmd + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Stuck is set on Stop timeout, cleared on successful detection or Force Stop. + +// Stop timeout transitions the model into Stuck (surfaces [f] Force Stop). +func TestActionResultMsg_TimeoutSetsStuck(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStop + assert.False(t, m.proc.stuck, "precondition: not stuck yet") + + updated, _ := m.Update(actionResultMsg{ + op: opStop, + result: "timeout", + err: procctl.ErrStopTimeout.Error(), + }) + res := updated.(model) + assert.True(t, res.proc.stuck, "stop timeout must set stuck=true") +} + +// Restart's stop-phase timeout also sets Stuck. +func TestActionResultMsg_RestartTimeoutSetsStuck(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opRestart + + updated, _ := m.Update(actionResultMsg{ + op: opRestart, + result: "timeout", + }) + res := updated.(model) + assert.True(t, res.proc.stuck) +} + +// Stuck applies only to Stop/Restart timeouts, not Start. +func TestActionResultMsg_StartTimeoutDoesNotSetStuck(t *testing.T) { + m := runningModel() + m.proc.inFlightOp = opStart + updated, _ := m.Update(actionResultMsg{op: opStart, result: "timeout"}) + res := updated.(model) + assert.False(t, res.proc.stuck, "Start timeout must not flip stuck") +} + +// Stuck auto-clears once detect shows the process is gone. +func TestProcDetected_ClearsStuckWhenProcessGone(t *testing.T) { + m := runningModel() + m.proc.stuck = true + + updated, _ := m.Update(procDetectedMsg{ + det: &procctl.Detection{Status: procctl.StatusStopped}, + }) + res := updated.(model) + assert.False(t, res.proc.stuck, + "stuck must clear when subsequent detect shows process is gone") +} + +// Force Stop success clears Stuck. +func TestActionResultMsg_ForceStopOkClearsStuck(t *testing.T) { + m := runningModel() + m.proc.stuck = true + + updated, _ := m.Update(actionResultMsg{op: opForceStop, result: "ok"}) + res := updated.(model) + assert.False(t, res.proc.stuck) +} + +// Force Stop on a non-stuck node is a silent no-op (no pre-emptive SIGKILL). +func TestHandleForceStopKey_NoOpWhenNotStuck(t *testing.T) { + // Absent PID file so checkSystemd() stays deterministic across machines. + t.Setenv("MITHRIL_PID_FILE", filepath.Join(t.TempDir(), "mithril.pid")) + m := runningModel() + cmd := m.handleForceStopKey() + assert.Nil(t, cmd, "Force Stop on a non-stuck running mithril is a no-op") + assert.False(t, m.confirmActive, "no modal should open") +} + +// Stuck → Force Stop opens a heavyweight warning modal. +func TestHandleForceStopKey_OpensSevereModalWhenStuck(t *testing.T) { + // Absent PID file → checkSystemd() returns "" → Force Stop modal, not a systemd one. + t.Setenv("MITHRIL_PID_FILE", filepath.Join(t.TempDir(), "mithril.pid")) + m := runningModel() + m.proc.stuck = true + + cmd := m.handleForceStopKey() + assert.Nil(t, cmd, "Force Stop opens a modal first") + assert.True(t, m.confirmActive) + assert.Contains(t, m.confirmTitle, "Force Stop") + assert.Contains(t, m.confirmTitle, "Data Loss", + "title must convey severity") + assert.Contains(t, m.confirmBody, "SIGKILL", "body must name the signal") + assert.Contains(t, m.confirmBody, "AccountsDB", "body must warn about data risk") +} + +// Even with a stale stuck=true, refuse Force Stop if no longer Running. +func TestHandleForceStopKey_NoOpWhenNotRunning(t *testing.T) { + t.Setenv("MITHRIL_PID_FILE", filepath.Join(t.TempDir(), "mithril.pid")) + m := stoppedModel(procctl.StatusStopped) + m.proc.stuck = true // stale flag + + cmd := m.handleForceStopKey() + assert.Nil(t, cmd) + // modal stays closed = refusal + assert.False(t, m.confirmActive, "no Force Stop modal may open when not Running") +} + +// Stuck-running shows all three keys: [x] Stop, [r] Restart, [f] Force Stop. +func TestRenderActionHints_StuckRunningShowsForceStop(t *testing.T) { + dim := dummyStyle() + out := renderActionHints(procctl.StatusRunning, "", true, nil, dim) + assert.Contains(t, out, "[f]") + assert.Contains(t, out, "Force Stop") +} + +// The three documented spawn tokens runLive emits are translated. +func TestFriendlySpawnedBy_TranslatesKnownTokens(t *testing.T) { + cases := []struct { + in, want string + }{ + {"cli", "command line"}, + {"dashboard", "this dashboard"}, + {"external", "external (started outside the dashboard)"}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + assert.Equal(t, c.want, friendlySpawnedBy(c.in)) + }) + } +} + +// Unknown tokens fall through as-is (forward-compat). +func TestFriendlySpawnedBy_PreservesUnknown(t *testing.T) { + assert.Equal(t, "systemd", friendlySpawnedBy("systemd")) + assert.Equal(t, "", friendlySpawnedBy("")) +} + +func TestMithrilSystemdUnit_IgnoresGenericLoginServices(t *testing.T) { + for _, cgroup := range []string{ + "0::/user.slice/user-1000.slice/session-2431.scope", + "0::/system.slice/ssh.service", + "0::/system.slice/tmux.service", + } { + _, ok := mithrilSystemdUnit(cgroup) + assert.False(t, ok, "generic session/service should not block dashboard control: %s", cgroup) + } +} + +func TestMithrilSystemdUnit_DetectsMithrilUnit(t *testing.T) { + cases := []string{ + "0::/system.slice/mithril.service", + "0::/system.slice/mithril-node.service", + "11:memory:/system.slice/mithril.service", + } + for _, cgroup := range cases { + unit, ok := mithrilSystemdUnit(cgroup) + assert.True(t, ok, "expected mithril unit from cgroup: %s", cgroup) + assert.Contains(t, unit, "mithril") + } +} + +// UID-mismatch gate: returns "" when not root and doesn't crash on a missing dir. +func TestPreflightCheck_RootMustNotSpawnOverNonRootAccountsDB(t *testing.T) { + reason := checkUIDMatch("") + assert.Equal(t, "", reason, "no accountsDir → no refusal (fresh setup)") + + reason = checkUIDMatch("/nonexistent/path") + assert.Equal(t, "", reason, "missing dir → no refusal (bootstrap creates it)") +} + +func TestPreflightCheck_RefusesUnwritableLogDirectoryParent(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root can create directories regardless of mode bits") + } + parent := t.TempDir() + require.NoError(t, os.Chmod(parent, 0555)) + defer os.Chmod(parent, 0755) + + reason := checkLogPathWritable(filepath.Join(parent, "mithril-logs")) + + assert.Contains(t, reason, "cannot write logs") + assert.Contains(t, reason, "Fix with safe folders") +} + +func TestPreflightCheck_RefusesOldAccountsStateWithoutAuthorizedVoters(t *testing.T) { + dir := t.TempDir() + st := state.NewReadyState(468306806, 1084, "", "", 0, 0) + st.Cluster = "devnet" + st.ManifestEpochStakes = map[uint64]string{1084: "{}"} + require.NoError(t, st.Save(dir)) + + reason := checkAccountsStateCompatible(&configData{cluster: "devnet"}, dir) + + assert.Contains(t, reason, "cannot be resumed") + assert.Contains(t, reason, "manifest_epoch_authorized_voters") +} + +func TestPreflightCheck_RefusesAccountsStateClusterMismatch(t *testing.T) { + dir := t.TempDir() + st := state.NewReadyState(1, 1, "", "", 0, 0) + st.Cluster = "mainnet-beta" + require.NoError(t, st.Save(dir)) + + reason := checkAccountsStateCompatible(&configData{cluster: "devnet"}, dir) + + assert.Contains(t, reason, "different cluster") + assert.Contains(t, reason, "mainnet-beta") + assert.Contains(t, reason, "devnet") +} + +// Stuck surfaces the data-loss-risk warning plus the [f] Force Stop hint. +func TestRenderProcessView_StuckShowsBanner(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }, + stuck: true, + fetchedAt: time.Now(), + }, + } + out := m.renderProcessView() + assert.Contains(t, out, "Stop timed out", "should explain WHY stuck is set") + assert.Contains(t, out, "[f]", "should advertise Force Stop key") + assert.Contains(t, out, "Force Stop") + assert.Contains(t, out, "data loss risk", "must warn about destructive op") +} + +// preflightErr surfaces clearly so the user knows why Start was refused. +func TestRenderProcessView_PreflightErrShowsRefusalBanner(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusStopped}, + preflightErr: "Mithril is managed by systemd. The dashboard would fight\n" + + "systemd's restart loop. To control it, use:\n\n" + + " sudo systemctl stop mithril", + fetchedAt: time.Now(), + }, + } + out := m.renderProcessView() + assert.Contains(t, out, "Setup needs attention", "must headline the blocker") + assert.Contains(t, out, "managed by systemd") + assert.Contains(t, out, "systemctl", "must surface remediation command") +} + +func TestRunActions_PreflightShowsSafeFolderFixBeforeStart(t *testing.T) { + m := stoppedModel(procctl.StatusStopped) + m.runFocused = true + m.proc.preflightErr = "Stored node data cannot be resumed safely." + + actions := m.runActions() + + require.NotEmpty(t, actions) + assert.Equal(t, runActionSafe, actions[0].id) + for _, action := range actions { + assert.NotEqual(t, runActionStart, action.id, "Start should not be offered while blockers are present") + } + out := m.renderProcessView() + assert.Contains(t, out, "Fix with safe folders") + assert.Contains(t, out, "Setup needs attention") +} + +func TestApplySafeStoragePathsCreatesUserOwnedFoldersAndUpdatesConfig(t *testing.T) { + // Stub disk to "plenty"; the safe-folders guard refuses too-small disks. + orig := diskFreeGBFn + diskFreeGBFn = func(string) (uint64, bool) { return 100000, true } + t.Cleanup(func() { diskFreeGBFn = orig }) + + home := t.TempDir() + t.Setenv("HOME", home) + configFile := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(configFile, []byte(` +[network] +cluster = "devnet" +rpc = ["https://api.devnet.solana.com"] + +[storage] +accounts = "/mnt/mithril-accounts" +snapshots = "/mnt/mithril-ledger/snapshots" +shredstore = "/mnt/mithril-ledger/shredstore" +logs = "/mnt/mithril-logs" + +[bootstrap] +mode = "accountsdb" +`), 0600)) + + summary, err := applySafeStoragePaths(configFile, &configData{cluster: "devnet"}) + require.NoError(t, err) + + root := filepath.Join(home, "mithril-data", "devnet") + assert.Contains(t, summary, root) + for _, dir := range []string{ + filepath.Join(root, "accounts"), + filepath.Join(root, "snapshots"), + filepath.Join(root, "shredstore"), + filepath.Join(root, "logs"), + } { + info, err := os.Stat(dir) + require.NoError(t, err) + assert.True(t, info.IsDir()) + } + cfg := readConfig(configFile) + require.NotNil(t, cfg) + assert.Equal(t, filepath.Join(root, "accounts"), cfg.accountsPath) + assert.Equal(t, filepath.Join(root, "snapshots"), cfg.snapshotsPath) + assert.Equal(t, filepath.Join(root, "shredstore"), cfg.shredstorePath) + assert.Equal(t, filepath.Join(root, "logs"), cfg.logsPath) + assert.Equal(t, "auto", cfg.bootstrapMode) +} + +// Too-small disk: refuse with actionable error, leave config unchanged. +func TestApplySafeStoragePathsRefusesTooSmallDisk(t *testing.T) { + orig := diskFreeGBFn + diskFreeGBFn = func(string) (uint64, bool) { return 10, true } // 10 GB free + t.Cleanup(func() { diskFreeGBFn = orig }) + + home := t.TempDir() + t.Setenv("HOME", home) + configFile := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(configFile, []byte(` +[network] +cluster = "mainnet-beta" +[storage] +accounts = "/mnt/mithril-accounts" +`), 0600)) + + _, err := applySafeStoragePaths(configFile, &configData{cluster: "mainnet-beta"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "10 GB free") + assert.Contains(t, err.Error(), "larger disk") + // config must stay unchanged on refusal + cfg := readConfig(configFile) + require.NotNil(t, cfg) + assert.Equal(t, "/mnt/mithril-accounts", cfg.accountsPath) +} diff --git a/cmd/mithril/dashboardcmd/process_view.go b/cmd/mithril/dashboardcmd/process_view.go new file mode 100644 index 000000000..679afa50f --- /dev/null +++ b/cmd/mithril/dashboardcmd/process_view.go @@ -0,0 +1,1339 @@ +package dashboardcmd + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/Overclock-Validator/mithril/pkg/tui" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +const ( + runActionStart = "start" + runActionStop = "stop" + runActionRestart = "restart" + runActionForce = "force" + runActionLogs = "logs" + runActionRawLogs = "raw_logs" + runActionDoctor = "doctor" + runActionEdit = "edit" + runActionSafe = "safe_folders" + runActionInPlace = "rebuild_in_place" +) + +type runAction struct { + id string + label string + desc string + key string + danger bool + disabled bool +} + +type startFlowState struct { + active bool + step int + onDone func(*model) tea.Cmd +} + +type startFlowCard struct { + kicker string + title string + detail string + positive bool + warn bool +} + +// renderProcessView renders the "Run Node" view: status badge, then a two-column +// body (actions left, activity/status right). Narrow panes stack into one column. +func (m model) renderProcessView() string { + var b strings.Builder + label := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + dim := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + + // PID file unreadable (corrupt, permission denied, etc.). + if m.proc.fetchErr != "" { + warn := lipgloss.NewStyle().Foreground(tui.ColorError) + b.WriteString(" " + warn.Render("⚠ ") + value.Render("Cannot determine status") + "\n\n") + b.WriteString(" " + dim.Render(m.proc.fetchErr) + "\n") + return b.String() + } + + // Loading path: first tick hasn't arrived yet. + if m.proc.detection == nil { + b.WriteString(" " + dim.Render("Loading process status…") + "\n") + return b.String() + } + + det := m.proc.detection + b.WriteString(" " + renderProcessHeadline(*det, m.proc.inFlightOp) + "\n") + + // Start failure: keep directly under the badge so it isn't pushed offscreen. + if m.proc.startFailStderr != "" { + warn := lipgloss.NewStyle().Foreground(tui.ColorError) + b.WriteString(" " + warn.Render("Start failed — last output from mithril:") + "\n") + for _, line := range strings.Split(strings.TrimSpace(m.proc.startFailStderr), "\n") { + b.WriteString(" " + dim.Render(redactLogLine(line)) + "\n") + } + b.WriteString("\n") + } + + // Guided-fix failure (e.g. safe folders refused — disk too small). + if m.proc.configFixErr != "" { + warnHdr := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + warnBody := lipgloss.NewStyle().Foreground(tui.ColorWarn) + b.WriteString(" " + warnHdr.Render("⚠ Couldn't apply the fix") + "\n") + for _, line := range wrapRunDetailLines([]string{m.proc.configFixErr}, m.runPanelContentWidth()) { + b.WriteString(" " + warnBody.Render(line) + "\n") + } + b.WriteString(" " + warnBody.Render("Set storage to a larger disk in Edit Config, then try again.") + "\n\n") + } + + // Body: two columns when wide, stacked when narrow. + cw := m.runPanelContentWidth() + actions := m.runActions() + if len(actions) > 0 { + selected := m.runActionIdx + if selected < 0 || selected >= len(actions) { + selected = 0 + } + if cw >= 72 { + leftWidth, rightWidth := runColumnWidths(cw) + left := m.actionColumnLines(actions, selected, leftWidth) + right := m.infoColumnLines(*det, rightWidth) + b.WriteString(joinColumns(left, right, leftWidth, rightWidth)) + } else { + for _, line := range m.actionColumnLines(actions, selected, cw) { + b.WriteString(" " + strings.TrimRight(line, " ") + "\n") + } + if info := m.infoColumnLines(*det, cw); len(info) > 0 { + b.WriteString("\n") + for _, line := range info { + b.WriteString(" " + strings.TrimRight(line, " ") + "\n") + } + } + } + b.WriteString("\n") + + if diskBlock := renderDiskSafety(m.disks, label, value, dim); diskBlock != "" { + b.WriteString(diskBlock) + b.WriteString("\n") + } + } + + // Stuck: prior Stop timed out, process still alive. + if m.proc.stuck && m.proc.inFlightOp == "" { + warn := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + b.WriteString("\n") + b.WriteString(" " + warn.Render("⚠ Stop timed out — mithril is taking longer than expected.") + "\n") + b.WriteString(" " + dim.Render("This is normal during AccountsDB rebuild (can take 30+ min).") + "\n") + b.WriteString(" " + dim.Render("Wait, or use [f] Force Stop (data loss risk).") + "\n") + } + + // In-flight action: progress block + dynamic status line. + if m.proc.inFlightOp != "" { + b.WriteString("\n") + b.WriteString(renderActionProgress(m.proc, dim, value)) + } + + // Pre-flight / control refusal — surfaced verbatim. + if m.proc.preflightInfo != "" { + ok := lipgloss.NewStyle().Foreground(tui.ColorSuccess).Bold(true) + b.WriteString("\n") + b.WriteString(" " + ok.Render("Safe folders are ready") + "\n") + for _, line := range wrapRunDetailLines([]string{m.proc.preflightInfo}, m.runPanelContentWidth()) { + b.WriteString(" " + value.Render(line) + "\n") + } + } + if m.shouldShowPreflightBlock(det.Status) { + warn := lipgloss.NewStyle().Foreground(tui.ColorError).Bold(true) + b.WriteString("\n") + b.WriteString(" " + warn.Render("Setup needs attention before Start") + "\n") + for _, line := range strings.Split(m.proc.preflightErr, "\n") { + if strings.TrimSpace(line) == "" { + b.WriteString("\n") + continue + } + for _, wrapped := range wrapRunDetailLines([]string{line}, m.runPanelContentWidth()) { + b.WriteString(" " + value.Render(wrapped) + "\n") + } + } + } + + if !m.proc.fetchedAt.IsZero() { + b.WriteString("\n") + b.WriteString(" " + dim.Render(fmt.Sprintf("Refreshed %s", humanizeAge(m.proc.fetchedAt))) + "\n") + } + return b.String() +} + +func (m model) shouldShowPreflightBlock(status procctl.Status) bool { + return m.proc.preflightErr != "" && + m.proc.inFlightOp == "" && + (status == procctl.StatusStopped || status == procctl.StatusCrashed) +} + +// runColumnWidths splits contentWidth into left (actions) + 3-cell divider + +// right (activity/status), summing exactly so the right edge stays flush. +func runColumnWidths(contentWidth int) (left, right int) { + left = contentWidth * 38 / 100 + if left < 26 { + left = 26 + } + if left > 36 { + left = 36 + } + right = contentWidth - left - 3 + if right < 24 { + right = 24 + } + return left, right +} + +// joinColumns lays two width-exact line slices side by side with a vertical +// divider, padding the shorter column. Cells must be pre-constrained; no truncation. +func joinColumns(left, right []string, leftWidth, rightWidth int) string { + rows := len(left) + if len(right) > rows { + rows = len(right) + } + bar := lipgloss.NewStyle().Foreground(tui.ColorBorder).Render("│") + leftBlank := strings.Repeat(" ", leftWidth) + rightBlank := strings.Repeat(" ", rightWidth) + + var b strings.Builder + for i := 0; i < rows; i++ { + l := leftBlank + if i < len(left) { + l = left[i] + } + r := rightBlank + if i < len(right) { + r = right[i] + } + b.WriteString(" " + l + " " + bar + " " + r + "\n") + } + return b.String() +} + +// actionColumnLines builds the left column: the action list plus the selected +// action's hint, every line padded to width. +func (m model) actionColumnLines(actions []runAction, selected, width int) []string { + head := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + dim := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + muted := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) + selectedStyle := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + dangerStyle := lipgloss.NewStyle().Foreground(tui.ColorError).Bold(true) + + var lines []string + add := func(style lipgloss.Style, text string) { + lines = append(lines, fixedWidthRender(style, text, width)) + } + + add(head, "Choose action") + for i, action := range actions { + marker := " " + titleStyle := value + if m.runFocused && i == selected { + marker = "▶ " + titleStyle = selectedStyle + } + if action.danger { + titleStyle = dangerStyle + } + if action.disabled { + titleStyle = muted + } + add(titleStyle, marker+action.label) + } + + // Selected action's hint, paired with the highlighted row above it. + detail := wrapRunDetailLines(m.runActionDetailParagraphs(actions[selected]), width) + if len(detail) > 0 { + lines = append(lines, strings.Repeat(" ", width)) // breathing room + for i, line := range detail { + st := dim + if i == 0 { + st = value + } + add(st, line) + } + } + return lines +} + +// infoColumnLines builds the right column: activity feed above per-status +// detail. Returns width-exact lines ready for joinColumns. +func (m model) infoColumnLines(det procctl.Detection, width int) []string { + lines := m.activityFeedLines(det.Status, width) + detail := m.statusDetailLines(det, width) + if len(detail) > 0 { + if len(lines) > 0 { + lines = append(lines, strings.Repeat(" ", width)) + } + lines = append(lines, detail...) + } + return lines +} + +// activityFeedLines renders recent progress events, only when the node is live, +// finishing, or erroring — otherwise an idle screen shows stale history. +func (m model) activityFeedLines(status procctl.Status, width int) []string { + events := m.progress + if len(events) == 0 { + return nil + } + show := status == procctl.StatusRunning || m.proc.inFlightOp != "" + if !show { + last := events[len(events)-1] + show = last.Status == "error" || last.Status == "warn" || + last.Phase == "completed" || last.Phase == "shutdown" + } + if !show { + return nil + } + + const maxRows = 5 + start := len(events) - maxRows + if start < 0 { + start = 0 + } + recent := events[start:] + current := recent[len(recent)-1] + + head := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + dim := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + + var lines []string + add := func(style lipgloss.Style, text string) { + lines = append(lines, fixedWidthRender(style, text, width)) + } + + add(head, "Recent activity") + for i, ev := range recent { + text := progressEventSymbol(ev) + " " + progressEventTitle(ev) + progressEventDetail(ev) + style := dim + if i == len(recent)-1 { + style = value + if !ev.TS.IsZero() { + text += " " + humanizeAge(ev.TS) + } + } + add(style, text) + } + if summary := bootstrapActivitySummary(m.snapshot, m.accounts, current); summary != "" { + for _, line := range wrapRunDetailLines([]string{summary}, width) { + add(dim, line) + } + } + return lines +} + +// statusDetailLines renders the per-status detail for the right column: +// session (running), last exit (stopped), or crash guidance (crashed). +func (m model) statusDetailLines(det procctl.Detection, width int) []string { + head := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + label := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + dim := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + + var lines []string + add := func(style lipgloss.Style, text string) { + lines = append(lines, fixedWidthRender(style, text, width)) + } + addWrapped := func(style lipgloss.Style, text string) { + for _, line := range wrapRunDetailLines([]string{text}, width) { + add(style, line) + } + } + + switch det.Status { + case procctl.StatusRunning: + add(head, "Session") + if det.Pid != 0 { + add(value, fmt.Sprintf("PID %d", det.Pid)) + } + if det.RunID != "" { + addWrapped(label, "Session "+det.RunID) + } + if det.SpawnedBy != "" { + addWrapped(label, "Started by "+friendlySpawnedBy(det.SpawnedBy)) + } + if det.BinaryPath != "" { + addWrapped(label, "Program "+det.BinaryPath) + } + if det.ConfigPath != "" { + addWrapped(label, "Config "+det.ConfigPath) + } + if det.LogDir != "" { + addWrapped(label, "Logs "+det.LogDir) + } + if !det.LockHeld { + addWrapped(dim, "Live, but the single-instance lock is not held. Start is disabled until this process stops.") + } + // Concurrent stop in progress — surface it so Stop isn't hammered. + if det.StopInProgressBy != 0 { + owner := "Another dashboard is shutting this node down." + if det.StopInProgressBy == os.Getpid() { + owner = "This dashboard is shutting this node down." + } + addWrapped(value, owner) + if !det.StopInProgressAt.IsZero() { + addWrapped(dim, fmt.Sprintf("(stop by pid %d, %s)", det.StopInProgressBy, humanizeAge(det.StopInProgressAt))) + } + } + + case procctl.StatusStopped: + if isReplayCompleted(det.LastShutdownReason) { + add(head, "Last run") + addWrapped(value, "completed configured replay range") + addWrapped(dim, "A clean finish, not a crash. Safe to start again.") + return lines + } + add(head, "Last exit") + if det.LastShutdownReason == "" || det.LastShutdownReason == "no state file" { + addWrapped(dim, "This node has never been started from this dashboard on this machine.") + } else { + addWrapped(value, det.LastShutdownReason) + } + if det.LastCleanExit { + addWrapped(dim, "Previous shutdown was clean. Safe to start.") + } else if det.LastShutdownReason == "no state file" { + addWrapped(dim, "No shutdown record. Safe to start if this is first setup.") + } + + case procctl.StatusCrashed: + add(head, "Last exit") + if det.LastShutdownReason != "" { + addWrapped(value, det.LastShutdownReason) + } + // Divergence lines kept single-line (tests assert exact phrasing). + crashText := m.crashDiagnosticText() + switch { + case isReplayDivergenceText(crashText) || isReplayDivergenceText(det.LastShutdownReason): + add(value, "Replay diverged from chain data.") + add(dim, "Safe path: do not retry this AccountsDB.") + add(dim, "Next step: rebuild from a fresh snapshot.") + case isRPCRateLimitOrStall(det.LastShutdownReason): + addWrapped(dim, "RPC catchup stalled or was rate-limited.") + addWrapped(dim, "Use a private/dedicated RPC endpoint.") + default: + addWrapped(dim, "The node stopped unexpectedly.") + addWrapped(dim, "Open Doctor before starting again.") + } + } + return lines +} + +func (m *model) beginStartFlow(onDone func(*model) tea.Cmd) { + m.startFlow = startFlowState{ + active: true, + step: 0, + onDone: onDone, + } + m.rightScroll = 0 +} + +func (m *model) cancelStartFlow() { + m.startFlow = startFlowState{} +} + +func (m *model) advanceStartFlow() tea.Cmd { + if !m.startFlow.active { + return nil + } + // Single confirm: one Enter starts. + onDone := m.startFlow.onDone + m.cancelStartFlow() + if onDone != nil { + return onDone(m) + } + return nil +} + +// renderStartFlow draws a single compact "Confirm run" card: the few things +// that matter (mode, network, storage, start point) and one Enter to start. +func (m model) renderStartFlow() string { + header := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + label := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + key := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + hint := lipgloss.NewStyle().Foreground(tui.ColorTextSecondary) + warn := lipgloss.NewStyle().Foreground(tui.ColorWarn) + + warnBold := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + + var b strings.Builder + row := func(k, v string, st lipgloss.Style) { + b.WriteString(" " + label.Render(fmt.Sprintf("%-9s", k)) + st.Render(v) + "\n") + } + cont := func(v string, st lipgloss.Style) { // continuation line, aligned under a row's value + b.WriteString(" " + label.Render(fmt.Sprintf("%-9s", "")) + st.Render(v) + "\n") + } + + // Title + a quiet mode · network subtitle. + b.WriteString("\n " + header.Render("Confirm run") + "\n") + ctx := confirmModeSummary(m.cfg) + if net := confirmNetwork(m.cfg); net != "" { + ctx += " · " + net + } + b.WriteString(" " + label.Render(ctx) + "\n\n") + + // The plan: where data goes and what Start will do. + if storage := confirmStorage(m.cfg); storage != "" { + row("Storage", storage, value) + } + if bc := m.startFlowBootstrapCard(); bc != nil { + st := value + if bc.warn { + st = warn + } + row("Plan", bc.title, st) + if bc.detail != "" { + cont(bc.detail, label) + } + } + + // Disk readiness — fresh builds only. + if m.startFlowWillBuild() && m.cfg != nil && m.cfg.accountsPath != "" && + !config.HasExistingAccountsDb(m.cfg.accountsPath) { + if free, ok := config.FreeDiskBytes(m.cfg.accountsPath); ok { + // Snapshot on the same disk must hold both at peak. + need := config.EstimatedAccountsDbBytes(m.cfg.cluster) + if m.cfg.snapshotsPath != "" && config.SameDisk(m.cfg.accountsPath, m.cfg.snapshotsPath) { + need = config.EstimatedBuildBytes(m.cfg.cluster) + } + freeGB, needGB := free/(1<<30), need/(1<<30) + b.WriteString("\n") + if free < need { + b.WriteString(" " + warnBold.Render("⚠ Low disk space") + "\n") + b.WriteString(" " + warn.Render(fmt.Sprintf("%d GB free here — the AccountsDB needs ~%d GB.", freeGB, needGB)) + "\n") + b.WriteString(" " + warn.Render("Point storage at a larger disk, or Start anyway to retry.") + "\n") + } else { + row("Disk", fmt.Sprintf("%d GB free — enough for the AccountsDB", freeGB), value) + } + } + } + + // Quiet notes: inform but don't block (UDP ports, RPC exposure, rate-limit). + var notes []string + if m.cfg != nil && m.cfg.lbEnabled { + if ports := managedLightbringerNetworkShort(m.cfg); ports != "" { + notes = append(notes, ports) + } + } + if exposure := rpcServerExposureSummary(m.cfg); exposure != "" { + notes = append(notes, exposure) + } + if note := rpcRuntimeSummary(m.cfg); note != "" { + notes = append(notes, note) + } + if len(notes) > 0 { + b.WriteString("\n") + for _, n := range notes { + b.WriteString(" " + label.Render("· "+n) + "\n") + } + } + + b.WriteString("\n " + key.Render("⏎") + hint.Render(" Start ") + + key.Render("esc") + hint.Render(" Cancel") + "\n") + return b.String() +} + +// confirmModeSummary / confirmNetwork / confirmStorage are the short lines on +// the Confirm-run card. +func confirmModeSummary(cfg *configData) string { + if cfg == nil { + return "loading…" + } + if cfg.lbEnabled { + return "Mithril + Lightbringer (managed)" + } + if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { + return "Mithril + external Lightbringer" + } + return "Mithril only · blocks from RPC" +} + +func confirmNetwork(cfg *configData) string { + if cfg == nil || cfg.cluster == "" || cfg.cluster == "unknown" { + return "" + } + return cfg.cluster +} + +func confirmStorage(cfg *configData) string { + if cfg == nil { + return "" + } + return cfg.accountsPath +} + +// startFlowWillBuild reports whether starting now builds the AccountsDB (vs +// resuming). Gates the disk-readiness row; mirrors startFlowBootstrapCard. +func (m model) startFlowWillBuild() bool { + if m.cfg == nil { + return false + } + switch strings.TrimSpace(m.cfg.bootstrapMode) { + case "new-snapshot", "snapshot": + return true + case "accountsdb": + return false + default: // auto — builds only when there's no existing valid state + return m.state == nil + } +} + +func (m model) startFlowBootstrapCard() *startFlowCard { + if m.cfg == nil { + return nil + } + mode := strings.TrimSpace(m.cfg.bootstrapMode) + if mode == "" { + mode = "auto" + } + + switch mode { + case "new-snapshot": + detail := "Build a fresh AccountsDB from snapshot." + if m.state != nil || m.accounts.hasData(m.cfg.accountsPath) { + detail = "Existing AccountsDB will be replaced." + } + return &startFlowCard{ + kicker: "Local data", + title: "Fresh rebuild", + detail: detail, + warn: true, + } + case "snapshot": + return &startFlowCard{ + kicker: "Local data", + title: "Rebuild from snapshot", + detail: "AccountsDB will be rebuilt before replay.", + warn: true, + } + case "accountsdb": + return &startFlowCard{ + kicker: "Local data", + title: "Use existing AccountsDB", + detail: "Start fails fast if local state is missing.", + } + default: + if m.state != nil { + return &startFlowCard{ + kicker: "Local data", + title: "Resume existing state", + detail: "Auto mode reuses valid AccountsDB.", + } + } + return &startFlowCard{ + kicker: "Local data", + title: "Create local state", + detail: "Auto mode downloads a snapshot if needed.", + } + } +} + +func managedLightbringerNetworkShort(cfg *configData) string { + if cfg == nil { + return "HTTP/gRPC stay local." + } + gossipPort, rangeStart, rangeEnd, err := parseLightbringerGossipPorts(cfg.lbGossipPort, cfg.lbPortRangeStart, cfg.lbPortRangeEnd) + if err != nil { + return "Check UDP ports in config." + } + return fmt.Sprintf("Open inbound UDP %d and %d-%d.", gossipPort, rangeStart, rangeEnd) +} + +func (m model) runPanelContentWidth() int { + if m.width <= 0 { + return 60 + } + if m.width < 60 { + return m.width - 4 + } + innerWidth := m.width - 3 + leftWidth := innerWidth * 22 / 100 + rightWidth := innerWidth - leftWidth + contentWidth := rightWidth - 2 // renderProcessView uses a two-space left inset. + if contentWidth < 42 { + return 42 + } + return contentWidth +} + +func (m model) runActionDetailParagraphs(action runAction) []string { + lines := []string{} + if action.desc != "" { + lines = append(lines, action.desc) + } + if action.id == runActionStart { + lines = append(lines, runModeSummary(m.cfg)) + } + if action.id == runActionRawLogs { + lines = append(lines, "Full-width raw tail. Same run, same log files.") + } + if action.id == runActionEdit { + lines = append(lines, "Review run mode, RPC, storage, and Lightbringer ports.") + } + if action.id == runActionSafe { + lines = append(lines, "Creates user-owned folders and updates storage paths. Existing data is not deleted.") + if m.hasReplayDivergenceCrash() { + lines = append(lines, "This avoids reusing the diverged AccountsDB that just crashed.") + } + if root := safeStorageRootForDisplay(m.cfg); root != "" { + line := "Target: " + root + // Show free space before confirming, so the operator sees if the + // target disk can hold the AccountsDB. + if free, ok := pathFreeGB(root); ok { + if need := estimatedAccountsDbGB(m.cfg); free < need { + line += fmt.Sprintf(" ⚠ only %dG free — needs ~%dG; pick a larger disk", free, need) + } else { + line += fmt.Sprintf(" (%dG free)", free) + } + } + lines = append(lines, line) + } + } + if action.disabled { + lines = append(lines, "Wait for the current operation to finish.") + } + if m.runFocused { + lines = append(lines, runActionEnterHint(action.id)+" Esc back.") + } else { + lines = append(lines, "Use → to focus actions.") + } + return lines +} + +func runActionEnterHint(actionID string) string { + switch actionID { + case runActionStart: + return "Enter to start." + case runActionStop: + return "Enter to stop." + case runActionRestart: + return "Enter to restart." + case runActionForce: + return "Enter to force stop." + case runActionLogs, runActionRawLogs: + return "Enter to open." + case runActionDoctor: + return "Enter to check." + case runActionEdit: + return "Enter to edit." + case runActionSafe: + return "Enter to fix." + default: + return "Enter to continue." + } +} + +func runModeSummary(cfg *configData) string { + if cfg == nil { + return "Run mode comes from your config." + } + if cfg.lbEnabled { + return "Mithril with a managed Lightbringer sidecar." + } + if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { + return "Mithril with an external Lightbringer." + } + if cfg.blockSource == "rpc" || cfg.blockSource == "" { + return "Mithril alone — blocks come from your RPC provider." + } + return "Set Block Source in your config." +} + +func wrapRunDetailLines(paragraphs []string, width int) []string { + if width <= 0 { + return nil + } + var lines []string + for _, paragraph := range paragraphs { + words := strings.Fields(paragraph) + if len(words) == 0 { + continue + } + line := words[0] + for _, word := range words[1:] { + if lipgloss.Width(line)+1+lipgloss.Width(word) > width { + lines = append(lines, line) + line = word + continue + } + line += " " + word + } + lines = append(lines, line) + } + return lines +} + +func fixedWidthRender(style lipgloss.Style, text string, width int) string { + if width <= 0 { + return "" + } + rendered := style.MaxWidth(width).Render(text) + pad := width - lipgloss.Width(rendered) + if pad > 0 { + rendered += strings.Repeat(" ", pad) + } + return rendered +} + +func (m model) runActions() []runAction { + if m.proc.detection == nil { + return nil + } + busy := m.proc.inFlightOp != "" + switch m.proc.detection.Status { + case procctl.StatusRunning: + actions := []runAction{ + {id: runActionLogs, label: "Watch live logs", desc: "Open Mithril and Lightbringer logs."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Full-width live tail."}, + {id: runActionStop, label: "Stop safely", desc: "Clean shutdown with SIGTERM.", key: "x", disabled: busy}, + {id: runActionRestart, label: "Restart node", desc: "Stop cleanly, then start again.", key: "r", disabled: busy}, + } + if m.proc.stuck { + actions = append(actions, runAction{ + id: runActionForce, label: "Force stop", desc: "Last resort only. May corrupt AccountsDB.", key: "f", danger: true, disabled: busy, + }) + } + return actions + case procctl.StatusStopped: + if m.shouldShowPreflightBlock(procctl.StatusStopped) { + return []runAction{ + {id: runActionSafe, label: "Fix with safe folders", desc: "Recommended. Create user-owned folders and keep old data untouched."}, + {id: runActionEdit, label: "Review config", desc: "Change storage paths manually."}, + {id: runActionDoctor, label: "Check readiness", desc: "Run health checks first."}, + {id: runActionLogs, label: "Open latest logs", desc: "Inspect output from the last run."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Full-width tail of the last run."}, + } + } + return []runAction{ + {id: runActionStart, label: primaryStartLabel(m.cfg), desc: "Run this config; logs open automatically.", key: "s", disabled: busy}, + {id: runActionDoctor, label: "Check readiness", desc: "Run health checks first."}, + {id: runActionEdit, label: "Review config", desc: "Edit run mode, RPC, storage, or ports."}, + {id: runActionLogs, label: "Open latest logs", desc: "Inspect output from the last run."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Full-width tail of the last run."}, + } + case procctl.StatusCrashed: + if m.hasReplayDivergenceCrash() { + return []runAction{ + {id: runActionSafe, label: "Rebuild fresh safely", desc: "Recommended when a spare disk has room. Build into fresh folders and keep the old AccountsDB untouched."}, + {id: runActionInPlace, label: "Rebuild in place (reclaim disk)", desc: "No spare disk? Wipe the current AccountsDB and rebuild from snapshot using the existing storage paths.", danger: true}, + {id: runActionEdit, label: "Set new snapshot", desc: "Set Bootstrap Mode to new-snapshot or choose fresh storage paths."}, + {id: runActionLogs, label: "Open latest logs", desc: "Inspect the divergence details."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Read last output full-width."}, + {id: runActionDoctor, label: "Check what happened", desc: "Open health checks before retrying if unsure."}, + } + } + if m.shouldShowPreflightBlock(procctl.StatusCrashed) { + return []runAction{ + {id: runActionSafe, label: "Fix with safe folders", desc: "Recommended. Create user-owned folders and keep old data untouched."}, + {id: runActionLogs, label: "Open latest logs", desc: "Inspect the last run output."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Read last output full-width."}, + {id: runActionEdit, label: "Review config", desc: "Check settings before retrying."}, + {id: runActionDoctor, label: "Check what happened", desc: "Open health checks before retrying if unsure."}, + } + } + return []runAction{ + {id: runActionStart, label: primaryStartLabel(m.cfg), desc: "Start again when the config looks safe.", key: "s", disabled: busy}, + {id: runActionDoctor, label: "Check what happened", desc: "Open health checks before retrying if unsure."}, + {id: runActionLogs, label: "Open latest logs", desc: "Inspect the last run output."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Read last output full-width."}, + {id: runActionEdit, label: "Review config", desc: "Check settings before retrying."}, + } + default: + return []runAction{ + {id: runActionDoctor, label: "Check readiness", desc: "Review health checks."}, + {id: runActionLogs, label: "Open logs", desc: "Inspect recent output."}, + {id: runActionRawLogs, label: "Terminal logs", desc: "Full-width tail."}, + } + } +} + +func (m *model) clampRunActionCursor() { + actions := m.runActions() + if len(actions) == 0 { + m.runActionIdx = 0 + return + } + if m.runActionIdx < 0 { + m.runActionIdx = 0 + } + if m.runActionIdx >= len(actions) { + m.runActionIdx = len(actions) - 1 + } +} + +func (m *model) moveRunAction(delta int) { + actions := m.runActions() + if len(actions) == 0 { + m.runActionIdx = 0 + return + } + m.runActionIdx = (m.runActionIdx + delta + len(actions)) % len(actions) +} + +func (m *model) activateRunAction() tea.Cmd { + actions := m.runActions() + if len(actions) == 0 { + return nil + } + m.clampRunActionCursor() + action := actions[m.runActionIdx] + if action.disabled { + return nil + } + switch action.id { + case runActionStart: + return m.handleStartKey() + case runActionStop: + return m.handleStopKey() + case runActionRestart: + return m.handleRestartKey() + case runActionForce: + return m.handleForceStopKey() + case runActionLogs: + m.openLogs(false) + return m.fetchDataCmd() + case runActionRawLogs: + m.openLogs(true) + return m.fetchDataCmd() + case runActionDoctor: + m.screen = screenDoctor + m.runFocused = false + m.rightScroll = 0 + m.setMenuCursor("doctor") + return m.fetchDataCmd() + case runActionEdit: + m.screen = screenEdit + m.runFocused = false + m.editIdx = 0 + m.moveEditCursor(0) + m.setMenuCursor("edit") + return nil + case runActionSafe: + return m.openSafeFoldersConfirmation() + case runActionInPlace: + return m.openRebuildInPlaceConfirmation() + default: + return nil + } +} + +func rpcRuntimeSummary(cfg *configData) string { + if cfg == nil || len(cfg.rpcEndpoints) == 0 { + return "" + } + endpoint := cfg.rpcEndpoints[0] + if usesLightbringerBlocks(cfg) { + if isMainnetPublicRPC(cfg, endpoint) { + return "used for catchup; public mainnet RPC may rate-limit" + } + return "used for catchup before Lightbringer live handoff" + } + if isMainnetPublicRPC(cfg, endpoint) { + return "public mainnet RPC may rate-limit on long runs" + } + return "" +} + +func rpcServerExposureSummary(cfg *configData) string { + if cfg == nil || strings.TrimSpace(cfg.rpcPort) == "" || strings.TrimSpace(cfg.rpcPort) == "0" { + return "" + } + return "listens on all interfaces :" + strings.TrimSpace(cfg.rpcPort) + "; restrict with firewall if public" +} + +// renderActionProgress shows the active operation's name, elapsed time, and any +// recorded status lines. +func renderActionProgress(p procState, dim, value lipgloss.Style) string { + var b strings.Builder + label := opLabel(p.inFlightOp) + elapsed := time.Since(p.opStartedAt).Round(time.Second) + b.WriteString(" " + value.Render(label) + " " + dim.Render(fmt.Sprintf("(%s elapsed)", elapsed)) + "\n") + for _, line := range p.progressLines { + b.WriteString(" " + dim.Render("· "+line) + "\n") + } + return b.String() +} + +func bootstrapActivitySummary(snapshot snapshotActivity, accounts accountsActivity, current progressEvent) string { + if current.Phase != "bootstrap_snapshot" || current.Status == "error" { + return "" + } + + parts := make([]string, 0, 3) + if snapshot.active() { + kind := "Snapshot file" + if strings.HasPrefix(snapshot.Name, "incremental-snapshot-") { + kind = "Incremental snapshot" + } + if snapshot.Partial { + kind += " downloading" + } else { + kind += " saved" + } + parts = append(parts, fmt.Sprintf("%s: %s", kind, formatBytes(snapshot.Bytes))) + if !snapshot.ModTime.IsZero() { + parts = append(parts, "snapshot updated "+humanizeAge(snapshot.ModTime)) + } + } + + if accounts.active() { + parts = append(parts, "AccountsDB updated "+humanizeAge(accounts.ModTime)) + } + + if len(parts) == 0 { + return "" + } + parts = append(parts, "building AccountsDB") + return strings.Join(parts, " · ") +} + +func renderDiskSafety(disks []diskUsage, label, value, dim lipgloss.Style) string { + warnings := diskSafetyWarnings(disks) + if len(warnings) == 0 { + return "" + } + + warn := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + var b strings.Builder + b.WriteString(" " + warn.Render("Disk watch") + "\n") + for _, d := range warnings { + free := int64(d.total) - int64(d.used) + if free < 0 { + free = 0 + } + severity := "warning" + if d.pct >= 90 { + severity = "critical" + } + b.WriteString(" " + label.Render(fmt.Sprintf("%-11s", d.label)) + + value.Render(fmt.Sprintf("%d%% used", d.pct)) + + dim.Render(fmt.Sprintf(" · %dG free · %s", free, severity)) + "\n") + } + b.WriteString(" " + dim.Render("Snapshot builds can grow quickly; stop safely before the disk is full.") + "\n") + return b.String() +} + +func diskSafetyWarnings(disks []diskUsage) []diskUsage { + warnings := make([]diskUsage, 0, len(disks)) + for _, d := range disks { + if d.pct >= 80 { + warnings = append(warnings, d) + } + } + return warnings +} + +func progressEventTitle(ev progressEvent) string { + if ev.Message != "" { + return ev.Message + } + switch ev.Phase { + case "starting": + return "Preparing Mithril" + case "lightbringer_config": + return "Preparing Lightbringer" + case "lightbringer_starting": + return "Starting Lightbringer" + case "lightbringer_ready": + return "Lightbringer ready" + case "lightbringer_fallback": + return "Using RPC fallback" + case "lightbringer_external": + return "Using external Lightbringer" + case "bootstrap_checking": + return "Checking local data" + case "bootstrap_resume": + return "Opening existing AccountsDB" + case "bootstrap_snapshot": + return "Building AccountsDB from snapshot" + case "bootstrap_ready": + return "AccountsDB ready" + case "replay_starting": + return "Starting replay" + case "replay_stopped": + return "Replay stopped" + case "shutdown": + return "Stopped cleanly" + case "completed": + return "Completed" + case "error": + return "Error" + default: + return strings.ReplaceAll(ev.Phase, "_", " ") + } +} + +func progressEventSymbol(ev progressEvent) string { + switch ev.Status { + case "ok": + return "✓" + case "warn": + return "!" + case "error": + return "x" + default: + return "→" + } +} + +func progressEventDetail(ev progressEvent) string { + if len(ev.Fields) == 0 { + return "" + } + if slot, ok := numericProgressField(ev.Fields, "slot"); ok { + return fmt.Sprintf(" slot %.0f", slot) + } + if slot, ok := numericProgressField(ev.Fields, "start_slot"); ok { + return fmt.Sprintf(" from %.0f", slot) + } + if src, ok := ev.Fields["block_source"].(string); ok && src != "" { + return " " + config.RedactSecretsInText(src) + } + return "" +} + +func numericProgressField(fields map[string]any, key string) (float64, bool) { + switch v := fields[key].(type) { + case float64: + return v, true + case int: + return float64(v), true + case int64: + return float64(v), true + case uint64: + return float64(v), true + default: + return 0, false + } +} + +func renderProcessHeadline(det procctl.Detection, inFlightOp string) string { + switch inFlightOp { + case opStart: + return lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true).Render("● Starting") + case opStop: + return lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true).Render("◆ Stopping") + case opRestart: + return lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true).Render("◆ Restarting") + case opForceStop: + return lipgloss.NewStyle().Foreground(tui.ColorError).Bold(true).Render("◆ Force stopping") + default: + if det.Status == procctl.StatusStopped && isReplayCompleted(det.LastShutdownReason) { + return lipgloss.NewStyle().Foreground(tui.ColorSuccess).Bold(true).Render("✓ Completed") + } + return renderProcessStatusBadge(det.Status) + } +} + +// opLabel returns the human-readable label for an inFlightOp value. +func opLabel(op string) string { + switch op { + case opStart: + return "Starting…" + case opStop: + return "Stopping…" + case opRestart: + return "Restarting…" + case opForceStop: + return "Force stopping…" + default: + return op + } +} + +// renderActionHints emits the secondary keyboard shortcuts for the current +// status (the action list is the primary interaction). +func renderActionHints(status procctl.Status, inFlightOp string, stuck bool, cfg *configData, dim lipgloss.Style) string { + if inFlightOp != "" { + return " " + dim.Render("(actions disabled until current operation completes)") + "\n" + } + key := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + warnKey := lipgloss.NewStyle().Foreground(tui.ColorError).Bold(true) + switch status { + case procctl.StatusRunning: + hints := " " + dim.Render("Shortcuts: ") + key.Render("[x]") + dim.Render(" Stop safely ") + + key.Render("[r]") + dim.Render(" Restart") + // Stuck = prior Stop timed out; show Force Stop in red as an escalation. + if stuck { + hints += " " + warnKey.Render("[f]") + dim.Render(" Force Stop") + } + return hints + "\n" + case procctl.StatusStopped, procctl.StatusCrashed: + return " " + dim.Render("Shortcut: ") + key.Render("[s]") + dim.Render(" "+primaryStartLabel(cfg)) + "\n" + default: + return "" + } +} + +func primaryStartLabel(cfg *configData) string { + if cfg == nil { + return "Start Mithril" + } + if cfg.lbEnabled { + return "Start Mithril + Lightbringer" + } + if cfg.blockSource == "lightbringer" && cfg.lbExternalEndpoint != "" { + return "Start Mithril with external Lightbringer" + } + return "Start Mithril" +} + +// renderConfirmModal renders the in-pane confirmation for destructive actions. +// Keys are the buttons: Y/Enter = Yes, N/Esc = No. +func renderConfirmModal(title, body string, width int) string { + var b strings.Builder + header := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) + hint := lipgloss.NewStyle().Foreground(tui.ColorTextSecondary) + key := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + bodyWidth := width - 4 + if bodyWidth < 24 { + bodyWidth = 24 + } + + b.WriteString("\n") + b.WriteString(" " + header.Render("◆ "+title) + "\n\n") + for _, line := range strings.Split(strings.TrimSpace(body), "\n") { + if strings.TrimSpace(line) == "" { + b.WriteString("\n") + continue + } + for _, wrapped := range wrapRunDetailLines([]string{line}, bodyWidth) { + b.WriteString(" " + value.Render(wrapped) + "\n") + } + } + b.WriteString("\n") + if width > 0 && width < 56 { + b.WriteString(" " + key.Render("[enter]") + hint.Render(" Confirm ") + + key.Render("[y]") + hint.Render(" Yes") + "\n") + b.WriteString(" " + key.Render("[n]") + hint.Render(" No ") + + key.Render("[esc]") + hint.Render(" Cancel") + "\n") + } else { + b.WriteString(" " + key.Render("[enter]") + hint.Render(" Confirm ") + + key.Render("[y]") + hint.Render(" Yes ") + + key.Render("[n]") + hint.Render(" No ") + + key.Render("[esc]") + hint.Render(" Cancel") + "\n") + } + return b.String() +} + +// renderProcessStatusBadge produces the colored "● Running" / "○ Stopped" / +// "⚠ Crashed" headline; an unknown status renders "? Unknown". +func renderProcessStatusBadge(status procctl.Status) string { + pass := lipgloss.NewStyle().Foreground(tui.ColorSuccess).Bold(true) + warn := lipgloss.NewStyle().Foreground(tui.ColorWarn).Bold(true) + // Stopped renders muted, not red — a clean shutdown isn't a failure. + muted := lipgloss.NewStyle().Foreground(tui.ColorTextMuted).Bold(true) + switch status { + case procctl.StatusRunning: + return pass.Render("● Running") + case procctl.StatusStopped: + return muted.Render("○ Stopped") + case procctl.StatusCrashed: + return warn.Render("⚠ Crashed") + default: + return warn.Render("? Unknown") + } +} + +func (m model) hasReplayDivergenceCrash() bool { + return isReplayDivergenceText(m.crashDiagnosticText()) +} + +func (m model) crashDiagnosticText() string { + var parts []string + if m.proc.startFailStderr != "" { + parts = append(parts, m.proc.startFailStderr) + } + if len(m.mithrilLines) > 0 { + parts = append(parts, strings.Join(m.mithrilLines, "\n")) + } + if m.proc.detection != nil && m.proc.detection.LastShutdownReason != "" { + parts = append(parts, m.proc.detection.LastShutdownReason) + } + return strings.Join(parts, "\n") +} + +func isReplayDivergenceText(text string) bool { + lower := strings.ToLower(text) + return strings.Contains(lower, "divergence") || + strings.Contains(lower, "pre-balance mismatch") || + strings.Contains(lower, "pre-balance divergence") || + strings.Contains(lower, "return value divergence") +} + +func isReplayCompleted(reason string) bool { + return reason == state.ShutdownReasonCompleted +} + +func isRPCRateLimitOrStall(reason string) bool { + reason = strings.ToLower(reason) + return strings.Contains(reason, strings.ToLower(state.ShutdownReasonStall)) || + strings.Contains(reason, "429") || + strings.Contains(reason, "rate limit") || + strings.Contains(reason, "rate-limit") +} + +// friendlySpawnedBy turns the raw SpawnedBy token into a plain-English phrase, +// falling back to the raw value for unknown markers. +func friendlySpawnedBy(v string) string { + switch v { + case "cli": + return "command line" + case "dashboard": + return "this dashboard" + case "external": + return "external (started outside the dashboard)" + default: + return v + } +} + +// humanizeAge formats a time as "a moment ago" / "5s ago" / "2m ago" / "1h ago". +// The "a moment" case avoids "0s ago" on immediate refresh. +func humanizeAge(t time.Time) string { + d := time.Since(t).Round(time.Second) + switch { + case d < 2*time.Second: + return "a moment ago" + case d < time.Minute: + return fmt.Sprintf("%ds ago", int(d.Seconds())) + case d < time.Hour: + return fmt.Sprintf("%dm ago", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh ago", int(d.Hours())) + default: + return fmt.Sprintf("%dd ago", int(d.Hours()/24)) + } +} + +func formatBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + value := float64(n) + for _, suffix := range []string{"KB", "MB", "GB", "TB"} { + value /= unit + if value < unit { + return fmt.Sprintf("%.1f %s", value, suffix) + } + } + return fmt.Sprintf("%.1f PB", value/unit) +} diff --git a/cmd/mithril/dashboardcmd/process_view_test.go b/cmd/mithril/dashboardcmd/process_view_test.go new file mode 100644 index 000000000..7be06e1a9 --- /dev/null +++ b/cmd/mithril/dashboardcmd/process_view_test.go @@ -0,0 +1,823 @@ +package dashboardcmd + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/charmbracelet/lipgloss" + "github.com/stretchr/testify/assert" +) + +// Dashboard "Process" view rendering tests. + +// Pre-fetch state shows a loading message, not a blank screen. +func TestRenderProcessView_LoadingState(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + // proc.detection deliberately nil — pre-fetch state. + } + out := m.renderProcessView() + assert.Contains(t, out, "Loading process status") +} + +// Alive mithril shows the Running badge and PID. +func TestRenderProcessView_RunningShowsBadgeAndPid(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + RunID: "20260519-143209Z_abc1234_def56789", + SpawnedBy: "cli", + BinaryPath: "/home/operator/mithril/mithril", + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Running", "should show Running badge") + assert.Contains(t, out, "12345", "should show PID") + assert.Contains(t, out, "20260519-143209Z_abc1234_def56789", "should show RunID") + // "cli" is translated to "command line"; "Started by cli" is the untranslated form. + assert.Contains(t, out, "command line", "should show friendly SpawnedBy") + assert.NotContains(t, out, "Started by cli", "raw 'cli' should be translated") + assert.Contains(t, out, "/home/operator/mithril/mithril", "should show BinaryPath") +} + +// Clean exit mentions the shutdown reason and reassures the operator. +func TestRenderProcessView_StoppedShowsCleanExit(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusStopped, + LastShutdownReason: "graceful shutdown (Ctrl+C)", + LastCleanExit: true, + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Stopped") + assert.Contains(t, out, "graceful shutdown") + assert.Contains(t, out, "Safe to start") +} + +func TestRenderProcessView_CompletedShowsCleanCompletion(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusStopped, + LastShutdownReason: state.ShutdownReasonCompleted, + LastCleanExit: true, + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Completed") + assert.Contains(t, out, "completed configured replay range") + assert.Contains(t, out, "not a crash") + assert.NotContains(t, out, "○ Stopped") +} + +// First-run case (no prior run) must not show stale data. +func TestRenderProcessView_StoppedWithoutHistory(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusStopped, + // No LastShutdownReason / LastCleanExit + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Stopped") + assert.Contains(t, out, "never been started", "should explain in plain English") +} + +func TestRenderProcessView_StartFailureAppearsBeforeStoppedGuidance(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusStopped}, + startFailStderr: strings.Join([]string{ + "lightbringer: gRPC slot stream ready at 127.0.0.1:3401", + "mode=accountsdb requires existing AccountsDB", + }, "\n"), + }, + } + out := m.renderProcessView() + failIdx := strings.Index(out, "Start failed") + guidanceIdx := strings.Index(out, "never been started") + assert.NotEqual(t, -1, failIdx, "start failure should render") + assert.NotEqual(t, -1, guidanceIdx, "stopped guidance should still render") + assert.Less(t, failIdx, guidanceIdx, + "start failure must be visible before generic stopped guidance") +} + +func TestRenderProcessView_RedactsStartFailureStderr(t *testing.T) { + secretURL := "https://rpc.example.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000" + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusStopped}, + startFailStderr: "startup failed while using " + secretURL, + }, + } + + out := m.renderProcessView() + assert.Contains(t, out, "api-key=REDACTED") + assert.NotContains(t, out, "test-key-00000000") +} + +func TestRenderProcessView_StartingHeadlineOverridesRunningDetection(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + inFlightOp: opStart, + detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }, + }, + } + out := m.renderProcessView() + assert.Contains(t, out, "Starting") + assert.NotContains(t, out, "● Running", + "start in-flight should not imply the startup has completed") +} + +func TestRenderProcessView_ShowsStructuredProgress(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + progress: []progressEvent{ + { + TS: time.Now().Add(-2 * time.Minute), + Phase: "bootstrap_snapshot", + Status: "running", + Message: "Downloading snapshot and building AccountsDB", + Fields: map[string]any{"slot": float64(12345)}, + }, + { + TS: time.Now().Add(-30 * time.Second), + Phase: "replay_starting", + Status: "running", + Message: "Starting block replay", + Fields: map[string]any{"start_slot": float64(12346)}, + }, + }, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Recent activity", "activity feed renders under a header") + assert.Contains(t, out, "Starting block replay") + assert.Contains(t, out, "Downloading snapshot and building AccountsDB") + assert.Contains(t, out, "from 12346") +} + +func TestRenderProcessView_ShowsSnapshotActivityDuringBootstrap(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + progress: []progressEvent{ + { + TS: time.Now().Add(-30 * time.Second), + Phase: "bootstrap_snapshot", + Status: "running", + Message: "Downloading snapshot and building AccountsDB", + }, + }, + snapshot: snapshotActivity{ + Name: "snapshot-12345-test.tar.zst.partial", + Path: "/snapshots/snapshot-12345-test.tar.zst.partial", + Bytes: 56 * 1024 * 1024 * 1024, + Partial: true, + ModTime: time.Now().Add(-3 * time.Second), + }, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } + + out := m.renderProcessView() + + assert.Contains(t, out, "Snapshot file downloading") + assert.Contains(t, out, "56.0 GB") + assert.Contains(t, out, "building AccountsDB") +} + +func TestRenderProcessView_ShowsAccountsActivityWithoutSnapshotDownload(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + progress: []progressEvent{ + { + TS: time.Now().Add(-30 * time.Second), + Phase: "bootstrap_snapshot", + Status: "running", + Message: "Building AccountsDB from existing snapshot", + }, + }, + accounts: accountsActivity{ + Name: "accounts", + Path: "/accounts/accounts", + ModTime: time.Now().Add(-4 * time.Second), + }, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } + + out := m.renderProcessView() + + assert.Contains(t, out, "AccountsDB updated") + assert.Contains(t, out, "building AccountsDB") +} + +func TestRenderProcessView_WarnsWhenDiskUsageIsHigh(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + disks: []diskUsage{ + {label: "accounts", path: "/mnt/accounts", used: 417, total: 469, pct: 89}, + }, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } + + out := m.renderProcessView() + + assert.Contains(t, out, "Disk watch") + assert.Contains(t, out, "accounts") + assert.Contains(t, out, "89% used") + assert.Contains(t, out, "52G free") + assert.Contains(t, out, "stop safely") +} + +func TestRenderProcessView_ShowsRPCServerExposure(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + cfg: &configData{ + cluster: "mainnet-beta", + rpcPort: "8899", + blockSource: "rpc", + }, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusStopped, + }}, + } + + // Idle screen stays clean; the firewall caveat lives on the Confirm-run card. + assert.NotContains(t, m.renderProcessView(), "Run plan") + confirm := m.renderStartFlow() + assert.Contains(t, confirm, "all interfaces") + assert.Contains(t, confirm, ":8899") + assert.Contains(t, confirm, "firewall") +} + +// Crashed surfaces a clear warning plus the Doctor next step. +func TestRenderProcessView_CrashedSurfacesWarning(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusCrashed, + LastShutdownReason: "session crashed (no shutdown recorded after start)", + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Crashed", "should surface Crashed badge") + assert.Contains(t, out, "stopped unexpectedly", "should describe what happened in plain English") + assert.Contains(t, out, "Doctor", "should point at the Doctor view") +} + +func TestRenderProcessView_CrashedKeepsActionsBeforeProgressHistory(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + runFocused: true, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusCrashed, + LastShutdownReason: "snapshot bootstrap stopped", + }}, + progress: []progressEvent{ + { + TS: time.Now().Add(-30 * time.Second), + Phase: "error", + Status: "error", + Message: "failed to build AccountsDB from snapshot: context canceled", + }, + }, + } + + out := m.renderProcessView() + actionIdx := strings.Index(out, "Choose action") + errorIdx := strings.Index(out, "failed to build AccountsDB") + assert.NotEqual(t, -1, actionIdx, "action panel should render") + assert.NotEqual(t, -1, errorIdx, "old progress error should still render") + assert.Less(t, actionIdx, errorIdx, "restart controls must stay visible before stale progress history") + assert.Contains(t, out, "▶ Start Mithril") +} + +func TestRenderProcessView_RPCStallExplainsRateLimitRisk(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusCrashed, + LastShutdownReason: state.ShutdownReasonStall, + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "RPC catchup") + assert.Contains(t, out, "private/dedicated RPC") +} + +// A concurrent stop by another dashboard is surfaced to the operator. +func TestRenderProcessView_StopInProgressNoted(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + StopInProgressBy: 67890, + StopInProgressAt: time.Now().Add(-30 * time.Second), + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Another dashboard is shutting this node down") + assert.Contains(t, out, "67890", "should mention the dashboard PID in the sub-line") +} + +func TestRenderProcessView_StopInProgressByThisDashboard(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + StopInProgressBy: os.Getpid(), + StopInProgressAt: time.Now().Add(-30 * time.Second), + }}, + } + out := m.renderProcessView() + assert.Contains(t, out, "This dashboard is shutting this node down") +} + +// Process status badge appears on the Overview screen. +func TestRenderOverview_ShowsProcessBadge(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenOverview, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusRunning, + Pid: 12345, + }}, + } + out := m.renderOverview() + assert.Contains(t, out, "Running", "overview should surface the Running badge") + // PID is reserved for the Process view, not the Overview banner. + assert.NotContains(t, out, "12345", "overview banner should NOT show PID") +} + +func TestRenderOverview_CompletedUsesCompletionBadge(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenOverview, + proc: procState{detection: &procctl.Detection{ + Status: procctl.StatusStopped, + LastShutdownReason: state.ShutdownReasonCompleted, + LastCleanExit: true, + }}, + } + out := m.renderOverview() + assert.Contains(t, out, "Completed") + assert.NotContains(t, out, "Stopped") +} + +// Banner shows no badge during the loading window (no detection yet). +func TestRenderOverview_NoBadgeBeforeFirstFetch(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenOverview, + // proc.detection is nil + } + out := m.renderOverview() + assert.NotContains(t, out, "Running") + assert.NotContains(t, out, "Stopped") + assert.NotContains(t, out, "Crashed") +} + +func TestRenderOverview_FirstRunPointsToProcessStart(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenOverview, + cfg: &configData{}, + configFile: "/tmp/config.toml", + } + out := m.renderOverview() + assert.Contains(t, out, "Run Node") + assert.Contains(t, out, "choose Start") + assert.NotContains(t, out, "mithril run --config") +} + +func TestManagedLightbringer_CleanStoppedScreenAndConfirmCaveats(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + cfg: &configData{ + cluster: "mainnet-beta", + bootstrapMode: "accountsdb", + blockSource: "lightbringer", + lbEnabled: true, + lbGrpcAddr: "127.0.0.1:3551", + lbGossipPort: "55000", + lbPortRangeStart: "55001", + lbPortRangeEnd: "55100", + accountsPath: "/home/ubuntu/mithril-tests/accounts", + rpcEndpoints: []string{"https://api.mainnet-beta.solana.com"}, + }, + proc: procState{detection: &procctl.Detection{Status: procctl.StatusStopped}}, + } + // Idle screen stays clean (no run plan) but still offers the action. + stopped := m.renderProcessView() + assert.NotContains(t, stopped, "Run plan") + assert.Contains(t, stopped, "Start Mithril + Lightbringer") + + // Confirm-run card carries run mode, inbound UDP ports, and the RPC rate-limit warning. + confirm := m.renderStartFlow() + flat := strings.Join(strings.Fields(confirm), " ") + assert.Contains(t, confirm, "Confirm run") + assert.Contains(t, confirm, "Mithril + Lightbringer") + assert.Contains(t, flat, "55000") + assert.Contains(t, flat, "55001-55100") + assert.Contains(t, flat, "rate-limit") + assert.NotContains(t, confirm, "Mithril only") +} + +func TestRenderProcessView_RPCConfigShowsMithrilOnlyStart(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + width: 80, + cfg: &configData{ + cluster: "mainnet-beta", + bootstrapMode: "accountsdb", + blockSource: "rpc", + }, + proc: procState{detection: &procctl.Detection{Status: procctl.StatusStopped}}, + } + out := m.renderProcessView() + assert.Contains(t, out, "Start Mithril") + assert.NotContains(t, out, "Start Mithril + Lightbringer") + assert.Contains(t, out, "Mithril alone") +} + +func TestStartFlowCardsWarnBeforeFreshRebuild(t *testing.T) { + m := model{ + cfg: &configData{ + bootstrapMode: "new-snapshot", + blockSource: "rpc", + }, + startFlow: startFlowState{active: true}, + } + + cards := []startFlowCard{*m.startFlowBootstrapCard()} + assert.Equal(t, "Fresh rebuild", cards[0].title) + assert.Contains(t, cards[0].detail, "fresh AccountsDB") + assert.True(t, cards[0].warn) + + out := m.renderStartFlow() + assert.Contains(t, out, "Fresh rebuild") + assert.Contains(t, out, "Build a fresh AccountsDB from snapshot.") +} + +func TestStartFlowCardsTreatEmptyAccountsRootAsFreshRebuild(t *testing.T) { + m := model{ + cfg: &configData{ + bootstrapMode: "new-snapshot", + blockSource: "rpc", + accountsPath: "/var/lib/mithril/accounts", + }, + accounts: accountsActivity{ + Path: "/var/lib/mithril/accounts", + Name: "accounts", + ModTime: time.Now(), + }, + startFlow: startFlowState{active: true}, + } + + cards := []startFlowCard{*m.startFlowBootstrapCard()} + assert.Equal(t, "Fresh rebuild", cards[0].title) + assert.Contains(t, cards[0].detail, "fresh AccountsDB") + assert.NotContains(t, cards[0].detail, "replaced") +} + +func TestStartFlowCardsWarnWhenFreshRebuildReplacesAccountsArtifact(t *testing.T) { + m := model{ + cfg: &configData{ + bootstrapMode: "new-snapshot", + blockSource: "rpc", + accountsPath: "/var/lib/mithril/accounts", + }, + accounts: accountsActivity{ + Path: "/var/lib/mithril/accounts/mithril_state.json", + Name: "mithril_state.json", + ModTime: time.Now(), + }, + startFlow: startFlowState{active: true}, + } + + cards := []startFlowCard{*m.startFlowBootstrapCard()} + assert.Equal(t, "Fresh rebuild", cards[0].title) + assert.Contains(t, cards[0].detail, "replaced") +} + +func TestStartFlowCardsWarnWhenFreshRebuildReplacesExistingState(t *testing.T) { + m := model{ + cfg: &configData{ + bootstrapMode: "new-snapshot", + blockSource: "rpc", + }, + state: &nodeState{LastSlot: 42}, + startFlow: startFlowState{active: true}, + } + + cards := []startFlowCard{*m.startFlowBootstrapCard()} + assert.Equal(t, "Fresh rebuild", cards[0].title) + assert.Contains(t, cards[0].detail, "replaced") +} + +func TestStartFlowCardsShowAccountsDBResumeMode(t *testing.T) { + m := model{ + cfg: &configData{ + bootstrapMode: "accountsdb", + blockSource: "rpc", + }, + } + + cards := []startFlowCard{*m.startFlowBootstrapCard()} + assert.Equal(t, "Use existing AccountsDB", cards[0].title) + assert.Contains(t, cards[0].detail, "fails fast") + assert.False(t, cards[0].warn) +} + +func TestRenderProcessView_DivergenceCrashGuidesRebuildNotRetry(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + width: 100, + cfg: &configData{cluster: "devnet", blockSource: "rpc", bootstrapMode: "auto"}, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusCrashed}, + startFailStderr: strings.Join([]string{ + "ERROR: DIVERGENCE in slot 468318825", + "panic: tx abc pre-balance divergence", + }, "\n"), + }, + } + + out := m.renderProcessView() + + assert.Contains(t, out, "Rebuild fresh safely") + assert.Contains(t, out, "do not retry this AccountsDB") + assert.Contains(t, out, "rebuild from a fresh snapshot") + assert.NotContains(t, out, "Start again when the config looks safe") +} + +func TestRunActions_DivergenceCrashDoesNotOfferStart(t *testing.T) { + m := model{ + cfg: &configData{blockSource: "rpc"}, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusCrashed}, + startFailStderr: "panic: return value divergence", + }, + } + + actions := m.runActions() + labels := make([]string, 0, len(actions)) + for _, action := range actions { + labels = append(labels, action.label) + assert.NotEqual(t, runActionStart, action.id) + } + + assert.Contains(t, labels, "Rebuild fresh safely") + assert.Contains(t, labels, "Set new snapshot") +} + +func TestRunActions_DivergenceCrashUsesLogTailAfterDashboardReopen(t *testing.T) { + m := model{ + cfg: &configData{blockSource: "rpc"}, + proc: procState{ + detection: &procctl.Detection{Status: procctl.StatusCrashed}, + }, + mithrilLines: []string{ + "ERROR: DIVERGENCE in slot 468318825", + "panic: tx abc pre-balance divergence", + }, + } + + actions := m.runActions() + for _, action := range actions { + assert.NotEqual(t, runActionStart, action.id) + } + + out := m.renderProcessView() + assert.Contains(t, out, "do not retry this AccountsDB") + assert.Contains(t, out, "Rebuild fresh safely") +} + +// Unparseable PID file shows a non-misleading error path. +func TestRenderProcessView_FetchError(t *testing.T) { + m := model{ + hasConfig: true, + screen: screenProcess, + proc: procState{ + fetchErr: "parse pid file /tmp/mithril.pid: unexpected end of JSON", + }, + } + out := m.renderProcessView() + assert.Contains(t, out, "Cannot determine status") + assert.Contains(t, out, "JSON", "should preserve the underlying error detail") +} + +// stoppedModelWithActivity builds a Stopped Run-Node model with a clean activity +// feed (ending in shutdown) so the two-column body renders feed + last-exit detail. +func stoppedModelWithActivity(width int) model { + return model{ + hasConfig: true, + screen: screenProcess, + runFocused: true, + width: width, + cfg: &configData{cluster: "devnet", blockSource: "rpc", bootstrapMode: "auto", accountsPath: "/data/accounts"}, + proc: procState{ + detection: &procctl.Detection{ + Status: procctl.StatusStopped, + LastCleanExit: true, + LastShutdownReason: "graceful shutdown (Ctrl+C)", + }, + fetchedAt: time.Now().Add(-30 * time.Second), + }, + progress: []progressEvent{ + {Phase: "bootstrap_accounts", Status: "ok", Message: "AccountsDB is ready", Fields: map[string]any{"slot": float64(426085232)}}, + {Phase: "replay", Status: "", Message: "Starting block replay", Fields: map[string]any{"start_slot": float64(426085232)}}, + {Phase: "shutdown", Status: "ok", Message: "Mithril stopped cleanly", TS: time.Now().Add(-25 * time.Minute)}, + }, + } +} + +// visibleWidth returns the terminal display width (ANSI stripped). +func visibleWidth(s string) int { return lipgloss.Width(s) } + +// Wide pane splits into aligned columns: actions left, activity/status right. +func TestRenderProcessView_TwoColumnLayout(t *testing.T) { + m := stoppedModelWithActivity(110) + out := m.renderProcessView() + + assert.Contains(t, out, "Choose action", "left column header") + assert.Contains(t, out, "Recent activity", "right column activity header") + assert.Contains(t, out, "Last exit", "right column status header") + assert.Contains(t, out, "│", "columns separated by a vertical divider") + assert.Contains(t, out, "Mithril stopped cleanly") +} + +// Resize guard: no rendered line exceeds the pane budget at any width. +func TestRenderProcessView_NoOverflowAcrossWidths(t *testing.T) { + for _, w := range []int{40, 55, 71, 72, 80, 100, 120, 160, 220} { + m := stoppedModelWithActivity(w) + budget := m.runPanelContentWidth() + 2 // +2 for the two-space inset + out := m.renderProcessView() + for i, line := range strings.Split(out, "\n") { + if got := visibleWidth(line); got > budget { + t.Errorf("width=%d line %d overflows: %d > %d budget\n%q", w, i, got, budget, line) + } + } + } +} + +// Every two-column row places the divider on the same visible column. +func TestRenderProcessView_DividerStaysAligned(t *testing.T) { + m := stoppedModelWithActivity(110) + out := m.renderProcessView() + + width := -1 + rows := 0 + for _, line := range strings.Split(out, "\n") { + if !strings.Contains(line, "│") { + continue + } + rows++ + w := visibleWidth(line) + if width == -1 { + width = w + continue + } + assert.Equalf(t, width, w, "divider rows must share one visible width (got %q)", line) + } + assert.Greater(t, rows, 3, "expected several divided rows") +} + +// Below the two-column threshold the body reflows to a single stacked column. +func TestRenderProcessView_NarrowReflowsToStacked(t *testing.T) { + wide := stoppedModelWithActivity(110).renderProcessView() + narrow := stoppedModelWithActivity(50).renderProcessView() + + assert.Contains(t, wide, "│", "wide pane uses a divided two-column body") + assert.NotContains(t, narrow, "│", "narrow pane reflows to a stacked single column") + assert.Contains(t, narrow, "Choose action", "stacked layout still lists actions") +} + +// renderProcessStatusBadge — the colored badge alone. + +// Each Status yields a visibly distinct badge. +func TestRenderProcessStatusBadge_DistinguishesStates(t *testing.T) { + running := renderProcessStatusBadge(procctl.StatusRunning) + stopped := renderProcessStatusBadge(procctl.StatusStopped) + crashed := renderProcessStatusBadge(procctl.StatusCrashed) + + assert.Contains(t, running, "Running") + assert.Contains(t, stopped, "Stopped") + assert.Contains(t, crashed, "Crashed") + + assert.NotEqual(t, running, stopped) + assert.NotEqual(t, running, crashed) + assert.NotEqual(t, stopped, crashed) +} + +// selectCurrent — the menu→screen dispatcher. + +// Picking "Process" sets the screen and returns a fetch cmd for immediate refresh. +func TestSelectCurrent_ProcessNavigatesAndRefreshes(t *testing.T) { + m := &model{ + hasConfig: true, + items: []menuItem{ + {label: "Overview", value: "overview"}, + {label: "Run Node", value: "process"}, + }, + cursor: 1, // pointing at Run Node + } + cmd := m.selectCurrent() + assert.Equal(t, screenProcess, m.screen) + assert.True(t, m.runFocused, "Run Node should move focus to the action panel") + assert.NotNil(t, cmd, "navigating to Process should kick a fetchProcessCmd") +} + +// humanizeAge — utility shared by the view and the status bar. + +// Very-recent renders as "a moment ago", not "0s ago". +func TestHumanizeAge_RecentTimesAreFriendly(t *testing.T) { + out := humanizeAge(time.Now()) + assert.Equal(t, "a moment ago", out) +} + +// Unit cascade (s→m→h→d): asserts exact unit token and magnitude so a +// cascade bug (e.g. "120s ago" instead of "2m ago") fails loudly. +func TestHumanizeAge_ScalesByUnit(t *testing.T) { + cases := []struct { + ago time.Duration + wantUnit string // unit suffix: "s ago", "m ago", "h ago", "d ago" + wantValue int // exact numeric value + }{ + {10 * time.Second, "s ago", 10}, + {2 * time.Minute, "m ago", 2}, + {3 * time.Hour, "h ago", 3}, + {49 * time.Hour, "d ago", 2}, + } + for _, c := range cases { + t.Run(c.wantUnit, func(t *testing.T) { + got := humanizeAge(time.Now().Add(-c.ago)) + assert.True(t, strings.HasSuffix(got, c.wantUnit), + "got %q, want suffix %q", got, c.wantUnit) + // ±1 tolerance for rounding as time.Since() advances mid-test. + expectedTokens := []string{ + fmt.Sprintf("%d%s", c.wantValue-1, c.wantUnit), + fmt.Sprintf("%d%s", c.wantValue, c.wantUnit), + fmt.Sprintf("%d%s", c.wantValue+1, c.wantUnit), + } + matched := false + for _, tok := range expectedTokens { + if strings.Contains(got, tok) { + matched = true + break + } + } + assert.True(t, matched, "got %q, want one of %v", got, expectedTokens) + }) + } +} diff --git a/cmd/mithril/dashboardcmd/ring_buffer.go b/cmd/mithril/dashboardcmd/ring_buffer.go new file mode 100644 index 000000000..36410e4b6 --- /dev/null +++ b/cmd/mithril/dashboardcmd/ring_buffer.go @@ -0,0 +1,34 @@ +package dashboardcmd + +import "sync" + +// ringBuffer keeps the tail of a process's stderr (oldest bytes dropped past +// cap) so a fast-failing start can surface its real error. Concurrent-safe. +type ringBuffer struct { + mu sync.Mutex + buf []byte + cap int +} + +func newRingBuffer(capacity int) *ringBuffer { + return &ringBuffer{cap: capacity} +} + +// Write appends p, evicting from the head to stay within cap. Always reports +// len(p) (never short-writes) for io.MultiWriter. +func (r *ringBuffer) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.buf = append(r.buf, p...) + if over := len(r.buf) - r.cap; over > 0 { + r.buf = r.buf[over:] + } + return len(p), nil +} + +// String returns a snapshot of the current buffer content. +func (r *ringBuffer) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return string(r.buf) +} diff --git a/cmd/mithril/dashboardcmd/views.go b/cmd/mithril/dashboardcmd/views.go index 6e58347db..f3f1b7352 100644 --- a/cmd/mithril/dashboardcmd/views.go +++ b/cmd/mithril/dashboardcmd/views.go @@ -3,8 +3,12 @@ package dashboardcmd import ( "fmt" "os" + "strconv" "strings" + "time" + "unicode/utf8" + "github.com/Overclock-Validator/mithril/pkg/config" "github.com/Overclock-Validator/mithril/pkg/tui" "github.com/charmbracelet/lipgloss" ) @@ -17,6 +21,8 @@ func (m model) renderRightPane() string { switch m.screen { case screenOverview: return m.renderOverview() + case screenProcess: + return m.renderProcessView() case screenConfig: return m.renderConfigView() case screenEdit: @@ -59,6 +65,11 @@ func (m model) renderOverview() string { value := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) header := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + // Running-state badge, shown before the health summary. + if m.proc.detection != nil { + b.WriteString(" " + renderProcessHeadline(*m.proc.detection, m.proc.inFlightOp) + "\n\n") + } + // Health summary passed := 0 total := len(m.checks) @@ -92,7 +103,7 @@ func (m model) renderOverview() string { dot = pass.Render("●") status = pass.Render("up") } - b.WriteString(" " + dot + " " + value.Render(fmt.Sprintf("%-14s", svc.name)) + label.Render(fmt.Sprintf("%-20s", svc.addr)) + status + "\n") + b.WriteString(" " + dot + " " + value.Render(fmt.Sprintf("%-14s", svc.name)) + label.Render(fmt.Sprintf("%-20s", displayServiceAddr(svc.addr))) + status + "\n") } b.WriteString("\n") } @@ -100,36 +111,41 @@ func (m model) renderOverview() string { // Node state if m.state != nil && m.state.LastSlot > 0 { b.WriteString(header.Render("Node State") + "\n\n") - b.WriteString(label.Render(" Slot ") + value.Render(formatNumber(m.state.LastSlot)) + "\n") - b.WriteString(label.Render(" Epoch ") + value.Render(fmt.Sprintf("%d", m.state.LastEpoch)) + "\n") - if m.state.LastBankhash != "" { - short := m.state.LastBankhash - if len(short) > 12 { - short = short[:12] + "..." - } - b.WriteString(label.Render(" Bankhash ") + value.Render(short) + "\n") + // Prefer live replay slot; state file slot is frozen between checkpoints. + slotVal, epochVal := m.state.LastSlot, m.state.LastEpoch + liveTag := "" + if s, e, ok := m.liveNodeSlot(); ok { + slotVal, epochVal = s, e + liveTag = lipgloss.NewStyle().Foreground(tui.ColorSuccess).Render(" ● live") } + b.WriteString(label.Render(" Slot ") + value.Render(formatNumber(slotVal)) + liveTag + "\n") + b.WriteString(label.Render(" Epoch ") + value.Render(fmt.Sprintf("%d", epochVal)) + "\n") if m.state.SnapshotSlot > 0 { b.WriteString(label.Render(" Snapshot ") + value.Render(formatNumber(m.state.SnapshotSlot)) + "\n") } - if m.state.LastShutdownReason != "" { + // Only show last shutdown when not running; otherwise it's a prior run's exit. + if m.state.LastShutdownReason != "" && !m.isNodeRunning() { reason := value.Render(m.state.LastShutdownReason) if m.state.LastShutdownAt != "" { - reason += label.Render(" at ") + value.Render(m.state.LastShutdownAt) + when := m.state.LastShutdownAt + if t, perr := time.Parse(time.RFC3339Nano, m.state.LastShutdownAt); perr == nil { + when = humanizeAge(t) + } + reason += label.Render(" ") + value.Render(when) } b.WriteString(label.Render(" Shutdown ") + reason + "\n") } if m.state.Stage != "" { b.WriteString(label.Render(" Stage ") + value.Render(m.state.Stage) + "\n") } - if m.state.LastWriterVersion != "" { - ver := value.Render(m.state.LastWriterVersion) - if m.state.LastWriterCommit != "" { - short := m.state.LastWriterCommit - if len(short) > 8 { - short = short[:8] + // Build version that last wrote this state; skipped for dev/unknown builds. + if v := m.state.LastWriterVersion; v != "" && v != "dev" && v != "unknown" { + ver := value.Render(v) + if c := m.state.LastWriterCommit; c != "" && c != "unknown" { + if len(c) > 8 { + c = c[:8] } - ver += label.Render(" (") + value.Render(short) + label.Render(")") + ver += label.Render(" (") + value.Render(c) + label.Render(")") } b.WriteString(label.Render(" Writer ") + ver + "\n") } @@ -143,8 +159,7 @@ func (m model) renderOverview() string { b.WriteString(label.Render(" Node has not been started yet.") + "\n\n") b.WriteString(label.Render(" 1. Review your config ") + cmd.Render("← Config") + "\n") b.WriteString(label.Render(" 2. Run health checks ") + cmd.Render("← Doctor") + "\n") - b.WriteString(label.Render(" 3. Start the node:") + "\n") - b.WriteString(cmd.Render(" $ mithril run --config "+m.configFile) + "\n") + b.WriteString(label.Render(" 3. Start the node ") + cmd.Render("← Run Node, choose Start, Enter") + "\n") } return b.String() @@ -172,8 +187,8 @@ func (m model) renderConfigView() string { if trimmed == "" || strings.HasPrefix(trimmed, "#") { continue } - if strings.HasPrefix(trimmed, "[") && !strings.HasPrefix(trimmed, "[[") { - sections = append(sections, configSection{name: strings.Trim(trimmed, "[] ")}) + if sectionName, ok := tomlSectionName(trimmed); ok { + sections = append(sections, configSection{name: sectionName}) continue } if len(sections) > 0 { @@ -193,26 +208,151 @@ func (m model) renderConfigView() string { valStyle := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) hintStyle := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) - // Count total lines needed for single-column display - totalLines := 0 + // Pack sections into columns that fit the visible height so the whole config + // shows without pgdn (-19 leaves room for the blank/hint/trailing rows). + availHeight := m.height - 19 + if availHeight < 10 { + availHeight = 10 + } + paneWidth := m.rightPaneContentWidth() + return renderConfigColumns(sections, paneWidth, availHeight, sectionStyle, keyStyle, valStyle, hintStyle) +} + +// stackedHeight is a column's line count: header + kvs per section, plus a +// blank line between sections. +func stackedHeight(secs []configSection) int { + h := 0 + for i, s := range secs { + if i > 0 { + h++ + } + h += 1 + len(s.keys) + } + return h +} + +// packConfigColumns greedily distributes sections into n balanced columns, +// keeping each section intact (never split across a column boundary). +func packConfigColumns(sections []configSection, n int) [][]configSection { + if n <= 1 { + return [][]configSection{sections} + } + target := (stackedHeight(sections) + n - 1) / n + var cols [][]configSection + var cur []configSection + curLines := 0 + for _, s := range sections { + sl := 1 + len(s.keys) + sep := 0 + if curLines > 0 { + sep = 1 + } + if curLines > 0 && curLines+sep+sl > target && len(cols) < n-1 { + cols = append(cols, cur) + cur, curLines, sep = nil, 0, 0 + } + cur = append(cur, s) + curLines += sep + sl + } + if len(cur) > 0 { + cols = append(cols, cur) + } + return cols +} + +// fillColumns packs sections top-to-bottom, starting a new column only when the +// current one would exceed maxHeight (fills left-first, doesn't balance). +func fillColumns(sections []configSection, maxHeight int) [][]configSection { + var cols [][]configSection + var cur []configSection + curLines := 0 for _, s := range sections { - totalLines += 1 + len(s.keys) // header + kvs + sl := 1 + len(s.keys) + sep := 0 + if curLines > 0 { + sep = 1 + } + if curLines > 0 && curLines+sep+sl > maxHeight { + cols = append(cols, cur) + cur, curLines, sep = nil, 0, 0 + } + cur = append(cur, s) + curLines += sep + sl } - totalLines += len(sections) - 1 // blank lines between sections + if len(cur) > 0 { + cols = append(cols, cur) + } + return cols +} - // Estimate available height in right pane - availHeight := m.height - 18 - if availHeight < 10 { - availHeight = 10 +// renderConfigColumns lays sections into as many side-by-side columns as fit +// paneWidth (up to availHeight); never splits a section, falls back to one column. +func renderConfigColumns(sections []configSection, paneWidth, availHeight int, sectionStyle, keyStyle, valStyle, hintStyle lipgloss.Style) string { + const gap = 3 + // Min column width — small enough that 2-3 columns engage at normal widths. + const minCol = 24 + + maxCols := (paneWidth + gap) / (minCol + gap) + if maxCols > len(sections) { + maxCols = len(sections) + } + if maxCols < 1 { + maxCols = 1 + } + // Fill columns top-to-bottom up to availHeight before spilling right. + cols := fillColumns(sections, availHeight) + if len(cols) > maxCols { + // Too tall to fit the width at this height; pack into the columns the + // width allows (the only case that can still scroll). + cols = packConfigColumns(sections, maxCols) + } + n := len(cols) + if n < 1 { + n = 1 + } + colWidth := (paneWidth - gap*(n-1)) / n + if colWidth < 1 { + colWidth = paneWidth } - // If it fits in one column, render single column - if totalLines <= availHeight { - return m.renderConfigSingleColumn(sections, sectionStyle, keyStyle, valStyle, hintStyle) + rendered := make([][]string, n) + colW := make([]int, n) // each column's natural width (its longest line), so columns pack flush + rows := 0 + for i := range cols { + rendered[i] = renderConfigColumn(cols[i], colWidth, sectionStyle, keyStyle, valStyle) + for _, ln := range rendered[i] { + if w := lipgloss.Width(ln); w > colW[i] { + colW[i] = w + } + } + if len(rendered[i]) > rows { + rows = len(rendered[i]) + } } - // Otherwise split into two columns side by side - return m.renderConfigTwoColumns(sections, sectionStyle, keyStyle, valStyle, hintStyle) + gapStr := strings.Repeat(" ", gap) + var b strings.Builder + for r := 0; r < rows; r++ { + var row strings.Builder + for i := 0; i < n; i++ { + cell := "" + if r < len(rendered[i]) { + cell = rendered[i][r] // already truncated to <= colWidth, never exceeds the pane + } + if i < n-1 { // pad to THIS column's natural width so the next column sits flush, no wide gap + if pad := colW[i] - lipgloss.Width(cell); pad > 0 { + cell += strings.Repeat(" ", pad) + } + cell += gapStr + } + row.WriteString(cell) + } + b.WriteString(strings.TrimRight(row.String(), " ") + "\n") + } + b.WriteString("\n") + ks := lipgloss.NewStyle().Foreground(tui.MithrilTeal) + b.WriteString(" " + ks.Render("e") + hintStyle.Render(" edit") + " " + ks.Render("r") + hintStyle.Render(" refresh") + "\n") + return b.String() } // renderConfigColumn renders sections as lines for a single column. @@ -246,17 +386,20 @@ func renderConfigColumn(secs []configSection, colWidth int, sectionStyle, keySty lines = append(lines, sectionStyle.Render(s.name)) for j := range s.keys { v := s.vals[j] + v = displayConfigValue(s.name, s.keys[j], v) // Mask sensitive values (tokens, secrets, passwords) k := strings.ToLower(s.keys[j]) if strings.Contains(k, "token") || strings.Contains(k, "secret") || strings.Contains(k, "password") { - if len(v) > 4 { - v = v[:2] + strings.Repeat("*", len(v)-4) + v[len(v)-2:] - } else if len(v) > 0 { + // Rune-safe: byte slicing could split a multibyte rune in a + // pasted token and emit invalid UTF-8. + if rv := []rune(v); len(rv) > 4 { + v = string(rv[:2]) + strings.Repeat("*", len(rv)-4) + string(rv[len(rv)-2:]) + } else if len(rv) > 0 { v = "****" } } - if len(v) > maxVal { - v = v[:maxVal-3] + "..." + if rv := []rune(v); len(rv) > maxVal { + v = string(rv[:maxVal-1]) + "…" // rune-safe, single-char ellipsis } lines = append(lines, " "+keyStyle.Render(fmt.Sprintf("%-*s ", keyPad, s.keys[j]))+valStyle.Render(v)) } @@ -264,76 +407,44 @@ func renderConfigColumn(secs []configSection, colWidth int, sectionStyle, keySty return lines } -func (m model) renderConfigSingleColumn(sections []configSection, sectionStyle, keyStyle, valStyle, hintStyle lipgloss.Style) string { - rightPaneWidth := (m.width - 3) * 78 / 100 - lines := renderConfigColumn(sections, rightPaneWidth, sectionStyle, keyStyle, valStyle) - - var b strings.Builder - for _, l := range lines { - b.WriteString(l + "\n") - } - b.WriteString("\n") - ks := lipgloss.NewStyle().Foreground(tui.MithrilTeal) - b.WriteString(" " + ks.Render("e") + hintStyle.Render(" edit") + " " + ks.Render("r") + hintStyle.Render(" refresh") + "\n") - return b.String() -} - -func (m model) renderConfigTwoColumns(sections []configSection, sectionStyle, keyStyle, valStyle, hintStyle lipgloss.Style) string { - // Split sections into two groups by total line count (balanced) - totalLines := 0 - for _, s := range sections { - totalLines += 2 + len(s.keys) // header + kvs + spacing +func displayConfigValue(section, key, value string) string { + if !shouldRedactFieldValue(section, key) { + return value } - midpoint := totalLines / 2 - - lineCount := 0 - splitIdx := len(sections) - for i, s := range sections { - sLines := 2 + len(s.keys) - if lineCount+sLines > midpoint && lineCount > 0 { - splitIdx = i - break - } - lineCount += sLines + fullKey := strings.ToLower(section + "." + key) + switch { + case fullKey == "network.rpc": + return redactEndpointListForDisplay(value) + case strings.Contains(fullKey, "endpoint"): + return redactEndpointListForDisplay(value) + case strings.Contains(fullKey, "rpc") && strings.Contains(value, "://"): + return redactEndpointListForDisplay(value) + default: + return value } +} - // Column sizing: right pane width → two columns with clean gap - rightPaneWidth := (m.width - 3) * 78 / 100 - colGap := 4 - colWidth := (rightPaneWidth - colGap) / 2 - - leftLines := renderConfigColumn(sections[:splitIdx], colWidth, sectionStyle, keyStyle, valStyle) - rightLines := renderConfigColumn(sections[splitIdx:], colWidth, sectionStyle, keyStyle, valStyle) - - maxRows := len(leftLines) - if len(rightLines) > maxRows { - maxRows = len(rightLines) - } +func displayServiceAddr(addr string) string { + return config.RedactSecretsInText(config.RedactEndpointForDisplay(addr)) +} - gap := strings.Repeat(" ", colGap) - truncStyle := lipgloss.NewStyle().MaxWidth(colWidth) +func shouldRedactFieldValue(section, key string) bool { + fullKey := strings.ToLower(section + "." + key) + return fullKey == "network.rpc" || + strings.Contains(fullKey, "endpoint") || + strings.Contains(fullKey, "rpc") +} - var b strings.Builder - for i := 0; i < maxRows; i++ { - left := "" - right := "" - if i < len(leftLines) { - left = truncStyle.Render(leftLines[i]) - } - if i < len(rightLines) { - right = rightLines[i] - } - leftPad := colWidth - lipgloss.Width(left) - if leftPad > 0 { - left += strings.Repeat(" ", leftPad) +func redactEndpointListForDisplay(value string) string { + parts := strings.Split(value, ",") + for i, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed == "" { + continue } - b.WriteString(left + gap + right + "\n") + parts[i] = strings.Replace(part, trimmed, config.RedactEndpointForDisplay(trimmed), 1) } - - b.WriteString("\n") - ks := lipgloss.NewStyle().Foreground(tui.MithrilTeal) - b.WriteString(" " + ks.Render("e") + hintStyle.Render(" edit") + " " + ks.Render("r") + hintStyle.Render(" refresh") + "\n") - return b.String() + return strings.Join(parts, ",") } // stripInlineComment removes the inline comment from a TOML value. @@ -392,6 +503,70 @@ func (m model) renderEditView() string { return m.renderEditList() } +// fieldHelp returns a one-line plain-language explanation of a config field, +// shown while editing and under the highlighted list row. +func fieldHelp(section, key string) string { + switch section + "." + key { + case "network.cluster": + return "Which Solana network to follow: mainnet-beta, devnet, or testnet." + case "network.rpc": + return "URL of a Solana RPC provider used to fetch blocks. Paste your provider's URL." + case "storage.accounts": + return "Folder for the account database (large — put it on your fastest disk)." + case "storage.snapshots": + return "Folder for downloaded snapshot files used to bootstrap a fresh start." + case "storage.shredstore": + return "Folder where raw block data (shreds) is stored." + case "storage.logs": + return "Folder where log files are written." + case "block.source": + return "Where blocks come from: 'rpc' (an RPC provider) or 'lightbringer' (peer-to-peer sidecar)." + case "block.turbine_bind_addr": + return "Local UDP address to receive blocks directly from the network (turbine mode)." + case "turbine.gossip_entrypoint": + return "host:port of a known Solana node used to join the network (turbine mode)." + case "turbine.gossip_bind_addr": + return "Local UDP address for network gossip traffic (turbine mode)." + case "turbine.advertised_ip": + return "Your machine's public IP so peers can reach you (turbine mode)." + case "turbine.shred_version": + return "Network data-format version. Leave 0 to auto-detect." + case "block.lightbringer_endpoint": + return "host:port of an already-running external Lightbringer sidecar." + case "block.max_rps": + return "Max requests per second to the RPC provider (lower it to avoid rate limits)." + case "block.max_inflight": + return "How many blocks to fetch in parallel." + case "lightbringer.enabled": + return "Let Mithril start and manage a Lightbringer sidecar for you." + case "lightbringer.binary_path": + return "Path to the lightbringer program file." + case "lightbringer.config_dir": + return "Folder where Mithril writes Lightbringer's config and data." + case "lightbringer.gossip_entrypoint": + return "host:port of a Solana node for Lightbringer to join the network." + case "lightbringer.gossip_port": + return "Inbound UDP port for Lightbringer gossip — open this in your firewall." + case "lightbringer.port_range_start", "lightbringer.port_range_end": + return "Inbound UDP port range for Lightbringer — open this range in your firewall." + case "lightbringer.grpc_addr": + return "Local address where Mithril reads blocks from Lightbringer." + case "lightbringer.rpc_addr": + return "Local address for Lightbringer's HTTP interface." + case "lightbringer.quiet": + return "Hide Lightbringer's detailed logs (less noise)." + case "tuning.txpar": + return "Parallel workers for replaying blocks. Empty = sequential (slower, simplest)." + case "rpc.port": + return "Port for Mithril's own RPC server. It listens on all interfaces — firewall it if public." + case "log.level": + return "How much detail to log: info, debug, warn, or error." + case "bootstrap.mode": + return "How to start: 'auto' reuses local data, or downloads a snapshot if needed." + } + return "" +} + // renderEditList shows all fields in a compact scrollable list. func (m model) renderEditList() string { label := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) @@ -413,12 +588,15 @@ func (m model) renderEditList() string { // Auto-detect indicator for unset txpar displayVal := val + if displayVal != "" { + displayVal = displayConfigValue(f.section, f.key, displayVal) + } isAuto := val == "" && f.section == "tuning" && f.key == "txpar" if isAuto { displayVal = "not set (sequential)" } - if len(displayVal) > 35 { - displayVal = displayVal[:32] + "..." + if rv := []rune(displayVal); len(rv) > 35 { + displayVal = string(rv[:32]) + "..." // rune-safe truncation } if isSelected { @@ -442,28 +620,87 @@ func (m model) renderEditList() string { if maxVisible < 10 { maxVisible = 10 } + scrollStart := 0 if len(lines) > maxVisible { - start := selectedStart - maxVisible/3 - if start < 0 { - start = 0 + scrollStart = selectedStart - maxVisible/3 + if scrollStart < 0 { + scrollStart = 0 } - end := start + maxVisible + end := scrollStart + maxVisible if end > len(lines) { end = len(lines) - start = end - maxVisible - if start < 0 { - start = 0 + scrollStart = end - maxVisible + if scrollStart < 0 { + scrollStart = 0 + } + } + lines = lines[scrollStart:end] + } + // Row of the selected field within the now-visible window. + selectedRow := selectedStart - scrollStart + + // Wide panes pair the highlighted field with a details column; narrow panes + // just show the list. + rightPaneWidth := (m.width - 3) * 78 / 100 + detailW := rightPaneWidth / 3 + if detailW > 32 { + detailW = 32 + } + if rightPaneWidth < 70 || detailW < 20 { + return strings.Join(lines, "\n") + "\n" + } + colGap := 3 + listW := rightPaneWidth - detailW - colGap + + var right []string + if m.editIdx >= 0 && m.editIdx < len(m.editFields) { + f := m.editFields[m.editIdx] + detail := []string{active.Render(f.label), ""} + if h := fieldHelp(f.section, f.key); h != "" { + for _, ln := range wrapRunDetailLines([]string{h}, detailW) { + detail = append(detail, hint.Render(ln)) } } - lines = lines[start:end] + // Align the detail with the selected row, clamped to the visible height. + top := selectedRow + if top+len(detail) > len(lines) { + top = len(lines) - len(detail) + } + if top < 0 { + top = 0 + } + right = make([]string, top) + right = append(right, detail...) } - return strings.Join(lines, "\n") + "\n" + maxRows := len(lines) + if len(right) > maxRows { + maxRows = len(right) + } + trunc := lipgloss.NewStyle().MaxWidth(listW) + gap := strings.Repeat(" ", colGap) + var b strings.Builder + for i := 0; i < maxRows; i++ { + l := "" + if i < len(lines) { + l = trunc.Render(lines[i]) + } + r := "" + if i < len(right) { + r = right[i] + } + if pad := listW - lipgloss.Width(l); pad > 0 { + l += strings.Repeat(" ", pad) + } + b.WriteString(l + gap + r + "\n") + } + return b.String() } // renderEditFocused shows a single field's edit UI in the full right pane. func (m model) renderEditFocused() string { f := m.editFields[m.editIdx] + redactValue := shouldRedactFieldValue(f.section, f.key) titleStyle := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) subtitleStyle := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) valueStyle := lipgloss.NewStyle().Foreground(tui.ColorTextPrimary) @@ -478,6 +715,9 @@ func (m model) renderEditFocused() string { b.WriteString("\n") b.WriteString(" " + titleStyle.Render(f.label) + "\n") b.WriteString(" " + subtitleStyle.Render(f.section+"."+f.key) + "\n") + if h := fieldHelp(f.section, f.key); h != "" { + b.WriteString(" " + hintStyle.Render(h) + "\n") + } b.WriteString("\n") if m.editMode == editMenu { @@ -523,19 +763,27 @@ func (m model) renderEditFocused() string { if isAuto { b.WriteString(" " + subtitleStyle.Render("Current: ") + hintStyle.Render("not set (sequential)") + "\n") } else if currentVal != "" { + if redactValue { + currentVal = displayConfigValue(f.section, f.key, currentVal) + } b.WriteString(" " + subtitleStyle.Render("Current: ") + valueStyle.Render(currentVal) + "\n") } b.WriteString("\n") // ── Input field ── text := m.editValue - if m.editCursor >= 0 && m.editCursor <= len(text) { + if redactValue { + text = displayConfigValue(f.section, f.key, text) + } else if m.editCursor >= 0 && m.editCursor <= len(text) { before := text[:m.editCursor] after := text[m.editCursor:] cur := lipgloss.NewStyle().Background(tui.MithrilTeal).Foreground(lipgloss.Color("#000000")).Render(" ") if m.editCursor < len(text) { - cur = lipgloss.NewStyle().Background(tui.MithrilTeal).Foreground(lipgloss.Color("#000000")).Render(string(after[0])) - after = after[1:] + // Decode a full rune under the cursor, not a single byte, so a + // multibyte character isn't split into mojibake. + r, sz := utf8.DecodeRuneInString(after) + cur = lipgloss.NewStyle().Background(tui.MithrilTeal).Foreground(lipgloss.Color("#000000")).Render(string(r)) + after = after[sz:] } text = before + cur + after } @@ -553,6 +801,9 @@ func (m model) renderEditFocused() string { if isAuto && m.editValue == "" { b.WriteString("\n " + hintStyle.Render("Leave empty for sequential mode (0), or set worker count") + "\n") } + if redactValue { + b.WriteString("\n " + hintStyle.Render("Sensitive URL values are hidden while editing; saving preserves the full value.") + "\n") + } b.WriteString("\n") if m.editErr != "" { @@ -612,76 +863,79 @@ func (m model) renderDoctorView() string { // ── Logs View ─────────────────────────────────────────────────────────── func (m model) renderLogsView() string { + if m.logRawMode || !m.hasLightbringerLogPane() || m.rightPaneContentWidth() < 88 { + return m.renderRawLogsView() + } if len(m.mithrilLines) == 0 && len(m.lbLines) == 0 { mutedStyle := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) cmdStyle := lipgloss.NewStyle().Foreground(tui.ColorTextSecondary) - return mutedStyle.Render(" Logs will appear here after starting the node.") + "\n\n" + - cmdStyle.Render(" $ mithril run --config "+m.configFile) + "\n" + controls := m.renderLogControlsLine() + if controls != "" { + controls += "\n\n" + } + return controls + mutedStyle.Render(" Logs will appear here after starting the node.") + "\n\n" + + cmdStyle.Render(" Open Run Node, choose Start, then press Enter.") + "\n" } titleStyle := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) - mutedStyle := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) hintStyle := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) // Calculate column widths - rightPaneWidth := (m.width - 3) * 78 / 100 + rightPaneWidth := m.rightPaneContentWidth() colGap := 3 colWidth := (rightPaneWidth - colGap) / 2 - - // Wrap long lines within column width so full messages are readable - mLines := wrapLogLines(m.mithrilLines, colWidth) - lLines := wrapLogLines(m.lbLines, colWidth) - if m.logFocused && m.logScroll > 0 { - if m.logPane == logPaneMithril { - if m.logScroll < len(mLines) { - mLines = mLines[m.logScroll:] - } else { - mLines = nil - } - } else { - if m.logScroll < len(lLines) { - lLines = lLines[m.logScroll:] - } else { - lLines = nil - } - } + if colWidth < 24 { + return m.renderRawLogsView() } - // Render log lines with color coding - colorLine := func(line string) string { - trimmed := strings.TrimSpace(line) - if trimmed == "" { - return "" - } - switch { - case strings.Contains(line, " WARN ") || strings.HasPrefix(trimmed, "WARN"): - return lipgloss.NewStyle().Foreground(tui.ColorWarn).Render(line) - case strings.Contains(line, " ERROR ") || strings.HasPrefix(trimmed, "ERROR") || strings.Contains(line, "FATAL"): - return lipgloss.NewStyle().Foreground(tui.ColorError).Render(line) - default: - return mutedStyle.Render(line) - } + // Full-width snapshot-download bar. When shown, drop the raw progress + // samples from the mithril column so they don't duplicate it. + downloadBlock := renderDownloadProgress(m.mithrilLines, rightPaneWidth) + mithrilSrc := m.mithrilLines + if downloadBlock != "" { + mithrilSrc = filterDownloadProgressLines(m.mithrilLines) } - // Build side-by-side output with divider + // Redact before wrapping so a long URL can't split a secret across lines. + mLines := wrapLogLines(redactLogLines(mithrilSrc), colWidth) + lLines := wrapLogLines(redactLogLines(m.lbLines), colWidth) maxRows := len(mLines) if len(lLines) > maxRows { maxRows = len(lLines) } availHeight := m.height - 22 + if downloadBlock != "" { + // Reserve rows for the bar so the log area shrinks instead of overflowing. + availHeight -= strings.Count(downloadBlock, "\n") + } if availHeight < 5 { availHeight = 5 } if maxRows > availHeight { maxRows = availHeight } + mScroll, lScroll := 0, 0 + if m.logFocused { + if m.logPane == logPaneMithril { + mScroll = m.logScroll + } else { + lScroll = m.logScroll + } + } + mLines = visibleLogWindow(mLines, maxRows, mScroll) + lLines = visibleLogWindow(lLines, maxRows, lScroll) divStyle := lipgloss.NewStyle().Foreground(tui.ColorBorder) div := divStyle.Render("│") - truncStyle := lipgloss.NewStyle().MaxWidth(colWidth) // Headers — underline the focused pane title var b strings.Builder + if controls := m.renderLogControlsLine(); controls != "" { + b.WriteString(controls + "\n\n") + } + if downloadBlock != "" { + b.WriteString(downloadBlock + "\n") + } var mTitle, lTitle string mTitleStyle := titleStyle lTitleStyle := titleStyle @@ -691,14 +945,9 @@ func (m model) renderLogsView() string { lTitleStyle = lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true).Underline(true) } mTitle = mTitleStyle.Render("mithril") - lTitle = lTitleStyle.Render("lightbringer") + lTitle = lTitleStyle.Render(m.lightbringerLogTitle()) - b.WriteString(mTitle) - leftPad := colWidth - lipgloss.Width(mTitle) - if leftPad > 0 { - b.WriteString(strings.Repeat(" ", leftPad)) - } - b.WriteString(" " + div + " " + lTitle + "\n") + b.WriteString(padStyledLine(mTitle, colWidth) + " " + div + " " + padStyledLine(lTitle, colWidth) + "\n") // Divider line under headers b.WriteString(divStyle.Render(strings.Repeat("─", colWidth)) + " " + div + " " + divStyle.Render(strings.Repeat("─", colWidth)) + "\n") @@ -713,33 +962,488 @@ func (m model) renderLogsView() string { if i < len(mLines) { if m.logFocused && m.logPane == logPaneMithril { // Active pane: brighter text - left = truncStyle.Render(activeLineStyle.Render(mLines[i])) + left = activeLineStyle.Render(redactLogLine(mLines[i])) } else { - left = truncStyle.Render(colorLine(mLines[i])) + left = colorLogLine(mLines[i]) } } if i < len(lLines) { if m.logFocused && m.logPane == logPaneLightbringer { - right = truncStyle.Render(activeLineStyle.Render(lLines[i])) + right = activeLineStyle.Render(redactLogLine(lLines[i])) } else { - right = truncStyle.Render(colorLine(lLines[i])) + right = colorLogLine(lLines[i]) } } - lPad := colWidth - lipgloss.Width(left) - if lPad > 0 { - left += strings.Repeat(" ", lPad) - } + left = padStyledLine(left, colWidth) + right = padStyledLine(right, colWidth) b.WriteString(left + " " + div + " " + right + "\n") } if m.logFocused { - b.WriteString("\n" + hintStyle.Render(" ↑↓ scroll ←→ switch esc back") + "\n") + b.WriteString("\n" + hintStyle.Render(" ↑↓ scroll ←→ switch t full-width esc menu q quit") + "\n") } return b.String() } +func (m model) renderRawLogsView() string { + return m.renderRawLogsViewWith(m.rightPaneContentWidth(), m.rawLogRows(false)) +} + +func (m model) renderRawLogsViewWith(contentWidth, logRows int) string { + titleStyle := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + mutedStyle := lipgloss.NewStyle().Foreground(tui.ColorTextMuted) + hintStyle := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) + divStyle := lipgloss.NewStyle().Foreground(tui.ColorBorder) + + if contentWidth < 10 { + contentWidth = 10 + } + if logRows < 5 { + logRows = 5 + } + + // Snapshot-download progress bar above the tail; reserve rows for it and + // strip the raw progress samples from the scrolling text. + downloadBlock := renderDownloadProgress(m.mithrilLines, contentWidth) + rawSrc := m.combinedRawLogLines() + if downloadBlock != "" { + rawSrc = filterDownloadProgressLines(rawSrc) + logRows -= strings.Count(downloadBlock, "\n") + if logRows < 5 { + logRows = 5 + } + } + lines := dashboardRawLogLines(redactLogLines(rawSrc), contentWidth) + lines = visibleLogWindow(lines, logRows, m.logScroll) + + var b strings.Builder + subtitle := " Mithril live tail" + if m.hasLightbringerLogPane() { + subtitle = " same run, grouped by source" + } + b.WriteString(titleStyle.Render("terminal logs") + mutedStyle.Render(subtitle) + "\n") + if controls := m.renderLogControlsLine(); controls != "" { + b.WriteString(controls + "\n") + } + if downloadBlock != "" { + b.WriteString(downloadBlock + "\n") + } + b.WriteString(divStyle.Render(strings.Repeat("─", contentWidth)) + "\n") + for _, line := range lines { + b.WriteString(padStyledLine(colorLogLine(line), contentWidth) + "\n") + } + tHint := "t full-width" + if m.logRawMode { + tHint = "t exit full-width" + } + scrollHint := "enter scroll" + if m.logFocused { + scrollHint = "↑↓ scroll" + } + b.WriteString("\n" + hintStyle.Render(" "+scrollHint+" "+tHint+" esc menu q quit") + "\n") + return b.String() +} + +func (m model) renderLogControlsLine() string { + if !m.logsStopShortcutAvailable() { + return "" + } + keyStyle := lipgloss.NewStyle().Foreground(tui.MithrilTeal).Bold(true) + hintStyle := lipgloss.NewStyle().Foreground(tui.ColorTextDisabled) + parts := []string{keyStyle.Render("x") + hintStyle.Render(" stop safely")} + if m.hasLightbringerLogPane() { + viewHint := "terminal view" + if m.logRawMode { + viewHint = "split view" + } + parts = append(parts, keyStyle.Render("t")+hintStyle.Render(" "+viewHint)) + } + return " " + strings.Join(parts, hintStyle.Render(" ")) +} + +func (m model) combinedRawLogLines() []string { + showSources := m.hasLightbringerLogPane() + if showSources { + combined := interleavedSourceLogLines(m.mithrilLines, m.lbLines) + if len(combined) == 0 { + return []string{"(no log lines yet)"} + } + return combined + } + + var combined []string + for _, line := range m.mithrilLines { + if strings.TrimSpace(line) == "" { + continue + } + combined = append(combined, line) + } + if len(combined) == 0 { + return []string{"(no log lines yet)"} + } + return combined +} + +func interleavedSourceLogLines(mithrilLines, lbLines []string) []string { + mLines := nonEmptyLogLines(mithrilLines) + lLines := nonEmptyLogLines(lbLines) + maxLen := len(mLines) + if len(lLines) > maxLen { + maxLen = len(lLines) + } + if maxLen == 0 { + return nil + } + + combined := make([]string, 0, len(mLines)+len(lLines)) + mStart := maxLen - len(mLines) + lStart := maxLen - len(lLines) + for i := 0; i < maxLen; i++ { + if i >= mStart { + combined = append(combined, "[mithril] "+mLines[i-mStart]) + } + if i >= lStart { + combined = append(combined, "[lightbringer] "+lLines[i-lStart]) + } + } + return combined +} + +func nonEmptyLogLines(lines []string) []string { + out := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + out = append(out, line) + } + return out +} + +func visibleLogWindow(lines []string, height int, scrollBack int) []string { + if height <= 0 || len(lines) == 0 { + return nil + } + if len(lines) <= height { + return lines + } + maxStart := len(lines) - height + if scrollBack < 0 { + scrollBack = 0 + } + start := maxStart - scrollBack + if start < 0 { + start = 0 + } + end := start + height + if end > len(lines) { + end = len(lines) + } + return lines[start:end] +} + +func (m model) maxLogScroll() int { + lineCount := m.currentLogLineCount() + _, logRows := m.rawLogLayout() + if lineCount <= logRows { + return 0 + } + return lineCount - logRows +} + +func (m model) currentLogLineCount() int { + width, _ := m.rawLogLayout() + var lines []string + if m.logRawMode || !m.hasLightbringerLogPane() || width < 88 { + lines = dashboardRawLogLines(redactLogLines(m.combinedRawLogLines()), width) + } else { + colWidth := (width - 3) / 2 + if colWidth < 24 { + lines = dashboardRawLogLines(redactLogLines(m.combinedRawLogLines()), width) + } else if m.logPane == logPaneLightbringer { + lines = wrapLogLines(redactLogLines(m.lbLines), colWidth) + } else { + lines = wrapLogLines(redactLogLines(m.mithrilLines), colWidth) + } + } + return len(lines) +} + +func (m model) rawLogLayout() (int, int) { + if m.fullWidthRawLogs() { + return m.fullPaneContentWidth(), m.rawLogRows(true) + } + return m.rightPaneContentWidth(), m.rawLogRows(false) +} + +func (m model) rawLogRows(fullWidth bool) int { + rows := m.height - 22 + if fullWidth { + rows = m.height - 21 + } + if rows < 5 { + return 5 + } + return rows +} + +func (m model) hasLightbringerLogPane() bool { + if m.cfg == nil { + return false + } + return m.cfg.lbEnabled || (m.cfg.blockSource == "lightbringer" && m.cfg.lbExternalEndpoint != "") +} + +func (m model) rightPaneContentWidth() int { + if m.width <= 0 { + return 80 + } + if m.width < 60 { + width := m.width - 2 + if width < 10 { + return 10 + } + return width + } + innerWidth := m.width - 3 + leftWidth := innerWidth * 22 / 100 + rightWidth := innerWidth - leftWidth + if rightWidth < 10 { + return 10 + } + return rightWidth +} + +func (m model) fullPaneContentWidth() int { + width := m.width - 4 + if width < 10 { + return 10 + } + return width +} + +func (m model) fullWidthRawLogs() bool { + return m.mode == modeDashboard && + m.screen == screenLogs && + m.logRawMode && + !m.startFlow.active && + !m.confirmActive +} + +func colorLogLine(line string) string { + line = redactLogLine(line) + trimmed := strings.TrimSpace(line) + if trimmed == "" { + return "" + } + switch { + case strings.Contains(line, " WARN ") || strings.Contains(line, " warn ") || strings.HasPrefix(trimmed, "WARN"): + return lipgloss.NewStyle().Foreground(tui.ColorWarn).Render(line) + case strings.Contains(line, " ERROR ") || strings.Contains(line, " error ") || strings.HasPrefix(trimmed, "ERROR") || strings.Contains(line, "FATAL"): + return lipgloss.NewStyle().Foreground(tui.ColorError).Render(line) + default: + return lipgloss.NewStyle().Foreground(tui.ColorTextMuted).Render(line) + } +} + +func redactLogLine(line string) string { + return config.RedactSecretsInText(sanitizeTerminalLogLine(line)) +} + +func redactLogLines(lines []string) []string { + if len(lines) == 0 { + return nil + } + redacted := make([]string, len(lines)) + for i, line := range lines { + redacted[i] = redactLogLine(line) + } + return redacted +} + +func dashboardRawLogLines(lines []string, width int) []string { + if width < 10 { + width = 10 + } + out := make([]string, 0, len(lines)) + for _, line := range lines { + line = compactReplaySlotLogLine(line, width) + out = append(out, wrapLogLines([]string{line}, width)...) + } + return out +} + +func compactReplaySlotLogLine(line string, width int) string { + source, rest := splitKnownLogSource(line) + if !strings.Contains(rest, " slot ") || !strings.Contains(rest, "| leader:") || !strings.Contains(rest, "| txns:") { + return line + } + + parts := strings.Split(rest, "|") + if len(parts) < 5 { + return line + } + + ts, slot, ok := parseSlotLogHead(parts[0]) + if !ok { + return line + } + + txns := "" + cu := "" + exec := "" + for _, part := range parts[1:] { + part = strings.TrimSpace(part) + switch { + case strings.HasPrefix(part, "txns:"): + txns = compactTxnsField(part) + case strings.HasPrefix(part, "cu:"): + cu = compactCUField(part) + case strings.HasPrefix(part, "exec:"): + exec = compactValueField(part) + } + } + if txns == "" || exec == "" { + return line + } + + prefix := source + if width < 90 { + prefix = compactLogSource(source) + } + head := strings.TrimSpace(prefix + ts) + candidates := [][]string{ + {compactSlotHead(head, formatSlotForDisplay(slot)), txns, exec, cu}, + {compactSlotHead(head, slot), txns, exec, cu}, + {compactSlotHead(head, slot), txns, exec}, + {"slot " + slot, txns, exec}, + } + for _, candidate := range candidates { + candidate = nonEmptyLogLines(candidate) + joined := strings.Join(candidate, " | ") + if lipgloss.Width(joined) <= width { + return joined + } + } + return strings.Join(nonEmptyLogLines([]string{"slot " + slot, txns, exec}), " | ") +} + +func compactSlotHead(head, slot string) string { + if head == "" { + return "slot " + slot + } + return head + " slot " + slot +} + +func splitKnownLogSource(line string) (string, string) { + for _, source := range []string{"[mithril] ", "[lightbringer] "} { + if strings.HasPrefix(line, source) { + return source, strings.TrimSpace(strings.TrimPrefix(line, source)) + } + } + return "", line +} + +func compactLogSource(source string) string { + switch source { + case "[mithril] ": + return "[m] " + case "[lightbringer] ": + return "[lb] " + default: + return source + } +} + +func parseSlotLogHead(head string) (timestamp, slot string, ok bool) { + head = strings.TrimSpace(head) + slotIdx := strings.Index(head, "slot ") + if slotIdx < 0 { + return "", "", false + } + timestamp = strings.Join(strings.Fields(strings.TrimSpace(head[:slotIdx])), "") + fields := strings.Fields(head[slotIdx:]) + if len(fields) < 2 { + return "", "", false + } + slot = fields[1] + return timestamp, slot, true +} + +func compactTxnsField(field string) string { + vote := "" + nonVote := "" + for _, part := range strings.Fields(field) { + switch { + case strings.HasPrefix(part, "v:"): + vote = strings.TrimPrefix(part, "v:") + case strings.HasPrefix(part, "nv:"): + nonVote = strings.TrimPrefix(part, "nv:") + } + } + if vote == "" && nonVote == "" { + return "" + } + if vote == "" { + return "txns nv" + nonVote + } + if nonVote == "" { + return "txns v" + vote + } + return "txns v" + vote + "/nv" + nonVote +} + +func compactCUField(field string) string { + value := strings.TrimSpace(strings.TrimPrefix(field, "cu:")) + n, err := strconv.ParseFloat(value, 64) + if err != nil { + return "cu " + value + } + if n >= 1_000_000 { + return fmt.Sprintf("cu %.1fM", n/1_000_000) + } + return "cu " + strconv.FormatFloat(n, 'f', 0, 64) +} + +func compactValueField(field string) string { + fields := strings.Fields(field) + if len(fields) < 2 { + return strings.TrimSuffix(field, ":") + } + return strings.TrimSuffix(fields[0], ":") + " " + fields[1] +} + +func formatSlotForDisplay(slot string) string { + n, err := strconv.ParseInt(slot, 10, 64) + if err != nil { + return slot + } + raw := strconv.FormatInt(n, 10) + var b strings.Builder + for i, r := range raw { + if i > 0 && (len(raw)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(r) + } + return b.String() +} + +func (m model) lightbringerLogTitle() string { + if m.cfg == nil { + return "lightbringer" + } + if m.cfg.blockSource == "lightbringer" && m.cfg.lbExternalEndpoint != "" && !m.cfg.lbEnabled { + return "lightbringer (external)" + } + if m.cfg.lbEnabled { + return "lightbringer (managed)" + } + return "lightbringer" +} + // wrapLogLines wraps each line to fit within the given width. // Continuation lines are indented with 2 spaces for readability. func wrapLogLines(lines []string, width int) []string { @@ -748,26 +1452,59 @@ func wrapLogLines(lines []string, width int) []string { } var result []string for _, line := range lines { - if len(line) <= width { + if lipgloss.Width(line) <= width { result = append(result, line) continue } // First chunk at full width, continuations indented - result = append(result, line[:width]) - remaining := line[width:] + chunk, remaining := splitDisplayWidth(line, width) + result = append(result, chunk) contWidth := width - 2 // indent continuation for len(remaining) > 0 { - if len(remaining) <= contWidth { + if lipgloss.Width(remaining) <= contWidth { result = append(result, " "+remaining) break } - result = append(result, " "+remaining[:contWidth]) - remaining = remaining[contWidth:] + chunk, remaining = splitDisplayWidth(remaining, contWidth) + result = append(result, " "+chunk) } } return result } +func splitDisplayWidth(s string, width int) (string, string) { + if width <= 0 || s == "" { + return "", s + } + used := 0 + cut := 0 + lastSpaceCut := 0 + lastSpaceWidth := 0 + for i, r := range s { + rw := lipgloss.Width(string(r)) + if used+rw > width { + break + } + used += rw + cut = i + utf8.RuneLen(r) + if r == ' ' || r == '\t' { + lastSpaceCut = cut + lastSpaceWidth = used + } + } + if cut <= 0 { + _, size := utf8.DecodeRuneInString(s) + if size <= 0 { + return "", "" + } + cut = size + } + if cut < len(s) && lastSpaceCut > 0 && lastSpaceWidth >= width/2 { + return strings.TrimRight(s[:lastSpaceCut], " \t"), strings.TrimLeft(s[lastSpaceCut:], " \t") + } + return s[:cut], s[cut:] +} + // ── Disk View ─────────────────────────────────────────────────────────── func (m model) renderDiskView() string { diff --git a/cmd/mithril/main.go b/cmd/mithril/main.go index 29cf85010..507ed4575 100644 --- a/cmd/mithril/main.go +++ b/cmd/mithril/main.go @@ -13,6 +13,7 @@ import ( "github.com/Overclock-Validator/mithril/cmd/mithril/setupcmd" "github.com/Overclock-Validator/mithril/cmd/mithril/statecmd" "github.com/Overclock-Validator/mithril/cmd/mithril/statuscmd" + "github.com/Overclock-Validator/mithril/cmd/mithril/stopcmd" "github.com/Overclock-Validator/mithril/pkg/config" "github.com/spf13/cobra" "k8s.io/klog/v2" @@ -50,13 +51,14 @@ func init() { cmd.PersistentFlags().StringVar(&config.ConfigFile, "config", "", "Path to TOML config file") cmd.AddCommand( - &node.Run, // Primary command for running Mithril - &configcmd.ConfigCmd, // Config management (init, etc.) - &statecmd.StateCmd, // State file inspection and management - &setupcmd.SetupCmd, // Interactive setup wizard - &setupcmd.DoctorCmd, // System health check - &statuscmd.StatusCmd, // Node status - &dashboardcmd.DashboardCmd, // Interactive dashboard + &node.Run, // Primary command for running Mithril + &stopcmd.StopCmd, // Stop a running Mithril (safe SIGTERM + wait) + &configcmd.ConfigCmd, // Config management (init, etc.) + &statecmd.StateCmd, // State file inspection and management + &setupcmd.SetupCmd, // Interactive setup TUI + &setupcmd.DoctorCmd, // System health check + &statuscmd.StatusCmd, // Node status + &dashboardcmd.DashboardCmd, // Interactive dashboard ) } diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index 845d62383..0ce334406 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2,14 +2,13 @@ package node import ( "bufio" - "bytes" "context" "encoding/base64" + "errors" "fmt" "io" "math" "os" - "os/exec" "path/filepath" "runtime" "runtime/debug" @@ -26,6 +25,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/lightbringer" "github.com/Overclock-Validator/mithril/pkg/lthash" "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/procctl" "github.com/Overclock-Validator/mithril/pkg/progress" "github.com/Overclock-Validator/mithril/pkg/replay" "github.com/Overclock-Validator/mithril/pkg/rpcserver" @@ -105,6 +105,9 @@ var ( lightbringerRpcAddr string lightbringerGrpcAddr string lightbringerConfigDir string + lightbringerGossipPort int + lightbringerPortRangeStart int + lightbringerPortRangeEnd int lightbringerInfluxdbHost string lightbringerInfluxdbDatabase string lightbringerInfluxdbToken string @@ -261,6 +264,9 @@ func init() { Run.Flags().IntVar(&blockMaxInflight, "block-max-inflight", 0, "Max concurrent block fetch workers (0 = use default)") Run.Flags().IntVar(&blockTipPollIntervalMs, "block-tip-poll-ms", 0, "Tip poll interval in milliseconds (0 = use default)") Run.Flags().IntVar(&blockTipSafetyMargin, "block-tip-safety-margin", 0, "Don't fetch within N slots of tip (0 = use default)") + Run.Flags().IntVar(&lightbringerGossipPort, "lightbringer-gossip-port", 0, "Managed Lightbringer public Solana gossip UDP port (0 = Lightbringer default)") + Run.Flags().IntVar(&lightbringerPortRangeStart, "lightbringer-port-range-start", 0, "Managed Lightbringer public Solana UDP port range start (0 = Lightbringer default)") + Run.Flags().IntVar(&lightbringerPortRangeEnd, "lightbringer-port-range-end", 0, "Managed Lightbringer public Solana UDP port range end (0 = Lightbringer default)") } @@ -529,6 +535,9 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { if lightbringerConfigDir == "" { lightbringerConfigDir = "." } + lightbringerGossipPort = getInt("lightbringer-gossip-port", "lightbringer.gossip_port") + lightbringerPortRangeStart = getInt("lightbringer-port-range-start", "lightbringer.port_range_start") + lightbringerPortRangeEnd = getInt("lightbringer-port-range-end", "lightbringer.port_range_end") lightbringerInfluxdbHost = config.GetString("lightbringer.influxdb_host") lightbringerInfluxdbDatabase = config.GetString("lightbringer.influxdb_database") lightbringerInfluxdbToken = config.GetString("lightbringer.influxdb_token") @@ -554,6 +563,7 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { mlog.Log.Warnf("lightbringer.grpc_addr (%s) differs from block.lightbringer_endpoint (%s) — using block.lightbringer_endpoint", lightbringerGrpcAddr, lightbringerEndpoint) } + mlog.Log.Warnf("managed Lightbringer opens public Solana UDP gossip/repair sockets; ensure firewall rules match lightbringer.gossip_port and lightbringer.port_range_*") } // Validate block source requirements @@ -785,6 +795,15 @@ func buildSnapshotConfig(rpcEndpoints []string) snapshotdl.SnapshotConfig { return cfg } +func spawnedByForRun() string { + switch v := os.Getenv(procctl.SpawnedByEnv); v { + case "dashboard", "external", "cli": + return v + default: + return "cli" + } +} + func runLive(c *cobra.Command, args []string) { if pprofPort != -1 { startPprofHandlers(int(pprofPort)) @@ -797,6 +816,127 @@ func runLive(c *cobra.Command, args []string) { // Generate run ID early so it's available for logging and state tracking replay.CurrentRunID = replay.GenerateRunID() + // Acquire single-instance lock + PID file before any state mutation to avoid concurrent-run corruption. + execPath, _ := os.Executable() + runHandle, err := procctl.AcquireForRun(procctl.RunOpts{ + PidPath: procctl.DefaultPidFile(), + LockPath: procctl.DefaultLockFile(), + RunID: replay.CurrentRunID, + BinaryPath: execPath, + ConfigPath: config.ConfigFile, + SpawnedBy: spawnedByForRun(), + }) + if err != nil { + if errors.Is(err, procctl.ErrLocked) { + // Read the existing PID file (if any) so we can show a helpful + // message about which mithril is already running. + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "E_ALREADY_RUNNING: another mithril is already running on this host.") + if info, rerr := procctl.ReadPidFile(procctl.DefaultPidFile()); rerr == nil { + fmt.Fprintf(os.Stderr, " PID: %d\n", info.Pid) + fmt.Fprintf(os.Stderr, " Run ID: %s\n", info.RunID) + fmt.Fprintf(os.Stderr, " Started by: %s\n", info.SpawnedBy) + if info.BinaryPath != "" { + fmt.Fprintf(os.Stderr, " Binary: %s\n", info.BinaryPath) + } + } + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "To stop it cleanly: mithril stop") + fmt.Fprintln(os.Stderr) + os.Exit(2) + } + klog.Fatalf("failed to acquire mithril single-instance lock: %v", err) + } + defer runHandle.Close() + var runProgress *runProgressEvents + emitProgress := func(phase, status, message string, fields map[string]any) { + if runProgress != nil { + runProgress.Emit(phase, status, message, fields) + } + } + replayStarted := false + recordStartupFailure := func(reason string) { + if replayStarted || accountsPath == "" { + return + } + st, err := state.LoadState(accountsPath) + if err != nil || st == nil || !st.IsReady() { + return + } + st.CurrentSessionStartedAt = time.Now() + reason = strings.Join(strings.Fields(config.RedactSecretsInText(reason)), " ") + shutdownReason := state.ShutdownReasonStartupFailed + if reason != "" { + shutdownReason += ": " + reason + } + shutdownCtx := &state.ShutdownContext{ + RunID: replay.CurrentRunID, + WriterVersion: getVersion(), + WriterCommit: getCommit(), + WriterBranch: getBranch(), + ShutdownReason: shutdownReason, + } + if err := st.RecordSessionShutdown(accountsPath, shutdownCtx); err != nil { + mlog.Log.Errorf("failed to record startup failure: %v", err) + return + } + state.RecordShutdown(accountsPath, st.GetCurrentSlot(), st.LastBankhash, replay.CurrentRunID, getVersion(), getCommit(), getBranch(), shutdownReason) + } + // Declared early so runFatalf can stop the sidecar (klog.Fatalf skips defers; no Pdeathsig on non-Linux). + var lbManager *lightbringer.Manager + runFatalf := func(format string, args ...interface{}) { + // klog.Fatalf calls os.Exit, so deferred runHandle.Close will not run. + // Remove only the PID file here; keep the flock held until process exit. + emitProgress("error", "error", fmt.Sprintf(format, args...), nil) + if runProgress != nil { + runProgress.Close() + } + // Best-effort stop of the managed sidecar so we don't orphan it. + if lbManager != nil { + _ = lbManager.Stop(5 * time.Second) + } + _ = procctl.RemovePidFile(procctl.DefaultPidFile()) + klog.Fatalf(format, args...) + } + runStartupFatalf := func(format string, args ...interface{}) { + recordStartupFailure(fmt.Sprintf(format, args...)) + runFatalf(format, args...) + } + bootstrapStartedAt := time.Now() + stopIfBootstrapCancelled := func(err error, scope string) bool { + if err == nil { + return false + } + if ctx.Err() == nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + return false + } + emitProgress("shutdown", "ok", "Mithril stopped cleanly", nil) + mlog.Log.Infof("%s cancelled during shutdown: %v", scope, err) + if accountsPath != "" { + st := &state.MithrilState{ + StateSchemaVersion: state.CurrentStateSchemaVersion, + CurrentRunID: replay.CurrentRunID, + LastWriterVersion: getVersion(), + LastWriterCommit: getCommit(), + LastWriterBranch: getBranch(), + LastCommit: getCommit(), + LastShutdownReason: state.ShutdownReasonNormal, + LastShutdownAt: time.Now(), + CurrentSessionStartedAt: bootstrapStartedAt, + Stage: "building", + BuildStartedAt: bootstrapStartedAt, + BuildMode: bootstrapMode, + Cluster: cluster, + } + if err := st.Save(accountsPath); err != nil { + mlog.Log.Errorf("failed to record clean bootstrap shutdown: %v", err) + } else { + state.RecordShutdown(accountsPath, 0, "", replay.CurrentRunID, getVersion(), getCommit(), getBranch(), state.ShutdownReasonNormal) + } + } + return true + } + // Initialize file logging with defaults // Use config.IsSet() to allow explicit empty/zero values: // dir = "" → disable file logging (stdout only) @@ -810,6 +950,8 @@ func runLive(c *cobra.Command, args []string) { // Dir: default to /mnt/mithril-logs, but "" disables file logging if config.IsSet("storage.logs") { logCfg.Dir = config.GetString("storage.logs") + } else if config.IsSet("log.dir") { + logCfg.Dir = config.GetString("log.dir") } else { logCfg.Dir = "/mnt/mithril-logs" } @@ -854,12 +996,21 @@ func runLive(c *cobra.Command, args []string) { // Non-fatal, continue with stdout-only logging fmt.Fprintf(os.Stderr, "warning: failed to initialize file logging: %v\n", err) } + if logDir := mlog.GetLogDir(); logDir != "" { + if err := procctl.UpdatePidLogDir(procctl.DefaultPidFile(), logDir); err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to update pid log dir: %v\n", err) + } + } + runProgress = newRunProgressEvents() + defer runProgress.Close() + emitProgress("starting", "running", "Mithril process started", map[string]any{ + "run_id": replay.CurrentRunID, + "spawned_by": spawnedByForRun(), + }) defer mlog.Shutdown() - // Kill any existing mithril processes to prevent zombie accumulation - if killed := killExistingMithrilProcesses(); killed > 0 { - fmt.Printf(" ⚠ Killed %d existing mithril process(es)\n\n", killed) - } + // Single-instance enforcement is done above via procctl.AcquireForRun + // before any AccountsDB state is mutated or service ports are bound. // Override bootstrap mode display when explicit snapshot paths are provided if snapshotArchivePath != "" { @@ -872,13 +1023,17 @@ func runLive(c *cobra.Command, args []string) { // Now start the metrics server (after banner so errors don't appear first) statsd.StartMetricsServer() - // Lightbringer sidecar management - var lbManager *lightbringer.Manager + // Lightbringer sidecar management (lbManager is declared above so runFatalf + // can stop it on a fatal abort). useLightbringer := blockSource == "lightbringer" useTurbine := blockSource == "turbine" if lightbringerEnabled { lbLogWriter := mlog.Log.CreateSubprocessWriter("lightbringer") + emitProgress("lightbringer_config", "running", "Preparing managed Lightbringer", map[string]any{ + "grpc_addr": lightbringerGrpcAddr, + "rpc_addr": lightbringerRpcAddr, + }) lbManager = lightbringer.NewManager(lightbringer.ManagerConfig{ BinaryPath: lightbringerBinaryPath, @@ -889,6 +1044,9 @@ func runLive(c *cobra.Command, args []string) { Storage: lightbringerStorage, RpcAddr: lightbringerRpcAddr, GrpcAddr: lightbringerGrpcAddr, + GossipPort: lightbringerGossipPort, + PortRangeStart: lightbringerPortRangeStart, + PortRangeEnd: lightbringerPortRangeEnd, InfluxdbHost: lightbringerInfluxdbHost, InfluxdbDatabase: lightbringerInfluxdbDatabase, InfluxdbToken: lightbringerInfluxdbToken, @@ -901,32 +1059,38 @@ func runLive(c *cobra.Command, args []string) { configPath, err := lbManager.WriteConfig() if err != nil { - klog.Fatalf("failed to write Lightbringer config: %v", err) + runStartupFatalf("failed to write Lightbringer config: %v", err) } mlog.Log.Infof("lightbringer: wrote config to %s", configPath) if err := lbManager.Start(); err != nil { + emitProgress("lightbringer_fallback", "warn", "Lightbringer failed to start; using RPC fallback", nil) mlog.Log.Warnf("lightbringer: failed to start: %v — falling back to RPC", err) useLightbringer = false if len(rpcEndpoints) == 0 { - klog.Fatalf("lightbringer failed to start and no RPC endpoints configured for fallback (set network.rpc)") + runStartupFatalf("lightbringer failed to start and no RPC endpoints configured for fallback (set network.rpc)") } } else { + emitProgress("lightbringer_starting", "running", "Waiting for Lightbringer stream service", nil) defer func() { if err := lbManager.Stop(10 * time.Second); err != nil { mlog.Log.Warnf("lightbringer: shutdown error: %v", err) } }() - // Monitor for crashes and auto-restart in background. - // Use sync.Once for safe channel close from both the fallback path and the defer. + // Auto-restart on crash. sync.Once guards the close (fallback path + defer). lbStopMonitor := make(chan struct{}) var lbStopOnce sync.Once stopMonitor := func() { lbStopOnce.Do(func() { close(lbStopMonitor) }) } go lbManager.MonitorAndRestart(lbStopMonitor, 5) defer stopMonitor() - if err := lbManager.WaitReady(30 * time.Second); err != nil { + if err := lbManager.WaitReadyContext(ctx, 30*time.Second); err != nil { + if ctx.Err() != nil { + mlog.Log.Infof("shutdown requested while waiting for Lightbringer readiness") + return + } + emitProgress("lightbringer_fallback", "warn", "Lightbringer was not ready; using RPC fallback", nil) mlog.Log.Warnf("lightbringer: %v — falling back to RPC", err) useLightbringer = false // Stop the monitor and Lightbringer immediately so they don't run unused during replay. @@ -935,13 +1099,20 @@ func runLive(c *cobra.Command, args []string) { mlog.Log.Warnf("lightbringer: stop on fallback: %v", stopErr) } if len(rpcEndpoints) == 0 { - klog.Fatalf("lightbringer not ready and no RPC endpoints configured for fallback (set network.rpc)") + runStartupFatalf("lightbringer not ready and no RPC endpoints configured for fallback (set network.rpc)") } + } else { + emitProgress("lightbringer_ready", "ok", "Lightbringer is ready", map[string]any{ + "grpc_addr": lightbringerGrpcAddr, + }) } } } else if useLightbringer { // block.source=lightbringer but lightbringer.enabled=false — standalone Lightbringer mode - mlog.Log.Infof("block.source=lightbringer with external Lightbringer at %s", lightbringerEndpoint) + emitProgress("lightbringer_external", "ok", "Using external Lightbringer endpoint", map[string]any{ + "endpoint": lightbringerEndpoint, + }) + mlog.Log.Infof("block.source=lightbringer with external Lightbringer at %s", config.RedactEndpointForDisplay(lightbringerEndpoint)) } else if useTurbine { mlog.Log.Infof("block.source=turbine with native turbine receiver on %s", turbineBindAddr) if turbineGossipEntrypoint != "" { @@ -953,12 +1124,12 @@ func runLive(c *cobra.Command, args []string) { dbgOpts, err := replay.NewDebugOptions(debugTxs, debugAcctWrites, debugDumpEpochVotingRewardDiff) if err != nil { - klog.Fatalf("failed to parse --transaction-signatures or --account-writes values: %v", err) + runStartupFatalf("failed to parse --transaction-signatures or --account-writes values: %v", err) } cpuprofWriter, cpuprofCleanup, err := createBufWriter(cpuprofPath) if err != nil { - klog.Fatalf("unable to create cpuprof writer to filename=%s: %v", cpuprofPath, err) + runStartupFatalf("unable to create cpuprof writer to filename=%s: %v", cpuprofPath, err) } defer cpuprofCleanup() if cpuprofWriter != nil { @@ -967,7 +1138,7 @@ func runLive(c *cobra.Command, args []string) { } if len(rpcEndpoints) == 0 { - rpcEndpoints = []string{"https://api.mainnet-beta.solana.com"} + runStartupFatalf("no RPC endpoints configured (set network.rpc)") } // Bootstrap: determine how to initialize AccountsDB based on mode @@ -976,6 +1147,9 @@ func runLive(c *cobra.Command, args []string) { var mithrilState *state.MithrilState // Use configured snapshot directory (storage.snapshots / snapshot.download_path), not scratch snapshotDownloadPath := snapshotDlPath + emitProgress("bootstrap_checking", "running", "Checking local AccountsDB and snapshot state", map[string]any{ + "mode": bootstrapMode, + }) // Prune old history entries if needed (keeps last 100) if accountsPath != "" { @@ -993,7 +1167,7 @@ func runLive(c *cobra.Command, args []string) { genesisHash := fetchGenesisHash(ctx) if genesisHash != "" { if err := mithrilState.ValidateGenesisHash(genesisHash); err != nil { - klog.Fatalf("FATAL: %v\nThis AccountsDB was built for a different cluster. Use --bootstrap snapshot to rebuild.", err) + runStartupFatalf("FATAL: %v\nThis AccountsDB was built for a different cluster. Use --bootstrap snapshot to rebuild.", err) } // If state has no genesis hash (older version), set it now if mithrilState.GenesisHash == "" { @@ -1004,6 +1178,10 @@ func runLive(c *cobra.Command, args []string) { } } } + // Stamp session start so a crash this session isn't mistaken for a prior clean exit. + if err := mithrilState.StartSession(accountsPath); err != nil { + mlog.Log.Infof("WARNING: failed to stamp session start in state file: %v", err) + } } // Fall back to legacy detection if no state file @@ -1016,11 +1194,14 @@ func runLive(c *cobra.Command, args []string) { // Handle explicit --snapshot flag (bypasses all auto-discovery, does NOT delete snapshot files) if snapshotArchivePath != "" { mlog.Log.Infof("Using full snapshot: %s", snapshotArchivePath) + emitProgress("bootstrap_snapshot", "running", "Building AccountsDB from selected snapshot files", map[string]any{ + "mode": "explicit", + }) // Parse full snapshot slot from filename for validation fullSnapshotSlot := parseSlotFromSnapshotName(filepath.Base(snapshotArchivePath)) if fullSnapshotSlot == 0 { - klog.Fatalf("could not parse slot from snapshot filename: %s", snapshotArchivePath) + runStartupFatalf("could not parse slot from snapshot filename: %s", snapshotArchivePath) } if incrementalSnapshotFilename != "" { @@ -1029,10 +1210,10 @@ func runLive(c *cobra.Command, args []string) { // Validate incremental base matches full snapshot slot incrBase, incrEnd := parseSlotsFromIncrementalName(filepath.Base(incrementalSnapshotFilename)) if incrBase == 0 { - klog.Fatalf("could not parse base slot from incremental snapshot filename: %s", incrementalSnapshotFilename) + runStartupFatalf("could not parse base slot from incremental snapshot filename: %s", incrementalSnapshotFilename) } if incrBase != fullSnapshotSlot { - klog.Fatalf("Incremental base slot %d does not match full snapshot slot %d", incrBase, fullSnapshotSlot) + runStartupFatalf("Incremental base slot %d does not match full snapshot slot %d", incrBase, fullSnapshotSlot) } mlog.Log.Infof("Incremental snapshot: base=%d end=%d (validated)", incrBase, incrEnd) } @@ -1042,7 +1223,10 @@ func runLive(c *cobra.Command, args []string) { dp := progress.NewDualProgress() accountsDb, manifest, err = snapshot.BuildAccountsDbPaths(ctx, snapshotArchivePath, incrementalSnapshotFilename, accountsPath, dp) if err != nil { - klog.Fatalf("failed to build AccountsDB from snapshot: %v", err) + if stopIfBootstrapCancelled(err, "snapshot bootstrap") { + return + } + runFatalf("failed to build AccountsDB from snapshot: %v", err) } // Write state file @@ -1061,19 +1245,22 @@ func runLive(c *cobra.Command, args []string) { case "accountsdb": // Mode: Require existing AccountsDB, never download if !hasValidState && !hasAccountsDB { - klog.Fatalf("mode=accountsdb requires existing AccountsDB at %s", accountsPath) + runStartupFatalf("mode=accountsdb requires existing AccountsDB at %s", accountsPath) } + emitProgress("bootstrap_resume", "running", "Opening existing AccountsDB", map[string]any{ + "slot": accountsDBSlot, + }) if !hasValidState { mlog.Log.Infof("WARNING: no state file found, AccountsDB may be from incomplete build") } mlog.Log.Infof("Resuming from existing AccountsDB at slot %d", accountsDBSlot) accountsDb, err = accountsdb.OpenDb(accountsPath) if err != nil { - klog.Fatalf("failed to open AccountsDB at %s: %v", accountsPath, err) + runFatalf("failed to open AccountsDB at %s: %v", accountsPath, err) } manifest, err = snapshot.LoadManifestFromFile(filepath.Join(accountsPath, "manifest")) if err != nil { - klog.Fatalf("failed to load manifest: %v", err) + runFatalf("failed to load manifest: %v", err) } refreshManifestSeedFromManifest(accountsPath, mithrilState, manifest) // Run integrity check if we have a state file (warn only, don't fail - user chose force mode) @@ -1087,8 +1274,11 @@ func runLive(c *cobra.Command, args []string) { case "new-snapshot": // Mode: Always download fresh snapshot, clean everything if snapshotDownloadPath == "" { - klog.Fatalf("mode=new-snapshot requires a snapshot directory (set storage.snapshots or snapshot.download_path in config)") + runStartupFatalf("mode=new-snapshot requires a snapshot directory (set storage.snapshots or snapshot.download_path in config)") } + emitProgress("bootstrap_snapshot", "running", "Downloading fresh snapshot and rebuilding AccountsDB", map[string]any{ + "mode": "new-snapshot", + }) mlog.Log.Infof("mode=new-snapshot: Downloading fresh snapshot") if accountsPath != "" { // Record rebuild in history before cleanup (history file is preserved) @@ -1111,7 +1301,10 @@ func runLive(c *cobra.Command, args []string) { } accountsDb, manifest, err = downloadAndBuildFromSnapshot(ctx, rpcEndpoints, snapshotDownloadPath, accountsPath, blockstorePath) if err != nil { - klog.Fatalf("failed to build AccountsDB from snapshot: %v", err) + if stopIfBootstrapCancelled(err, "snapshot bootstrap") { + return + } + runFatalf("failed to build AccountsDB from snapshot: %v", err) } // Write state file to mark build as complete snapshotEpoch := snapshotEpochForState(manifest) @@ -1127,8 +1320,11 @@ func runLive(c *cobra.Command, args []string) { case "snapshot": // Mode: Rebuild AccountsDB from snapshot, reuse existing snapshot file if fresh enough if snapshotDownloadPath == "" { - klog.Fatalf("mode=snapshot requires a snapshot directory (set storage.snapshots or snapshot.download_path in config)") + runStartupFatalf("mode=snapshot requires a snapshot directory (set storage.snapshots or snapshot.download_path in config)") } + emitProgress("bootstrap_snapshot", "running", "Preparing snapshot rebuild", map[string]any{ + "mode": "snapshot", + }) mlog.Log.Infof("mode=snapshot: Will rebuild AccountsDB from snapshot") if accountsPath != "" { // Record rebuild in history before cleanup (history file is preserved) @@ -1150,10 +1346,14 @@ func runLive(c *cobra.Command, args []string) { if existingSnap != nil { // Reuse existing snapshot + emitProgress("bootstrap_snapshot", "running", "Building AccountsDB from existing snapshot", map[string]any{ + "slot": existingSnap.slot, + }) mlog.Log.Infof("Reusing existing snapshot file at slot %d", existingSnap.slot) accountsDb, manifest, err = buildFromExistingSnapshot(ctx, existingSnap, snapshotDownloadPath, accountsPath, blockstorePath, rpcEndpoints) } else { // Download fresh + emitProgress("bootstrap_snapshot", "running", "Downloading snapshot and building AccountsDB", nil) mlog.Log.Infof("no fresh snapshot file found, downloading new one") // Clean up old snapshot files based on retention settings if snapshotDownloadPath != "" { @@ -1166,7 +1366,10 @@ func runLive(c *cobra.Command, args []string) { accountsDb, manifest, err = downloadAndBuildFromSnapshot(ctx, rpcEndpoints, snapshotDownloadPath, accountsPath, blockstorePath) } if err != nil { - klog.Fatalf("failed to build AccountsDB from snapshot: %v", err) + if stopIfBootstrapCancelled(err, "snapshot bootstrap") { + return + } + runFatalf("failed to build AccountsDB from snapshot: %v", err) } // Write state file to mark build as complete snapshotEpoch := snapshotEpochForState(manifest) @@ -1209,8 +1412,11 @@ func runLive(c *cobra.Command, args []string) { if choice == 2 { // User chose to start fresh from snapshot if snapshotDownloadPath == "" { - klog.Fatalf("cannot rebuild from snapshot: no snapshot directory configured (set storage.snapshots or snapshot.download_path in config)") + runStartupFatalf("cannot rebuild from snapshot: no snapshot directory configured (set storage.snapshots or snapshot.download_path in config)") } + emitProgress("bootstrap_snapshot", "running", "Rebuilding stale AccountsDB from snapshot", map[string]any{ + "slots_behind": slotsBehind, + }) mlog.Log.Infof("User chose to rebuild from latest snapshot") if accountsPath != "" { // Record rebuild in history before cleanup (history file is preserved) @@ -1222,10 +1428,14 @@ func runLive(c *cobra.Command, args []string) { // Check for existing fresh snapshot existingSnap := detectFreshSnapshot(snapshotDownloadPath, fullThreshold, rpcEndpoints, ctx) if existingSnap != nil { + emitProgress("bootstrap_snapshot", "running", "Building AccountsDB from existing snapshot", map[string]any{ + "slot": existingSnap.slot, + }) mlog.Log.Infof("Reusing existing snapshot file at slot %d", existingSnap.slot) accountsDb, manifest, err = buildFromExistingSnapshot(ctx, existingSnap, snapshotDownloadPath, accountsPath, blockstorePath, rpcEndpoints) } else { // Clean up old snapshot files + emitProgress("bootstrap_snapshot", "running", "Downloading snapshot and building AccountsDB", nil) if snapshotDownloadPath != "" { maxSnapshots := config.GetInt("snapshot.max_full_snapshots") if maxSnapshots == 0 { @@ -1236,7 +1446,10 @@ func runLive(c *cobra.Command, args []string) { accountsDb, manifest, err = downloadAndBuildFromSnapshot(ctx, rpcEndpoints, snapshotDownloadPath, accountsPath, blockstorePath) } if err != nil { - klog.Fatalf("failed to build AccountsDB from snapshot: %v", err) + if stopIfBootstrapCancelled(err, "snapshot bootstrap") { + return + } + runFatalf("failed to build AccountsDB from snapshot: %v", err) } snapshotEpoch := snapshotEpochForState(manifest) mithrilState = state.NewReadyState(manifest.Bank.Slot, snapshotEpoch, "", "", 0, 0) @@ -1253,15 +1466,18 @@ func runLive(c *cobra.Command, args []string) { } mlog.Log.Infof("mode=auto: Resuming from existing AccountsDB at slot %d", accountsDBSlot) + emitProgress("bootstrap_resume", "running", "Opening existing AccountsDB", map[string]any{ + "slot": accountsDBSlot, + }) // Record resume in history state.RecordResume(accountsPath, mithrilState.LastSlot, mithrilState.LastBankhash, replay.CurrentRunID, getVersion(), getCommit(), getBranch()) accountsDb, err = accountsdb.OpenDb(accountsPath) if err != nil { - klog.Fatalf("failed to open AccountsDB at %s: %v", accountsPath, err) + runFatalf("failed to open AccountsDB at %s: %v", accountsPath, err) } manifest, err = snapshot.LoadManifestFromFile(filepath.Join(accountsPath, "manifest")) if err != nil { - klog.Fatalf("failed to load manifest: %v", err) + runFatalf("failed to load manifest: %v", err) } refreshManifestSeedFromManifest(accountsPath, mithrilState, manifest) @@ -1284,18 +1500,21 @@ func runLive(c *cobra.Command, args []string) { accountsDb.CloseDb() mlog.Log.Infof("restart mithril to automatically rebuild from snapshot") - klog.Fatalf("AccountsDB corrupted - restart to rebuild") + runFatalf("AccountsDB corrupted - restart to rebuild") } } else { // No valid state - need to clean and rebuild from snapshot if snapshotDownloadPath == "" { - klog.Fatalf("mode=auto requires a snapshot directory to rebuild (set storage.snapshots or snapshot.download_path in config)") + runStartupFatalf("mode=auto requires a snapshot directory to rebuild (set storage.snapshots or snapshot.download_path in config)") } if hasAccountsDB { mlog.Log.Infof("mode=auto: AccountsDB exists but state invalid, rebuilding from snapshot") } else { mlog.Log.Infof("mode=auto: No existing AccountsDB, will download snapshot") } + emitProgress("bootstrap_snapshot", "running", "Building AccountsDB from snapshot", map[string]any{ + "mode": "auto", + }) if accountsPath != "" { // Record rebuild in history before cleanup (history file is preserved) // Try to load any existing state (even invalid) to capture slot info @@ -1317,10 +1536,14 @@ func runLive(c *cobra.Command, args []string) { // Check for existing fresh snapshot existingSnap := detectFreshSnapshot(snapshotDownloadPath, fullThreshold, rpcEndpoints, ctx) if existingSnap != nil { + emitProgress("bootstrap_snapshot", "running", "Building AccountsDB from existing snapshot", map[string]any{ + "slot": existingSnap.slot, + }) mlog.Log.Infof("Reusing existing snapshot file at slot %d", existingSnap.slot) accountsDb, manifest, err = buildFromExistingSnapshot(ctx, existingSnap, snapshotDownloadPath, accountsPath, blockstorePath, rpcEndpoints) } else { // Clean up old snapshot files based on retention settings + emitProgress("bootstrap_snapshot", "running", "Downloading snapshot and building AccountsDB", nil) maxSnapshots := config.GetInt("snapshot.max_full_snapshots") if maxSnapshots == 0 { maxSnapshots = 1 // default: keep 1 snapshot @@ -1329,7 +1552,10 @@ func runLive(c *cobra.Command, args []string) { accountsDb, manifest, err = downloadAndBuildFromSnapshot(ctx, rpcEndpoints, snapshotDownloadPath, accountsPath, blockstorePath) } if err != nil { - klog.Fatalf("failed to build AccountsDB from snapshot: %v", err) + if stopIfBootstrapCancelled(err, "snapshot bootstrap") { + return + } + runFatalf("failed to build AccountsDB from snapshot: %v", err) } // Write state file to mark build as complete snapshotEpoch := snapshotEpochForState(manifest) @@ -1461,10 +1687,14 @@ postBootstrap: // Record bootstrap in history state.RecordBootstrap(accountsPath, manifest.Bank.Slot, "", replay.CurrentRunID, getVersion(), getCommit(), getBranch()) } + emitProgress("bootstrap_ready", "ok", "AccountsDB is ready", map[string]any{ + "snapshot_slot": snapshotBaseSlot, + "start_slot": startSlot, + }) // Support finite replay: --end-slot or --num-slots if endSlot != -1 && numReplaySlots != 0 { - klog.Fatalf("specify at most one of --end-slot and --num-slots") + runStartupFatalf("specify at most one of --end-slot and --num-slots") } liveEndSlot := uint64(math.MaxUint64) if endSlot != -1 { @@ -1481,7 +1711,7 @@ postBootstrap: replayTimingsPath := filepath.Join(mlog.GetLogDir(), "replay_timings.jsonl") metricsWriter, metricsWriterCleanup, err := createBufWriter(replayTimingsPath) if err != nil { - klog.Fatalf("unable to create replay timings writer: %v", err) + runStartupFatalf("unable to create replay timings writer: %v", err) } defer metricsWriterCleanup() @@ -1497,7 +1727,7 @@ postBootstrap: var rpcServer *rpcserver.RpcServer if rpcPort < 0 || rpcPort > 65535 { - klog.Fatalf("invalid port: %d", rpcPort) + runStartupFatalf("invalid port: %d", rpcPort) } else if rpcPort != 0 { rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), epochScheduleFromState(mithrilState)) rpcServer.Start() @@ -1557,7 +1787,31 @@ postBootstrap: if rpcServer != nil { slotCtxSetter = rpcServer } + replayStarted = true + emitProgress("replay_starting", "running", "Starting block replay", map[string]any{ + "start_slot": startSlot, + "end_slot": liveEndSlot, + "block_source": blockSource, + "use_lightbringer": useLightbringer, + "tx_parallelism": txParallelism, + "rpc_server_enabled": rpcServer != nil, + }) result := runReplayWithRecovery(ctx, accountsDb, accountsPath, manifest, resumeState, uint64(startSlot), liveEndSlot, rpcEndpoints, lightbringerEndpoint, turbineBindAddr, turbineGossipEntrypoint, turbineGossipBindAddr, turbineAdvertisedIP, uint16(turbineShredVersion), blockstorePath, int(txParallelism), true, useLightbringer, useTurbine, dbgOpts, metricsWriter, slotCtxSetter, mithrilState, blockFetchOpts, consensusOpts, replayStartTime) + switch { + case result.Error != nil: + emitProgress("replay_stopped", "error", "Replay stopped with an error", map[string]any{ + "last_persisted_slot": result.LastPersistedSlot, + "error": result.Error.Error(), + }) + case result.WasCancelled: + emitProgress("shutdown", "ok", "Mithril stopped cleanly", map[string]any{ + "last_persisted_slot": result.LastPersistedSlot, + }) + default: + emitProgress("completed", "ok", "Replay completed", map[string]any{ + "last_persisted_slot": result.LastPersistedSlot, + }) + } if result.Error != nil { if result.LastPersistedSlot == 0 { @@ -1636,6 +1890,20 @@ postBootstrap: } } } + if result.LastPersistedSlot == 0 && result.WasCancelled && mithrilState != nil && !result.StateWrittenOnCancel { + shutdownCtx := &state.ShutdownContext{ + RunID: replay.CurrentRunID, + WriterVersion: getVersion(), + WriterCommit: getCommit(), + WriterBranch: getBranch(), + ShutdownReason: state.ShutdownReasonNormal, + } + if err := mithrilState.RecordSessionShutdown(accountsPath, shutdownCtx); err != nil { + mlog.Log.Errorf("failed to record clean shutdown without replay progress: %v", err) + } else { + state.RecordShutdown(accountsPath, mithrilState.GetCurrentSlot(), mithrilState.LastBankhash, replay.CurrentRunID, getVersion(), getCommit(), getBranch(), state.ShutdownReasonNormal) + } + } // Print shutdown summary if cancelled or error if (result.WasCancelled || result.Error != nil) && result.LastPersistedSlot > 0 { @@ -1962,9 +2230,9 @@ func printStartupInfo(commandName string) { // RPC endpoints - show auxiliary (network.rpc) endpoints if len(rpcEndpoints) > 0 { - fmt.Printf(" RPC: %s%s%s (primary)\n", gold, rpcEndpoints[0], reset) + fmt.Printf(" RPC: %s%s%s (primary)\n", gold, config.RedactEndpointForDisplay(rpcEndpoints[0]), reset) for _, ep := range rpcEndpoints[1:] { - fmt.Printf(" %s%s%s (fallback)\n", gold, ep, reset) + fmt.Printf(" %s%s%s (fallback)\n", gold, config.RedactEndpointForDisplay(ep), reset) } } if blockSource == "lightbringer" && lightbringerEndpoint != "" { @@ -1982,10 +2250,31 @@ func printStartupInfo(commandName string) { if blockSource == "turbine" && turbineAdvertisedIP != "" { fmt.Printf(" Advertised: %s%s%s\n", gold, turbineAdvertisedIP, reset) } + if lightbringerEnabled { + gossipPort, portRangeStart, portRangeEnd := effectiveLightbringerGossipPorts() + fmt.Printf(" LB gossip: %sUDP %d, range %d-%d%s %s(public Solana gossip/repair)%s\n", + gold, gossipPort, portRangeStart, portRangeEnd, reset, dim, reset) + } fmt.Println() } +func effectiveLightbringerGossipPorts() (int, int, int) { + gossipPort := lightbringerGossipPort + if gossipPort == 0 { + gossipPort = 65400 + } + portRangeStart := lightbringerPortRangeStart + if portRangeStart == 0 { + portRangeStart = 65401 + } + portRangeEnd := lightbringerPortRangeEnd + if portRangeEnd == 0 { + portRangeEnd = 65500 + } + return gossipPort, portRangeStart, portRangeEnd +} + // snapshotInfo holds information about a detected snapshot file type snapshotInfo struct { filename string @@ -2221,8 +2510,20 @@ func queryLatestSnapshotSlot(ctx context.Context, rpcEndpoints []string) (uint64 return uint64(info.Slot), nil } +// ensureDiskSpaceForBuild fails fast if accountsPath/snapshotDir can't hold the AccountsDB. +func ensureDiskSpaceForBuild(accountsPath, snapshotDir string) error { + c := config.CheckBuildSpace(config.GetString("network.cluster"), accountsPath, snapshotDir) + if !c.Determined || c.OK { + return nil // unknown free space — don't block; or there's room + } + return errors.New(c.Reason) +} + // buildFromExistingSnapshot builds AccountsDB from an existing downloaded snapshot file. func buildFromExistingSnapshot(ctx context.Context, snap *snapshotInfo, snapshotDir, accountsPath, blockstorePath string, rpcEndpoints []string) (*accountsdb.AccountsDb, *snapshot.SnapshotManifest, error) { + if err := ensureDiskSpaceForBuild(accountsPath, snapshotDir); err != nil { + return nil, nil, err + } snapCfg := buildSnapshotConfig(rpcEndpoints) // Construct full path to snapshot file @@ -2243,6 +2544,9 @@ func buildFromExistingSnapshot(ctx context.Context, snap *snapshotInfo, snapshot // downloadAndBuildFromSnapshot finds, downloads, and builds AccountsDB from a snapshot func downloadAndBuildFromSnapshot(ctx context.Context, rpcEndpoints []string, snapshotDownloadPath, accountsPath, blockstorePath string) (*accountsdb.AccountsDb, *snapshot.SnapshotManifest, error) { + if err := ensureDiskSpaceForBuild(accountsPath, snapshotDownloadPath); err != nil { + return nil, nil, err + } snapCfg := buildSnapshotConfig(rpcEndpoints) fullSnapshotDlStart := time.Now() fullSnapshotInfo, err := snapshotdl.GetSnapshotURLWithInfo(ctx, snapCfg) @@ -2275,56 +2579,6 @@ func downloadAndBuildFromSnapshot(ctx context.Context, rpcEndpoints []string, sn return accountsDb, manifest, nil } -// killExistingMithrilProcesses finds and kills any other running mithril processes. -// This prevents zombie processes from accumulating and holding disk space. -// Returns the number of processes killed. -func killExistingMithrilProcesses() int { - myPID := os.Getpid() - myPPID := os.Getppid() - - // Use pgrep to find mithril processes by executable name (not full command line) - // This avoids matching sudo or shell processes that have "mithril" in args - cmd := exec.Command("pgrep", "-x", "mithril") - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - // No processes found or pgrep not available - return 0 - } - - lines := strings.Split(strings.TrimSpace(out.String()), "\n") - killed := 0 - - for _, line := range lines { - if line == "" { - continue - } - pid, err := strconv.Atoi(strings.TrimSpace(line)) - if err != nil { - continue - } - - // Don't kill ourselves or our parent (sudo) - if pid == myPID || pid == myPPID { - continue - } - - // Try to kill the process - proc, err := os.FindProcess(pid) - if err != nil { - continue - } - - // Send SIGKILL - if err := proc.Signal(syscall.SIGKILL); err == nil { - killed++ - } - } - - return killed -} - // decodeRecentBlockhashes converts state.BlockhashEntry list to sealevel.SysvarRecentBlockhashes func decodeRecentBlockhashes(entries []state.BlockhashEntry) sealevel.SysvarRecentBlockhashes { result := make(sealevel.SysvarRecentBlockhashes, 0, len(entries)) diff --git a/cmd/mithril/node/progress_events.go b/cmd/mithril/node/progress_events.go new file mode 100644 index 000000000..9bd3ea8fe --- /dev/null +++ b/cmd/mithril/node/progress_events.go @@ -0,0 +1,51 @@ +package node + +import ( + "path/filepath" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/progress" +) + +type runProgressEvents struct { + emitter *progress.JSONLEmitter +} + +func newRunProgressEvents() *runProgressEvents { + runDir := mlog.GetLogDir() + if runDir == "" { + return &runProgressEvents{} + } + emitter, err := progress.NewJSONLEmitter(filepath.Join(runDir, progress.JSONLFileName)) + if err != nil { + mlog.Log.Warnf("progress: failed to open JSONL event stream: %v", err) + return &runProgressEvents{} + } + return &runProgressEvents{emitter: emitter} +} + +func (r *runProgressEvents) Emit(phase, status, message string, fields map[string]any) { + if r == nil || r.emitter == nil { + return + } + event := map[string]any{ + "phase": phase, + "status": status, + "message": config.RedactSecretsInText(message), + } + for k, v := range fields { + if s, ok := v.(string); ok { + event[k] = config.RedactSecretsInText(s) + continue + } + event[k] = v + } + r.emitter.Emit(event) +} + +func (r *runProgressEvents) Close() { + if r != nil && r.emitter != nil { + r.emitter.Close() + } +} diff --git a/cmd/mithril/setupcmd/doctor.go b/cmd/mithril/setupcmd/doctor.go index cc659afee..3b19c2312 100644 --- a/cmd/mithril/setupcmd/doctor.go +++ b/cmd/mithril/setupcmd/doctor.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/lightbringer" ) func runDoctor() { @@ -28,12 +29,22 @@ func runDoctor() { fmt.Printf(" %s Config file found (%s)\n", successStyle.Render("✓"), configPath) passed++ - // Check if config needs migration (missing new sections) - data, _ := os.ReadFile(configPath) - content := string(data) - if !strings.Contains(content, "[lightbringer]") || !strings.Contains(content, "[consensus]") { - fmt.Printf(" %s Config is missing new sections (lightbringer/consensus)\n", warnStyle.Render("~")) - fmt.Printf(" %s Run: mithril doctor --migrate to add them\n", dimStyle.Render("→")) + // Can't read config; skip migration check. + if data, readErr := os.ReadFile(configPath); readErr != nil { + fmt.Printf(" %s Could not read config for migration check: %v\n", warnStyle.Render("~"), readErr) + } else { + content := string(data) + var missingSections []string + if !hasTomlSection(content, "lightbringer") { + missingSections = append(missingSections, "lightbringer") + } + if !hasTomlSection(content, "consensus") { + missingSections = append(missingSections, "consensus") + } + if len(missingSections) > 0 { + fmt.Printf(" %s Config is missing new section(s): %s\n", warnStyle.Render("~"), strings.Join(missingSections, ", ")) + fmt.Printf(" %s Run: mithril doctor --migrate to add them\n", dimStyle.Render("→")) + } } } else { fmt.Printf(" %s Config file not found (%s)\n", errorStyle.Render("✗"), configPath) @@ -42,8 +53,9 @@ func runDoctor() { // Load config for further checks if err := config.InitConfig(); err != nil { + total++ // count the parse attempt as a check fmt.Printf(" %s Failed to parse config: %v\n", errorStyle.Render("✗"), err) - fmt.Printf("\n %d/%d checks passed\n", passed, total) + fmt.Printf("\n %s\n", warnStyle.Render(fmt.Sprintf("%d/%d checks passed", passed, total))) return } @@ -62,8 +74,11 @@ func runDoctor() { // 3. RPC endpoint total++ rpcEndpoints := config.GetStringSlice("network.rpc") + if len(rpcEndpoints) == 0 { + rpcEndpoints = config.GetStringSlice("rpc.rpc") + } if len(rpcEndpoints) > 0 { - ep := rpcEndpoints[0] + ep := config.RedactEndpointForDisplay(rpcEndpoints[0]) fmt.Printf(" %s RPC endpoint configured (%s)\n", successStyle.Render("✓"), ep) passed++ } else { @@ -74,11 +89,19 @@ func runDoctor() { // 4. Storage paths total++ accountsPath := config.GetString("storage.accounts") + if accountsPath == "" { + accountsPath = config.GetString("ledger.accounts_path") + } if accountsPath != "" { - if info, err := os.Stat(accountsPath); err == nil && info.IsDir() { + info, err := os.Stat(accountsPath) + switch { + case err == nil && info.IsDir(): fmt.Printf(" %s AccountsDB path exists (%s)\n", successStyle.Render("✓"), accountsPath) passed++ - } else if accountsPath != "" { + case err == nil && !info.IsDir(): + // path is a file, not a usable AccountsDB dir + fmt.Printf(" %s storage.accounts exists but is not a directory (%s)\n", errorStyle.Render("✗"), accountsPath) + default: fmt.Printf(" %s AccountsDB path: %s (will be created)\n", warnStyle.Render("~"), accountsPath) passed++ } @@ -114,6 +137,27 @@ func runDoctor() { fmt.Printf(" %s lightbringer.gossip_entrypoint not set\n", errorStyle.Render("✗")) } + gossipPort := config.GetInt("lightbringer.gossip_port") + if gossipPort == 0 { + gossipPort = 65400 + } + portRangeStart := config.GetInt("lightbringer.port_range_start") + if portRangeStart == 0 { + portRangeStart = 65401 + } + portRangeEnd := config.GetInt("lightbringer.port_range_end") + if portRangeEnd == 0 { + portRangeEnd = 65500 + } + total++ + if err := lightbringer.ValidateGossipPorts(gossipPort, portRangeStart, portRangeEnd); err != nil { + fmt.Printf(" %s Lightbringer gossip/repair ports invalid: %v\n", errorStyle.Render("✗"), err) + } else { + fmt.Printf(" %s Lightbringer opens public Solana UDP gossip/repair ports: %d, %d-%d\n", + warnStyle.Render("~"), gossipPort, portRangeStart, portRangeEnd) + passed++ + } + total++ grpcAddr := config.GetString("lightbringer.grpc_addr") if grpcAddr == "" { @@ -134,7 +178,7 @@ func runDoctor() { blockSource := config.GetString("block.source") lbEndpoint := config.GetString("block.lightbringer_endpoint") if blockSource == "lightbringer" && lbEndpoint != "" { - fmt.Printf(" %s Lightbringer: external at %s\n", successStyle.Render("✓"), lbEndpoint) + fmt.Printf(" %s Lightbringer: external at %s\n", successStyle.Render("✓"), config.RedactEndpointForDisplay(lbEndpoint)) passed++ total++ } else if blockSource == "lightbringer" { @@ -144,7 +188,6 @@ func runDoctor() { fmt.Printf(" %s Lightbringer: disabled\n", dimStyle.Render("-")) } } - // 6. Logs directory total++ logsDir := config.GetString("storage.logs") @@ -152,8 +195,17 @@ func runDoctor() { logsDir = config.GetString("log.dir") } if logsDir != "" { - fmt.Printf(" %s Log directory: %s\n", successStyle.Render("✓"), logsDir) - passed++ + info, err := os.Stat(logsDir) + switch { + case err == nil && info.IsDir(): + fmt.Printf(" %s Log directory: %s\n", successStyle.Render("✓"), logsDir) + passed++ + case err == nil && !info.IsDir(): + fmt.Printf(" %s storage.logs exists but is not a directory (%s)\n", errorStyle.Render("✗"), logsDir) + default: + fmt.Printf(" %s Log directory: %s (will be created)\n", warnStyle.Render("~"), logsDir) + passed++ + } } else { fmt.Printf(" %s No log directory configured (logs go to stderr only)\n", warnStyle.Render("~")) passed++ // Not critical @@ -168,3 +220,28 @@ func runDoctor() { } fmt.Println() } + +func hasTomlSection(content, section string) bool { + for _, line := range strings.Split(content, "\n") { + if sectionName, ok := tomlSectionName(line); ok && sectionName == section { + return true + } + } + return false +} + +func tomlSectionName(line string) (string, bool) { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || !strings.HasPrefix(trimmed, "[") || strings.HasPrefix(trimmed, "[[") { + return "", false + } + end := strings.Index(trimmed, "]") + if end <= 1 { + return "", false + } + tail := strings.TrimSpace(trimmed[end+1:]) + if tail != "" && !strings.HasPrefix(tail, "#") { + return "", false + } + return strings.TrimSpace(trimmed[1:end]), true +} diff --git a/cmd/mithril/setupcmd/migrate.go b/cmd/mithril/setupcmd/migrate.go index a0684dc1c..9555c0318 100644 --- a/cmd/mithril/setupcmd/migrate.go +++ b/cmd/mithril/setupcmd/migrate.go @@ -13,12 +13,14 @@ import ( func MigrateConfig(configPath string) bool { data, err := os.ReadFile(configPath) if err != nil { + // don't treat a read failure as up-to-date + fmt.Printf(" %s Failed to read config: %v\n", errorStyle.Render("✗"), err) return false } content := string(data) - hasLB := strings.Contains(content, "[lightbringer]") - hasConsensus := strings.Contains(content, "[consensus]") + hasLB := hasTomlSection(content, "lightbringer") + hasConsensus := hasTomlSection(content, "consensus") if hasLB && hasConsensus { return false // already up to date @@ -31,10 +33,10 @@ func MigrateConfig(configPath string) bool { # ============================================================================ # [consensus] - Vote-Anchored Consensus (added by mithril setup --migrate) # ============================================================================ -# [consensus] -# skip_path_max_depth = 64 -# unresolved_policy = "halt" -# enforce_on_source = "stream" +[consensus] +skip_path_max_depth = 64 +unresolved_policy = "halt" +enforce_on_source = "stream" ` } @@ -43,15 +45,16 @@ func MigrateConfig(configPath string) bool { # ============================================================================ # [lightbringer] - Lightbringer Sidecar (added by mithril setup --migrate) # ============================================================================ -# Enable to manage Lightbringer from Mithril. See config.example.toml for details. -# -# [lightbringer] -# enabled = false -# binary_path = "./lightbringer" -# gossip_entrypoint = "" +[lightbringer] +enabled = false +binary_path = "./lightbringer" +gossip_entrypoint = "" +gossip_port = 65400 +port_range_start = 65401 +port_range_end = 65500 # shredstore path is in [storage] section (storage.shredstore) -# rpc_addr = "127.0.0.1:3000" -# grpc_addr = "127.0.0.1:3001" +rpc_addr = "127.0.0.1:3000" +grpc_addr = "127.0.0.1:3001" ` } diff --git a/cmd/mithril/setupcmd/migrate_test.go b/cmd/mithril/setupcmd/migrate_test.go new file mode 100644 index 000000000..9643104a0 --- /dev/null +++ b/cmd/mithril/setupcmd/migrate_test.go @@ -0,0 +1,82 @@ +package setupcmd + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMigrateConfigAddsRealSectionsAndIsIdempotent(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(` +[network] +cluster = "mainnet-beta" +`), 0600); err != nil { + t.Fatalf("write config: %v", err) + } + + if !MigrateConfig(path) { + t.Fatal("expected first migration to modify config") + } + first, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read migrated config: %v", err) + } + body := string(first) + if !hasTomlSection(body, "consensus") { + t.Fatal("migration should add a real [consensus] section") + } + if !hasTomlSection(body, "lightbringer") { + t.Fatal("migration should add a real [lightbringer] section") + } + if countExactLine(body, "[lightbringer]") != 1 { + t.Fatalf("expected one [lightbringer] section, got:\n%s", body) + } + + if MigrateConfig(path) { + t.Fatal("second migration should be a no-op") + } + second, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read config after second migration: %v", err) + } + if string(second) != body { + t.Fatal("second migration changed config content") + } +} + +func TestMigrateConfigRecognizesInlineCommentSections(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + original := ` +[consensus] # vote-anchored consensus +skip_path_max_depth = 64 + +[lightbringer] # sidecar config +enabled = false +` + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + t.Fatalf("write config: %v", err) + } + + if MigrateConfig(path) { + t.Fatal("migration should be a no-op when sections exist with inline comments") + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read config: %v", err) + } + if string(data) != original { + t.Fatal("migration changed an already migrated config") + } +} + +func countExactLine(content, want string) int { + count := 0 + for _, line := range strings.Split(content, "\n") { + if strings.TrimSpace(line) == want { + count++ + } + } + return count +} diff --git a/cmd/mithril/setupcmd/setup.go b/cmd/mithril/setupcmd/setup.go index cae51a337..f5ee6de05 100644 --- a/cmd/mithril/setupcmd/setup.go +++ b/cmd/mithril/setupcmd/setup.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -138,12 +139,14 @@ type setupModel struct { func newSetupModel() setupModel { absPath, _ := filepath.Abs(outputPath) storage := config.DefaultStoragePaths() + cluster := "mainnet-beta" return setupModel{ screen: scrMode, cpuCores: runtime.NumCPU(), disks: DetectDisks(), - cluster: "mainnet-beta", + cluster: cluster, rpcEndpoint: "https://api.mainnet-beta.solana.com", + gossipEntry: defaultGossipEntrypoint(cluster), lbQuiet: config.LightbringerQuietDefault, accountsPath: storage.Accounts, snapshotsPath: storage.Snapshots, @@ -163,6 +166,39 @@ func newSetupModel() setupModel { func (m setupModel) Init() tea.Cmd { return nil } +func defaultGossipEntrypoint(cluster string) string { + switch cluster { + case "mainnet-beta": + return "entrypoint.mainnet-beta.solana.com:8001" + case "testnet": + return "entrypoint.testnet.solana.com:8001" + case "devnet": + return "entrypoint.devnet.solana.com:8001" + default: + return "" + } +} + +// defaultLightbringerBinary returns the first built lightbringer binary found +// in the usual locations, else a PATH lookup, else "./lightbringer". +func defaultLightbringerBinary() string { + candidates := []string{ + "/mnt/mithril-ledger/lightbringer/target/release/lightbringer", + "lightbringer/target/release/lightbringer", + "target/release/lightbringer", + "./lightbringer", + } + for _, c := range candidates { + if info, err := os.Stat(c); err == nil && !info.IsDir() { + return c + } + } + if p, err := exec.LookPath("lightbringer"); err == nil { + return p + } + return "./lightbringer" +} + // ── Navigation helpers ────────────────────────────────────────────────── // inputValueForScreen returns the current config value for an input screen. @@ -399,6 +435,7 @@ func (m setupModel) handleSelect(value string) (tea.Model, tea.Cmd) { case "devnet": m.rpcEndpoint = "https://api.devnet.solana.com" } + m.gossipEntry = defaultGossipEntrypoint(value) m.pushInput(scrRPC) case scrLightbringer: @@ -445,8 +482,7 @@ func (m setupModel) handleSelect(value string) (tea.Model, tea.Cmd) { case scrOverwrite: switch value { case "overwrite": - // User confirmed — proceed with save (scrOverwrite is set, so the - // existence check in generateConfig/generateManual will be skipped) + // confirmed overwrite if m.mode == "manual" { return m.generateManual() } @@ -495,10 +531,17 @@ func (m setupModel) updateInput(msg tea.KeyMsg) (tea.Model, tea.Cmd) { case "ctrl+e": m.inputCur = len(m.inputVal) default: - ch := msg.String() - if len(ch) == 1 && ch[0] >= 32 { - m.inputVal = m.inputVal[:m.inputCur] + ch + m.inputVal[m.inputCur:] - m.inputCur++ + // ASCII-only insert keeps the byte-indexed cursor correct + var ins []rune + for _, r := range msg.Runes { + if r >= 32 && r < 127 { + ins = append(ins, r) + } + } + if len(ins) > 0 { + s := string(ins) + m.inputVal = m.inputVal[:m.inputCur] + s + m.inputVal[m.inputCur:] + m.inputCur += len(s) } } m.inputErr = "" @@ -553,7 +596,7 @@ func (m *setupModel) validateAndApplyInput() bool { } host, portStr, err := net.SplitHostPort(val) if err != nil || host == "" { - m.inputErr = "must be IP:port (e.g., 1.2.3.4:8000)" + m.inputErr = "must be host:port (e.g., entrypoint.mainnet-beta.solana.com:8001)" return false } if p, perr := strconv.Atoi(portStr); perr != nil || p < 1 || p > 65535 { @@ -649,6 +692,15 @@ func (m *setupModel) advanceFromInput() { // ── View ──────────────────────────────────────────────────────────────── func (m setupModel) View() string { + out := m.viewBody() + // When embedded, drop leading blank lines so the view top-anchors. + if m.embedded { + out = strings.TrimLeft(out, "\n") + } + return out +} + +func (m setupModel) viewBody() string { // Skip logo when embedded in dashboard right pane banner := "" if !m.embedded { @@ -672,27 +724,22 @@ func (m setupModel) View() string { case scrGossip: return banner + "\n" + renderInput("Gossip Entrypoint", - "IP:port of a Solana validator running gossip\n"+ - "Used to receive shreds from the network", + "Host:port of a Solana gossip entrypoint\n"+ + "Default uses the official DNS entrypoint for your selected cluster", m.inputVal, m.inputErr, m.inputCur) case scrStorage: desc := "AccountsDB stores all ~500M on-chain accounts · needs fastest NVMe\n" + "Heavy random I/O — put this on your best drive" - if config.IsProductionLayout(config.StoragePaths{ - Accounts: m.accountsPath, - Snapshots: m.snapshotsPath, - Logs: m.logsPath, - Shredstore: m.shredstorePath, - }) { - desc += "\nDefault: production /mnt/* paths (run scripts/disk-setup.sh first)" + // Hint off the accounts path itself (per-path defaults can mix /mnt and home). + if strings.HasPrefix(m.accountsPath, "/mnt/") { + desc += "\nDefault: production NVMe (" + m.accountsPath + ")" } else { desc += "\nDefault: home directory (no /mnt setup detected) — see scripts/disk-setup.sh for production NVMe layout" } if len(m.disks) > 0 { - desc += "\n" for _, d := range m.disks { - desc += "› " + d.FormatDiskOption() + desc += "\n› " + d.FormatDiskOption() } } return banner + "\n" + renderInput("AccountsDB Path", desc, m.inputVal, m.inputErr, m.inputCur) @@ -737,7 +784,7 @@ func (m setupModel) View() string { case scrReview: rows := [][]string{ {"Cluster", m.cluster}, - {"RPC", m.rpcEndpoint}, + {"RPC", config.RedactEndpointForDisplay(m.rpcEndpoint)}, } if m.enableLB { summary := "enabled (gossip: " + m.gossipEntry + ")" @@ -853,8 +900,13 @@ func (m setupModel) generateConfig() (tea.Model, tea.Cmd) { if m.enableLB { cfg.WriteString("[lightbringer]\n") cfg.WriteString("enabled = true\n") - cfg.WriteString("binary_path = \"./lightbringer\"\n") + fmt.Fprintf(&cfg, "binary_path = %q\n", defaultLightbringerBinary()) fmt.Fprintf(&cfg, "gossip_entrypoint = %q\n", m.gossipEntry) + cfg.WriteString("# Managed Lightbringer opens public Solana UDP gossip/repair sockets.\n") + cfg.WriteString("# Keep these aligned with your firewall/security-group rules.\n") + cfg.WriteString("gossip_port = 65400\n") + cfg.WriteString("port_range_start = 65401\n") + cfg.WriteString("port_range_end = 65500\n") cfg.WriteString("grpc_addr = \"127.0.0.1:3001\"\n") cfg.WriteString("rpc_addr = \"127.0.0.1:3000\"\n") fmt.Fprintf(&cfg, "quiet = %t\n", m.lbQuiet) @@ -932,7 +984,10 @@ max_inflight = 8 # [lightbringer] # enabled = false # binary_path = "./lightbringer" -# gossip_entrypoint = "1.2.3.4:8000" +# gossip_entrypoint = "entrypoint.mainnet-beta.solana.com:8001" +# gossip_port = 65400 # Public Solana gossip UDP port +# port_range_start = 65401 # Public Solana repair/TVU UDP range +# port_range_end = 65500 # shredstore stored in [storage] section above # rpc_addr = "127.0.0.1:3000" # grpc_addr = "127.0.0.1:3001" @@ -997,7 +1052,7 @@ func SetupIsDone(m tea.Model) bool { return false } -// SetupIsFirstScreen returns true if the setup wizard is on the initial mode selection screen. +// SetupIsFirstScreen returns true if the setup TUI is on the initial mode selection screen. func SetupIsFirstScreen(m tea.Model) bool { if sm, ok := m.(setupModel); ok { return sm.screen == scrMode diff --git a/cmd/mithril/statuscmd/status.go b/cmd/mithril/statuscmd/status.go index 7515c8014..3539e00a7 100644 --- a/cmd/mithril/statuscmd/status.go +++ b/cmd/mithril/statuscmd/status.go @@ -9,6 +9,8 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + statepkg "github.com/Overclock-Validator/mithril/pkg/state" "github.com/Overclock-Validator/mithril/pkg/tui" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" @@ -53,59 +55,58 @@ func runStatus() { fmt.Println(titleStyle.Render("◎ Mithril Status")) fmt.Println() - // Try to find state file - stateFound := false - searchPaths := []string{accountsPath} - if accountsPath == "" { - searchPaths = []string{ - config.DefaultStoragePaths().Accounts, - "./data/accounts", - ".", - } - } + // Load config before state discovery so status follows the same storage + // path runtime uses when --accounts is not provided. + configErr := config.InitConfig() - var state mithrilState - var statePath string - for _, dir := range searchPaths { - p := filepath.Join(dir, "mithril_state.json") - data, err := os.ReadFile(p) - if err != nil { - continue - } - if err := json.Unmarshal(data, &state); err != nil { - continue - } - statePath = p - stateFound = true - break + configuredAccounts := "" + legacyAccounts := "" + if configErr == nil { + configuredAccounts = config.GetString("storage.accounts") + legacyAccounts = config.GetString("ledger.accounts_path") } + process, processErr := procctl.Detect( + procctl.DefaultPidFile(), + procctl.DefaultLockFile(), + statusDetectionAccountsPath(accountsPath, configuredAccounts, legacyAccounts), + ) + + nodeState, statePath, stateFound := loadStatusState(statusStateSearchPaths( + accountsPath, + configuredAccounts, + legacyAccounts, + )) + + printProcessStatus(process, processErr) + if stateFound { fmt.Printf(" %s State file: %s\n", successStyle.Render("✓"), dimStyle.Render(statePath)) - fmt.Printf(" %s Last slot: %s\n", successStyle.Render("✓"), valueStyle.Render(fmt.Sprintf("%d", state.LastSlot))) - if state.SnapshotSlot > 0 { - fmt.Printf(" %s Snapshot: %s\n", dimStyle.Render("-"), valueStyle.Render(fmt.Sprintf("slot %d", state.SnapshotSlot))) + fmt.Printf(" %s Last slot: %s\n", successStyle.Render("✓"), valueStyle.Render(fmt.Sprintf("%d", nodeState.LastSlot))) + if nodeState.SnapshotSlot > 0 { + fmt.Printf(" %s Snapshot: %s\n", dimStyle.Render("-"), valueStyle.Render(fmt.Sprintf("slot %d", nodeState.SnapshotSlot))) } - if state.ShutdownReason != "" { - fmt.Printf(" %s Last stop: %s\n", dimStyle.Render("-"), valueStyle.Render(state.ShutdownReason)) + if nodeState.ShutdownReason != "" { + fmt.Printf(" %s Last stop: %s\n", dimStyle.Render("-"), valueStyle.Render(nodeState.ShutdownReason)) } - if state.LastBankhash != "" { - short := state.LastBankhash + if nodeState.LastBankhash != "" { + short := nodeState.LastBankhash if len(short) > 12 { short = short[:12] + "..." } fmt.Printf(" %s Bankhash: %s\n", dimStyle.Render("-"), dimStyle.Render(short)) } } else { - fmt.Printf(" %s No state file found\n", warnStyle.Render("~")) - fmt.Printf(" %s Mithril hasn't run yet, or --accounts path is wrong\n", dimStyle.Render("")) + headline, detail := missingStateStatusText(processRunning(process)) + fmt.Printf(" %s %s\n", warnStyle.Render("~"), headline) + fmt.Printf(" %s %s\n", dimStyle.Render(""), detail) } fmt.Println() // Read service addresses from config (fall back to defaults) - if err := config.InitConfig(); err != nil { - fmt.Printf(" %s Failed to read config: %v\n", warnStyle.Render("~"), err) + if configErr != nil { + fmt.Printf(" %s Failed to read config: %v\n", warnStyle.Render("~"), configErr) fmt.Println(" Using default service addresses") } rpcAddr := "127.0.0.1:8899" @@ -146,9 +147,9 @@ func runStatus() { conn, err := net.DialTimeout("tcp", rpcAddr, 2*time.Second) if err == nil { conn.Close() - fmt.Printf(" %s Mithril RPC responding on %s\n", successStyle.Render("✓"), rpcAddr) + fmt.Printf(" %s Mithril RPC responding on %s\n", successStyle.Render("✓"), redactedStatusAddr(rpcAddr)) } else { - fmt.Printf(" %s Mithril RPC not responding on %s\n", dimStyle.Render("-"), rpcAddr) + fmt.Printf(" %s Mithril RPC not responding on %s\n", dimStyle.Render("-"), redactedStatusAddr(rpcAddr)) } } @@ -157,9 +158,9 @@ func runStatus() { conn, err := net.DialTimeout("tcp", lbAddr, 2*time.Second) if err == nil { conn.Close() - fmt.Printf(" %s Lightbringer gRPC responding on %s\n", successStyle.Render("✓"), lbAddr) + fmt.Printf(" %s Lightbringer gRPC responding on %s\n", successStyle.Render("✓"), redactedStatusAddr(lbAddr)) } else { - fmt.Printf(" %s Lightbringer gRPC not responding on %s\n", dimStyle.Render("-"), lbAddr) + fmt.Printf(" %s Lightbringer gRPC not responding on %s\n", dimStyle.Render("-"), redactedStatusAddr(lbAddr)) } // Only probe HTTP when using managed sidecar (not external) @@ -167,9 +168,9 @@ func runStatus() { conn, err = net.DialTimeout("tcp", lbHTTP, 2*time.Second) if err == nil { conn.Close() - fmt.Printf(" %s Lightbringer HTTP responding on %s\n", successStyle.Render("✓"), lbHTTP) + fmt.Printf(" %s Lightbringer HTTP responding on %s\n", successStyle.Render("✓"), redactedStatusAddr(lbHTTP)) } else { - fmt.Printf(" %s Lightbringer HTTP not responding on %s\n", dimStyle.Render("-"), lbHTTP) + fmt.Printf(" %s Lightbringer HTTP not responding on %s\n", dimStyle.Render("-"), redactedStatusAddr(lbHTTP)) } if config.GetBool("lightbringer.quiet") { fmt.Printf(" %s Lightbringer quiet mode: enabled (warn/error only)\n", dimStyle.Render("·")) @@ -179,3 +180,99 @@ func runStatus() { fmt.Println() } + +func redactedStatusAddr(addr string) string { + return config.RedactSecretsInText(config.RedactEndpointForDisplay(addr)) +} + +func printProcessStatus(process *procctl.Detection, err error) { + if err != nil { + fmt.Printf(" %s Process state unavailable: %v\n", warnStyle.Render("~"), err) + return + } + if process == nil { + return + } + + switch process.Status { + case procctl.StatusRunning: + fmt.Printf(" %s Process: running (pid %d)\n", successStyle.Render("✓"), process.Pid) + if process.SpawnedBy != "" { + fmt.Printf(" %s Started by: %s\n", dimStyle.Render("-"), valueStyle.Render(process.SpawnedBy)) + } + if process.LogDir != "" { + fmt.Printf(" %s Logs: %s\n", dimStyle.Render("-"), dimStyle.Render(process.LogDir)) + } + case procctl.StatusCrashed: + fmt.Printf(" %s Process: not running (previous run may need attention)\n", warnStyle.Render("~")) + if process.LastShutdownReason != "" { + fmt.Printf(" %s Last stop: %s\n", dimStyle.Render("-"), valueStyle.Render(process.LastShutdownReason)) + } + } +} + +func processRunning(process *procctl.Detection) bool { + return process != nil && process.Status == procctl.StatusRunning +} + +func missingStateStatusText(running bool) (string, string) { + if running { + return "State file: not ready yet", "Mithril is running; AccountsDB has not produced a ready state yet (bootstrap/build in progress)" + } + return "No state file found", "Mithril hasn't run yet, or --accounts path is wrong" +} + +func statusDetectionAccountsPath(cliAccountsPath, configuredAccountsPath, legacyAccountsPath string) string { + if cliAccountsPath != "" { + return cliAccountsPath + } + if configuredAccountsPath != "" { + return configuredAccountsPath + } + if legacyAccountsPath != "" { + return legacyAccountsPath + } + return config.DefaultStoragePaths().Accounts +} + +func statusStateSearchPaths(cliAccountsPath, configuredAccountsPath, legacyAccountsPath string) []string { + if cliAccountsPath != "" { + return []string{cliAccountsPath} + } + + paths := make([]string, 0, 5) + paths = appendUniquePath(paths, configuredAccountsPath) + paths = appendUniquePath(paths, legacyAccountsPath) + paths = appendUniquePath(paths, config.DefaultStoragePaths().Accounts) + paths = appendUniquePath(paths, "./data/accounts") + paths = appendUniquePath(paths, ".") + return paths +} + +func appendUniquePath(paths []string, path string) []string { + if path == "" { + return paths + } + for _, existing := range paths { + if existing == path { + return paths + } + } + return append(paths, path) +} + +func loadStatusState(searchPaths []string) (mithrilState, string, bool) { + for _, dir := range searchPaths { + p := filepath.Join(dir, statepkg.StateFileName) + data, err := os.ReadFile(p) + if err != nil { + continue + } + var nodeState mithrilState + if err := json.Unmarshal(data, &nodeState); err != nil { + continue + } + return nodeState, p, true + } + return mithrilState{}, "", false +} diff --git a/cmd/mithril/statuscmd/status_test.go b/cmd/mithril/statuscmd/status_test.go new file mode 100644 index 000000000..29ddae1f6 --- /dev/null +++ b/cmd/mithril/statuscmd/status_test.go @@ -0,0 +1,105 @@ +package statuscmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/config" + statepkg "github.com/Overclock-Validator/mithril/pkg/state" +) + +func TestStatusStateSearchPaths_UsesConfiguredAccountsBeforeDefaults(t *testing.T) { + paths := statusStateSearchPaths("", "/var/mithril/accounts", "/legacy/accounts") + + if len(paths) < 2 { + t.Fatalf("expected configured and legacy paths, got %v", paths) + } + if paths[0] != "/var/mithril/accounts" { + t.Fatalf("expected configured accounts path first, got %q", paths[0]) + } + if paths[1] != "/legacy/accounts" { + t.Fatalf("expected legacy accounts path second, got %q", paths[1]) + } +} + +func TestStatusStateSearchPaths_CLIAccountsOverridesConfig(t *testing.T) { + paths := statusStateSearchPaths("/cli/accounts", "/var/mithril/accounts", "/legacy/accounts") + + if len(paths) != 1 || paths[0] != "/cli/accounts" { + t.Fatalf("expected CLI accounts path to be the only search path, got %v", paths) + } +} + +func TestStatusStateSearchPaths_DeduplicatesFallbacks(t *testing.T) { + paths := statusStateSearchPaths("", "/same/accounts", "/same/accounts") + + seen := map[string]bool{} + for _, path := range paths { + if seen[path] { + t.Fatalf("path %q appeared more than once in %v", path, paths) + } + seen[path] = true + } +} + +func TestLoadStatusState_FindsStateInConfiguredPath(t *testing.T) { + dir := t.TempDir() + statePath := filepath.Join(dir, statepkg.StateFileName) + if err := os.WriteFile(statePath, []byte(`{ + "snapshot_slot": 100, + "last_slot": 123, + "last_bankhash": "abcdef1234567890", + "last_shutdown_reason": "graceful shutdown (Ctrl+C)" +}`), 0644); err != nil { + t.Fatalf("write state file: %v", err) + } + + nodeState, foundPath, ok := loadStatusState([]string{filepath.Join(t.TempDir(), "missing"), dir}) + if !ok { + t.Fatal("expected state file to be found") + } + if foundPath != statePath { + t.Fatalf("expected state path %q, got %q", statePath, foundPath) + } + if nodeState.LastSlot != 123 || nodeState.SnapshotSlot != 100 { + t.Fatalf("unexpected state loaded: %+v", nodeState) + } +} + +func TestMissingStateStatusText_RunningProcessMeansBootstrapPending(t *testing.T) { + headline, detail := missingStateStatusText(true) + + if headline != "State file: not ready yet" { + t.Fatalf("unexpected headline: %q", headline) + } + if detail != "Mithril is running; AccountsDB has not produced a ready state yet (bootstrap/build in progress)" { + t.Fatalf("unexpected detail: %q", detail) + } +} + +func TestMissingStateStatusText_NotRunningPreservesOriginalGuidance(t *testing.T) { + headline, detail := missingStateStatusText(false) + + if headline != "No state file found" { + t.Fatalf("unexpected headline: %q", headline) + } + if detail != "Mithril hasn't run yet, or --accounts path is wrong" { + t.Fatalf("unexpected detail: %q", detail) + } +} + +func TestStatusDetectionAccountsPath_UsesRuntimeStoragePrecedence(t *testing.T) { + if got := statusDetectionAccountsPath("/cli", "/configured", "/legacy"); got != "/cli" { + t.Fatalf("expected CLI path, got %q", got) + } + if got := statusDetectionAccountsPath("", "/configured", "/legacy"); got != "/configured" { + t.Fatalf("expected configured path, got %q", got) + } + if got := statusDetectionAccountsPath("", "", "/legacy"); got != "/legacy" { + t.Fatalf("expected legacy path, got %q", got) + } + if got := statusDetectionAccountsPath("", "", ""); got != config.DefaultStoragePaths().Accounts { + t.Fatalf("expected default storage path, got %q", got) + } +} diff --git a/cmd/mithril/stopcmd/stop.go b/cmd/mithril/stopcmd/stop.go new file mode 100644 index 000000000..6168aae39 --- /dev/null +++ b/cmd/mithril/stopcmd/stop.go @@ -0,0 +1,157 @@ +// Package stopcmd implements `mithril stop`: a scriptable clean shutdown path +// that shares procctl's PID identity checks with the dashboard. +package stopcmd + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/procctl" + "github.com/spf13/cobra" +) + +var ( + timeoutFlag time.Duration + statusOnlyFlag bool + accountsDirFlag string + + // StopCmd is the cobra command, registered by cmd/mithril/main.go. + StopCmd = cobra.Command{ + Use: "stop", + Short: "Stop the running Mithril process cleanly", + Long: `Stop sends SIGTERM to the running Mithril process and waits for clean exit. + +Safety: + - Re-verifies process identity (PID + start-time + binary inode) before + sending the signal, so a recycled PID will not be targeted by accident. + - Never SIGKILLs. If --timeout elapses, exits 1 so the operator can decide + whether to use the dashboard's Force Stop (with corruption warning). + - Records the action in the audit log at + $XDG_STATE_HOME/mithril/control.audit. + +Examples: + mithril stop # default 60s wait + mithril stop --timeout 5m # wait up to 5 minutes (long bootstraps) + mithril stop --status # show current state, don't stop`, + RunE: func(cmd *cobra.Command, args []string) error { + return runStop() + }, + } +) + +func init() { + StopCmd.Flags().DurationVar(&timeoutFlag, "timeout", 60*time.Second, + "how long to wait for Mithril to exit after SIGTERM before giving up") + StopCmd.Flags().BoolVar(&statusOnlyFlag, "status", false, + "read-only: report current state without sending a signal") + StopCmd.Flags().StringVar(&accountsDirFlag, "accounts", "", + "path to AccountsDB directory (for crash detection; default: read from config)") +} + +func runStop() error { + pidPath := procctl.DefaultPidFile() + lockPath := procctl.DefaultLockFile() + auditPath := procctl.DefaultAuditLog() + accountsDir := resolveAccountsDir() + + // --status: read-only probe. + if statusOnlyFlag { + return printStatus(pidPath, lockPath, accountsDir) + } + + // Send SIGTERM. + if err := procctl.SignalStop(pidPath, auditPath, os.Getpid()); err != nil { + if errors.Is(err, procctl.ErrPidFileNotFound) { + fmt.Fprintln(os.Stderr, "Mithril is not running (no PID file found).") + os.Exit(2) + } + // Stale PID file: desired end state (not running) already holds. + if errors.Is(err, procctl.ErrProcessNotFound) { + fmt.Fprintln(os.Stderr, "Mithril is not running (process already exited).") + os.Exit(2) + } + return fmt.Errorf("send SIGTERM: %w", err) + } + + fmt.Fprintf(os.Stderr, "Sent SIGTERM. Waiting up to %s for Mithril to exit...\n", timeoutFlag) + det, err := procctl.WaitStopped(pidPath, lockPath, accountsDir, timeoutFlag) + if err != nil { + if errors.Is(err, procctl.ErrStopTimeout) { + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "TIMEOUT: Mithril did not exit within %s.\n", timeoutFlag) + fmt.Fprintln(os.Stderr, " This is normal during long operations (snapshot load, AccountsDB build).") + fmt.Fprintln(os.Stderr, " Options:") + fmt.Fprintln(os.Stderr, " - Re-run with a longer --timeout (e.g. --timeout 30m for bootstrap)") + fmt.Fprintln(os.Stderr, " - Use the dashboard's Force Stop (acknowledges data-loss risk)") + fmt.Fprintln(os.Stderr) + os.Exit(1) + } + return fmt.Errorf("wait for exit: %w", err) + } + + // Report final state. + switch det.Status { + case procctl.StatusStopped: + fmt.Fprintln(os.Stderr, "Mithril stopped cleanly.") + case procctl.StatusCrashed: + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "WARNING: Mithril exited but the state file does not record a clean shutdown.") + if det.LastShutdownReason != "" { + fmt.Fprintf(os.Stderr, " Last reason: %s\n", det.LastShutdownReason) + } + fmt.Fprintln(os.Stderr, " Consider running `mithril doctor` before the next start.") + fmt.Fprintln(os.Stderr) + default: + fmt.Fprintf(os.Stderr, "Mithril is %s.\n", det.Status) + } + return nil +} + +// printStatus implements --status: a read-only one-shot Detect. +func printStatus(pidPath, lockPath, accountsDir string) error { + det, err := procctl.Detect(pidPath, lockPath, accountsDir) + if err != nil { + return fmt.Errorf("detect: %w", err) + } + fmt.Printf("Status: %s\n", det.Status) + if det.Pid != 0 { + fmt.Printf("PID: %d\n", det.Pid) + } + if det.RunID != "" { + fmt.Printf("Run ID: %s\n", det.RunID) + } + if det.SpawnedBy != "" { + fmt.Printf("Spawned by: %s\n", det.SpawnedBy) + } + if det.BinaryPath != "" { + fmt.Printf("Binary: %s\n", det.BinaryPath) + } + if det.LastShutdownReason != "" { + fmt.Printf("Last exit: %s\n", det.LastShutdownReason) + } + if det.StopInProgressBy != 0 { + fmt.Printf("Stopping: dashboard pid %d (since %s)\n", + det.StopInProgressBy, det.StopInProgressAt.Format(time.RFC3339)) + } + return nil +} + +// resolveAccountsDir prefers --accounts, else the configured storage.accounts. +// Empty makes Detect skip crash classification (Status falls back to Stopped). +func resolveAccountsDir() string { + if accountsDirFlag != "" { + return accountsDirFlag + } + if err := config.InitConfig(); err == nil { + if v := config.GetString("storage.accounts"); v != "" { + return v + } + if v := config.GetString("ledger.accounts_path"); v != "" { + return v + } + } + return "" +} diff --git a/cmd/mithril/stopcmd/stop_test.go b/cmd/mithril/stopcmd/stop_test.go new file mode 100644 index 000000000..6edc74792 --- /dev/null +++ b/cmd/mithril/stopcmd/stop_test.go @@ -0,0 +1,63 @@ +package stopcmd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// stop is a thin CLI wrapper around pkg/procctl. These tests cover only the +// CLI surface (flag defaults, registration); signal/wait paths live in procctl. + +// Cobra flag defaults match the contract (60s wait, status off). +func TestStopCmd_DefaultFlags(t *testing.T) { + // Reset in case other tests changed them. + timeoutFlag = 0 + statusOnlyFlag = false + accountsDirFlag = "" + + tf := StopCmd.Flags().Lookup("timeout") + if assert.NotNil(t, tf) { + assert.Equal(t, "1m0s", tf.DefValue, "default --timeout should be 60s") + } + sf := StopCmd.Flags().Lookup("status") + if assert.NotNil(t, sf) { + assert.Equal(t, "false", sf.DefValue) + } + af := StopCmd.Flags().Lookup("accounts") + if assert.NotNil(t, af) { + assert.Equal(t, "", af.DefValue) + } +} + +// Smoke check: Use, Short, and RunE are populated. +func TestStopCmd_Registered(t *testing.T) { + assert.Equal(t, "stop", StopCmd.Use) + assert.NotEmpty(t, StopCmd.Short) + assert.NotNil(t, StopCmd.RunE, "stop must have a RunE") +} + +// --timeout parses durations via cobra's binding. Drives the actual flag so a +// type change (breaking values like "5m") fails here. +func TestTimeoutDurationParses(t *testing.T) { + cases := map[string]time.Duration{ + "30s": 30 * time.Second, + "5m": 5 * time.Minute, + "1h": time.Hour, + "2h30m": 2*time.Hour + 30*time.Minute, + } + for in, want := range cases { + t.Run(in, func(t *testing.T) { + // GetDuration only works if the flag is bound as a duration. + if err := StopCmd.Flags().Set("timeout", in); err != nil { + t.Fatalf("Set(--timeout %q): %v", in, err) + } + got, err := StopCmd.Flags().GetDuration("timeout") + assert.NoError(t, err, "timeout must be a duration flag") + assert.Equal(t, want, got) + }) + } + // Restore default so ordering can't leak a mutated global. + _ = StopCmd.Flags().Set("timeout", "60s") +} diff --git a/config.example.toml b/config.example.toml index baab2e898..ed9f20fd2 100644 --- a/config.example.toml +++ b/config.example.toml @@ -278,8 +278,14 @@ name = "mithril" binary_path = "./lightbringer" # Solana gossip entrypoint (REQUIRED when enabled) - # This is the IP:port of a Solana validator or RPC node running gossip. - # gossip_entrypoint = "1.2.3.4:8000" + # This is the host:port of a Solana validator or RPC node running gossip. + # gossip_entrypoint = "entrypoint.mainnet-beta.solana.com:8001" + + # Managed Lightbringer joins Solana gossip and opens public UDP gossip/repair sockets. + # Keep these aligned with firewall/security-group rules, especially for mainnet. + gossip_port = 65400 + port_range_start = 65401 + port_range_end = 65500 # Lightbringer's debug HTTP endpoint (for inspecting stored shreds) rpc_addr = "127.0.0.1:3000" diff --git a/pkg/config/config.go b/pkg/config/config.go index b4fe0bdaf..d87e09a74 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -166,6 +166,9 @@ type LightbringerConfig struct { RpcAddr string `toml:"rpc_addr" mapstructure:"rpc_addr"` // Debug HTTP endpoint GrpcAddr string `toml:"grpc_addr" mapstructure:"grpc_addr"` // gRPC stream endpoint (auto-synced to block.lightbringer_endpoint) ConfigDir string `toml:"config_dir" mapstructure:"config_dir"` // Directory to write Lightbringer.toml + GossipPort int `toml:"gossip_port" mapstructure:"gossip_port"` // Public Solana gossip UDP port + PortRangeStart int `toml:"port_range_start" mapstructure:"port_range_start"` // Start of public Solana UDP repair/TVU port range + PortRangeEnd int `toml:"port_range_end" mapstructure:"port_range_end"` // End of public Solana UDP repair/TVU port range // Optional: InfluxDB metrics — written as [influxdb] section in generated Lightbringer.toml InfluxdbHost string `toml:"influxdb_host" mapstructure:"influxdb_host"` diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index cd5545740..b8bea5e6d 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -23,23 +23,27 @@ func productionStoragePaths() StoragePaths { } } -// DefaultStoragePaths returns /mnt/mithril-* when /mnt/mithril-accounts is -// writable, otherwise ~/.mithril/*. Detection is all-or-nothing on the -// /mnt/mithril-accounts probe so path roots are never mixed. +// DefaultStoragePaths picks each /mnt/mithril-* path that's creatable, else the +// ~/.mithril/* fallback for that folder (production dirs live on different mounts). func DefaultStoragePaths() StoragePaths { - if isWritable("/mnt/mithril-accounts") { - return productionStoragePaths() - } + prod := productionStoragePaths() home, err := os.UserHomeDir() if err != nil || home == "" { home = "." } base := filepath.Join(home, ".mithril") + + pick := func(prodPath, name string) string { + if isWritable(prodPath) { + return prodPath + } + return filepath.Join(base, name) + } return StoragePaths{ - Accounts: filepath.Join(base, "accounts"), - Snapshots: filepath.Join(base, "snapshots"), - Logs: filepath.Join(base, "logs"), - Shredstore: filepath.Join(base, "shredstore"), + Accounts: pick(prod.Accounts, "accounts"), + Snapshots: pick(prod.Snapshots, "snapshots"), + Logs: pick(prod.Logs, "logs"), + Shredstore: pick(prod.Shredstore, "shredstore"), } } diff --git a/pkg/config/defaults_test.go b/pkg/config/defaults_test.go index e5ebd5e87..6e2f9ecfa 100644 --- a/pkg/config/defaults_test.go +++ b/pkg/config/defaults_test.go @@ -4,7 +4,6 @@ import ( "os" "path/filepath" "runtime" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -64,19 +63,19 @@ func TestDefaultStoragePaths_AllFieldsPopulated(t *testing.T) { assert.NotEmpty(t, p.Shredstore) } -func TestDefaultStoragePaths_AllOrNothingRoot(t *testing.T) { +func TestDefaultStoragePaths_PerPathFallback(t *testing.T) { p := DefaultStoragePaths() - // Either all paths are under /mnt (production) or all are under HOME - // (fallback). We never mix to avoid configs that span multiple roots. - allMnt := strings.HasPrefix(p.Accounts, "/mnt/") && - strings.HasPrefix(p.Snapshots, "/mnt/") && - strings.HasPrefix(p.Logs, "/mnt/") && - strings.HasPrefix(p.Shredstore, "/mnt/") - allHome := strings.Contains(p.Accounts, ".mithril") && - strings.Contains(p.Snapshots, ".mithril") && - strings.Contains(p.Logs, ".mithril") && - strings.Contains(p.Shredstore, ".mithril") - assert.True(t, allMnt || allHome, "paths should be all /mnt or all under .mithril, got %+v", p) + prod := productionStoragePaths() + home, err := os.UserHomeDir() + if err != nil || home == "" { + home = "." + } + base := filepath.Join(home, ".mithril") + // Each field: its production path or its own home-dir fallback. + assert.Contains(t, []string{prod.Accounts, filepath.Join(base, "accounts")}, p.Accounts) + assert.Contains(t, []string{prod.Snapshots, filepath.Join(base, "snapshots")}, p.Snapshots) + assert.Contains(t, []string{prod.Logs, filepath.Join(base, "logs")}, p.Logs) + assert.Contains(t, []string{prod.Shredstore, filepath.Join(base, "shredstore")}, p.Shredstore) } func TestIsProductionLayout(t *testing.T) { diff --git a/pkg/config/diskspace.go b/pkg/config/diskspace.go new file mode 100644 index 000000000..46baa75e5 --- /dev/null +++ b/pkg/config/diskspace.go @@ -0,0 +1,180 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +const gib = 1 << 30 + +// accountsDbBytes is the extracted AccountsDB size estimate by cluster. +// Generous to fail fast before the disk fills mid-build. +func accountsDbBytes(cluster string) uint64 { + switch normCluster(cluster) { + case "mainnet-beta", "mainnet": + return 450 * gib // extracted AccountsDB + case "testnet": + return 100 * gib + default: // devnet / unknown — small AccountsDB + return 40 * gib + } +} + +func snapshotBytes(cluster string) uint64 { + switch normCluster(cluster) { + case "mainnet-beta", "mainnet": + return 150 * gib // compressed full+incremental snapshot saved while streaming + case "testnet": + return 30 * gib + default: + return 15 * gib + } +} + +func normCluster(cluster string) string { return strings.ToLower(strings.TrimSpace(cluster)) } + +// EstimatedAccountsDbBytes is the free space the AccountsDB needs. +func EstimatedAccountsDbBytes(cluster string) uint64 { return accountsDbBytes(cluster) } + +// EstimatedSnapshotBytes is the free space the compressed snapshot needs. +func EstimatedSnapshotBytes(cluster string) uint64 { return snapshotBytes(cluster) } + +// EstimatedBuildBytes is the peak free space a build needs when AccountsDB and +// snapshot share one disk (the common case, always true for safe-folders). +func EstimatedBuildBytes(cluster string) uint64 { + return accountsDbBytes(cluster) + snapshotBytes(cluster) +} + +// BuildSpaceCheck is the reclaim-aware verdict on whether a snapshot rebuild +// has room. Shared by the run-path fail-fast and the TUI confirm cards. +type BuildSpaceCheck struct { + Determined bool // false when free space can't be read (unsupported OS / stat error) + OK bool // disks have room (only meaningful when Determined) + SameDisk bool // accounts and snapshot share one filesystem + UsableGB uint64 // usable space on the accounts disk in GB (free + reclaimable old DB) + NeedGB uint64 // space needed on the accounts disk in GB + Reason string // shortfall message when !OK (empty otherwise) +} + +// CheckBuildSpace reports whether a snapshot rebuild fits. The existing AccountsDB +// is reclaimable; a shared disk must hold both AccountsDB and snapshot at peak. +func CheckBuildSpace(cluster, accountsPath, snapshotDir string) BuildSpaceCheck { + accFree, ok := FreeDiskBytes(accountsPath) + if !ok { + return BuildSpaceCheck{Determined: false} + } + accUsable := accFree + if HasExistingAccountsDb(accountsPath) { + accUsable += ReclaimableDirBytes(accountsPath) + } + accNeed := EstimatedAccountsDbBytes(cluster) + snapNeed := EstimatedSnapshotBytes(cluster) + + same := snapshotDir != "" && SameDisk(accountsPath, snapshotDir) + if same { + total := accNeed + snapNeed + c := BuildSpaceCheck{Determined: true, SameDisk: true, UsableGB: accUsable / gib, NeedGB: total / gib, OK: accUsable >= total} + if !c.OK { + c.Reason = fmt.Sprintf("not enough disk space: %s has ~%d GB usable but a %s build needs ~%d GB (AccountsDB + snapshot share this disk); free space or point storage at a larger disk", + accountsPath, accUsable/gib, cluster, total/gib) + } + return c + } + + c := BuildSpaceCheck{Determined: true, UsableGB: accUsable / gib, NeedGB: accNeed / gib, OK: accUsable >= accNeed} + if !c.OK { + c.Reason = fmt.Sprintf("not enough disk space: %s has ~%d GB usable but the %s AccountsDB needs ~%d GB; free space or point storage at a larger disk", + accountsPath, accUsable/gib, cluster, accNeed/gib) + return c + } + // Snapshot is on a separate disk — check it too. + if snapshotDir != "" { + if snapFree, sok := FreeDiskBytes(snapshotDir); sok && snapFree < snapNeed { + c.OK = false + c.Reason = fmt.Sprintf("not enough disk space for the snapshot: %s has ~%d GB free but ~%d GB is needed; free space or point snapshots at a larger disk", + snapshotDir, snapFree/gib, snapNeed/gib) + } + } + return c +} + +// nearestExistingAncestor walks up to the first existing directory, so +// stat/statfs work even when path hasn't been created yet. +func nearestExistingAncestor(path string) string { + p := strings.TrimSpace(path) + for p != "" { + if _, err := os.Stat(p); err == nil { + return p + } + parent := filepath.Dir(p) + if parent == p { + return p + } + p = parent + } + return p +} + +// FreeDiskBytes returns free bytes on the filesystem that would hold path. ok +// is false when undeterminable (unsupported platform or stat error). +func FreeDiskBytes(path string) (uint64, bool) { + if strings.TrimSpace(path) == "" { + return 0, false + } + return freeBytesForFS(nearestExistingAncestor(path)) +} + +// SameDisk reports whether a and b are on the same filesystem. Returns true when +// undeterminable (the conservative answer that never under-checks free space). +func SameDisk(a, b string) bool { + da, oka := deviceID(nearestExistingAncestor(a)) + db, okb := deviceID(nearestExistingAncestor(b)) + if !oka || !okb { + return true + } + return da == db +} + +// WritableDir reports whether dir (or its nearest existing ancestor) is a +// writable directory, via access(2) (no create-probe; safe on the render path). +func WritableDir(dir string) bool { + if strings.TrimSpace(dir) == "" { + return false + } + p := nearestExistingAncestor(dir) + info, err := os.Stat(p) + return err == nil && info.IsDir() && writableFS(p) +} + +// HasExistingAccountsDb reports whether dir already holds an AccountsDB, by +// checking for the manifest file every build writes. +func HasExistingAccountsDb(dir string) bool { + if strings.TrimSpace(dir) == "" { + return false + } + fi, err := os.Stat(filepath.Join(dir, "manifest")) + return err == nil && fi.Size() > 0 +} + +// ReclaimableDirBytes sums regular-file bytes under dir (0 if missing); these +// bytes add to free space when judging whether a rebuild fits in place. +func ReclaimableDirBytes(dir string) uint64 { + if strings.TrimSpace(dir) == "" { + return 0 + } + var total uint64 + _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() { + if info, ierr := d.Info(); ierr == nil && info.Mode().IsRegular() { + total += uint64(info.Size()) + } + } + return nil + }) + return total +} diff --git a/pkg/config/diskspace_other.go b/pkg/config/diskspace_other.go new file mode 100644 index 000000000..4b64c1b49 --- /dev/null +++ b/pkg/config/diskspace_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin + +package config + +// No portable statfs off Linux/Darwin; the false return makes callers skip the +// disk-space guard. +func freeBytesForFS(string) (uint64, bool) { return 0, false } + +func writableFS(string) bool { return false } + +func deviceID(string) (uint64, bool) { return 0, false } diff --git a/pkg/config/diskspace_unix.go b/pkg/config/diskspace_unix.go new file mode 100644 index 000000000..ce0823cf9 --- /dev/null +++ b/pkg/config/diskspace_unix.go @@ -0,0 +1,29 @@ +//go:build linux || darwin + +package config + +import "syscall" + +// freeBytesForFS reports free bytes via statfs: Bavail * Bsize. Both cast through +// uint64 since field types differ across OS. +func freeBytesForFS(path string) (uint64, bool) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, false + } + return uint64(st.Bavail) * uint64(st.Bsize), true +} + +// writableFS checks write access via access(2). 0x2 is W_OK (same on Linux/Darwin). +func writableFS(path string) bool { + return syscall.Access(path, 0x2) == nil +} + +// deviceID returns the filesystem device ID, used to tell whether two paths share a disk. +func deviceID(path string) (uint64, bool) { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return 0, false + } + return uint64(st.Dev), true +} diff --git a/pkg/config/redact.go b/pkg/config/redact.go new file mode 100644 index 000000000..db3c19a94 --- /dev/null +++ b/pkg/config/redact.go @@ -0,0 +1,138 @@ +package config + +import ( + "net/url" + "regexp" + "strings" +) + +var sensitiveEndpointAssignmentRE = regexp.MustCompile(`(?i)(^|[?&\s])([a-z0-9_.-]*(?:api[-_]?key|apikey|access[-_]?token|x[-_]?api[-_]?key|authorization|auth|token|secret|password|passwd|bearer|jwt)[a-z0-9_.-]*|key)=([^\s&"'<>;,)]+)`) +var sensitiveURLUserInfoRE = regexp.MustCompile(`(?i)\b((?:https?|wss?)://)[^\s/@]+@`) + +// pathSecretRE matches a path segment that looks like an embedded credential +// (20+ token-ish chars) — some RPC providers put the key in the path, not a query. +var pathSecretRE = regexp.MustCompile(`^[A-Za-z0-9_-]{20,}$`) + +// looksLikePathSecret reports whether a path segment is likely a secret token. +// 20+ token-charset path segment; over-redaction is harmless (display-only). +func looksLikePathSecret(seg string) bool { + return pathSecretRE.MatchString(seg) +} + +// urlInTextRE matches a URL token embedded in free-form log text. +var urlInTextRE = regexp.MustCompile(`(?i)(?:https?|wss?)://[^\s"'<>]+`) + +// RedactEndpointForDisplay hides credentials in RPC URLs for display only; +// callers must keep using the original endpoint for network calls. +func RedactEndpointForDisplay(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return endpoint + } + + u, err := url.Parse(endpoint) + if err != nil { + return redactEndpointQueryFallback(endpoint) + } + + if u.User != nil { + u.User = url.User("REDACTED") + } + + query, qErr := url.ParseQuery(u.RawQuery) + if qErr != nil { + // Malformed query: ParseQuery drops params silently, so regex-scrub the raw string. + return RedactSecretsInText(endpoint) + } + changed := false + for key := range query { + if isSensitiveEndpointParam(key) { + query[key] = []string{"REDACTED"} + changed = true + } + } + if changed { + u.RawQuery = query.Encode() + } + + // Redact path-embedded tokens (e.g. Alchemy /v2/, rpcpool //). + if u.Path != "" { + segs := strings.Split(u.Path, "/") + for i, seg := range segs { + if looksLikePathSecret(seg) { + segs[i] = "REDACTED" + } + } + u.Path = strings.Join(segs, "/") + } + + return u.String() +} + +// RedactSecretsInText hides endpoint-style credentials in log lines without +// URL-encoding the surrounding text. +func RedactSecretsInText(text string) string { + text = sensitiveEndpointAssignmentRE.ReplaceAllString(text, "${1}${2}=REDACTED") + text = sensitiveURLUserInfoRE.ReplaceAllString(text, "${1}REDACTED@") + // Also scrub path-embedded tokens inside any URL in the text (Alchemy /v2/, + // rpcpool //), which the query/userinfo regexes above don't catch. + return urlInTextRE.ReplaceAllStringFunc(text, redactURLPathSecrets) +} + +// redactURLPathSecrets redacts secret-looking path segments in a single URL, +// preserving any trailing punctuation that isn't part of the URL. +func redactURLPathSecrets(rawURL string) string { + trailing := "" + for len(rawURL) > 0 && strings.IndexByte(".,);]}", rawURL[len(rawURL)-1]) >= 0 { + trailing = string(rawURL[len(rawURL)-1]) + trailing + rawURL = rawURL[:len(rawURL)-1] + } + u, err := url.Parse(rawURL) + if err != nil || u.Path == "" { + return rawURL + trailing + } + segs := strings.Split(u.Path, "/") + changed := false + for i, seg := range segs { + if looksLikePathSecret(seg) { + segs[i] = "REDACTED" + changed = true + } + } + if !changed { + return rawURL + trailing + } + u.Path = strings.Join(segs, "/") + return u.String() + trailing +} + +func isSensitiveEndpointParam(key string) bool { + normalized := strings.ToLower(strings.ReplaceAll(key, "_", "-")) + switch normalized { + case "api-key", "apikey", "key", "token", "access-token", "auth", "authorization", "x-api-key", "passwd", "bearer", "jwt": + return true + default: + return strings.Contains(normalized, "secret") || + strings.Contains(normalized, "password") + } +} + +func redactEndpointQueryFallback(endpoint string) string { + // url.Parse failed, so regex-scrub user:pass@host and sensitive query keys + // from the raw string instead. + endpoint = sensitiveURLUserInfoRE.ReplaceAllString(endpoint, "${1}REDACTED@") + queryStart := strings.Index(endpoint, "?") + if queryStart == -1 { + return endpoint + } + prefix := endpoint[:queryStart+1] + query := endpoint[queryStart+1:] + parts := strings.Split(query, "&") + for i, part := range parts { + key, _, found := strings.Cut(part, "=") + if found && isSensitiveEndpointParam(key) { + parts[i] = key + "=REDACTED" + } + } + return prefix + strings.Join(parts, "&") +} diff --git a/pkg/config/redact_test.go b/pkg/config/redact_test.go new file mode 100644 index 000000000..484415db3 --- /dev/null +++ b/pkg/config/redact_test.go @@ -0,0 +1,87 @@ +package config + +import ( + "strings" + "testing" +) + +func TestRedactEndpointForDisplay(t *testing.T) { + tests := []struct { + name string + endpoint string + want string + }{ + { + name: "api key query", + endpoint: "https://rpc.example.invalid/?api-key=test-key-00000000-0000-4000-8000-000000000000", + want: "https://rpc.example.invalid/?api-key=REDACTED", + }, + { + name: "preserve non secret query", + endpoint: "https://example.invalid/rpc?network=mainnet", + want: "https://example.invalid/rpc?network=mainnet", + }, + { + name: "redact userinfo", + endpoint: "https://user:pass@example.invalid/rpc", + want: "https://REDACTED@example.invalid/rpc", + }, + { + name: "fallback malformed query", + endpoint: "://bad?token=secret&network=mainnet", + want: "://bad?token=REDACTED&network=mainnet", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RedactEndpointForDisplay(tt.endpoint); got != tt.want { + t.Fatalf("RedactEndpointForDisplay() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestRedactSecretsInText(t *testing.T) { + got := RedactSecretsInText("Reference slot from https://rpc.invalid/?api-key=test-key-00000000&network=mainnet") + if got != "Reference slot from https://rpc.invalid/?api-key=REDACTED&network=mainnet" { + t.Fatalf("RedactSecretsInText() = %q", got) + } + + got = RedactSecretsInText("Reference slot from https://rpc.invalid/?key=test-key-00000000&network=mainnet") + if got != "Reference slot from https://rpc.invalid/?key=REDACTED&network=mainnet" { + t.Fatalf("RedactSecretsInText() bare key = %q", got) + } + + got = RedactSecretsInText("RPC failed at https://user:pass@rpc.invalid/path") + if got != "RPC failed at https://REDACTED@rpc.invalid/path" { + t.Fatalf("RedactSecretsInText() userinfo = %q", got) + } + + plain := "2026-06-01 ERROR [lightbringer::repair::peer_manager] no repair peers available" + if got := RedactSecretsInText(plain); got != plain { + t.Fatalf("RedactSecretsInText() changed plain log line: %q", got) + } +} + +// redacts path-embedded keys and passwd/bearer params; keeps short legit segments +func TestRedact_PathAndExtraParams(t *testing.T) { + cases := []struct{ in, mustNot, must string }{ + {"https://solana-mainnet.g.alchemy.com/v2/aB3xK9mN2pQ7rS5tU8vW1yZ4cD6eF0gH", "aB3xK9mN2pQ7rS5tU8vW1yZ4cD6eF0gH", "REDACTED"}, + {"https://free.rpcpool.com/9f8e7d6c5b4a3210fedcba9876543210/", "9f8e7d6c5b4a3210fedcba9876543210", "REDACTED"}, + {"https://x.io/?passwd=hunter2", "hunter2", "REDACTED"}, + {"https://x.io/?bearer=abc123def456", "abc123def456", "REDACTED"}, + {"https://prov.io/ALLUPPERCASESECRETTOKEN/", "ALLUPPERCASESECRETTOKEN", "REDACTED"}, + {"https://api.mainnet-beta.solana.com", "", "solana.com"}, // legit, unchanged + {"https://x.io/v2/rpc", "", "/v2/rpc"}, // legit short segments unchanged + } + for _, c := range cases { + got := RedactEndpointForDisplay(c.in) + if c.mustNot != "" && strings.Contains(got, c.mustNot) { + t.Errorf("secret leaked: %q -> %q (still contains %q)", c.in, got, c.mustNot) + } + if c.must != "" && !strings.Contains(got, c.must) { + t.Errorf("expected %q in output for %q, got %q", c.must, c.in, got) + } + } +} diff --git a/pkg/config/sanitize.go b/pkg/config/sanitize.go new file mode 100644 index 000000000..7b2abc54b --- /dev/null +++ b/pkg/config/sanitize.go @@ -0,0 +1,31 @@ +package config + +import ( + "strings" + "unicode" + "unicode/utf8" +) + +// maxUserInputLen caps a config value to bound TUI paste size. +const maxUserInputLen = 4096 + +// SanitizeUserInput cleans a TUI config value for single-line storage and terminal +// rendering: drops newlines (TOML injection) and ESC, keeps printable Unicode. +func SanitizeUserInput(s string) string { + var b strings.Builder + b.Grow(len(s)) + n := 0 + for _, r := range s { + // Drop control chars, Cf format chars (bidi overrides, zero-width, + // BOM — invisible, enable display-spoofing), and invalid UTF-8. + if r == utf8.RuneError || unicode.IsControl(r) || unicode.In(r, unicode.Cf) { + continue + } + b.WriteRune(r) + n++ + if n >= maxUserInputLen { + break + } + } + return b.String() +} diff --git a/pkg/config/sanitize_test.go b/pkg/config/sanitize_test.go new file mode 100644 index 000000000..ce5944a9a --- /dev/null +++ b/pkg/config/sanitize_test.go @@ -0,0 +1,62 @@ +package config + +import ( + "strings" + "testing" +) + +func TestSanitizeUserInput(t *testing.T) { + rtl := string(rune(0x202e)) // RIGHT-TO-LEFT OVERRIDE + zwsp := string(rune(0x200b)) // ZERO WIDTH SPACE + bom := string(rune(0xFEFF)) // BYTE ORDER MARK / ZWNBSP + cases := []struct { + name string + in string + want string + }{ + {"plain endpoint", "https://mainnet.helius-rpc.com/?api-key=abc123", "https://mainnet.helius-rpc.com/?api-key=abc123"}, + {"host:port", "203.0.113.10:8000", "203.0.113.10:8000"}, + {"ipv6", "[2001:db8::1]:8001", "[2001:db8::1]:8001"}, + {"keeps spaces", "foo bar", "foo bar"}, + {"strips trailing newline (paste)", "1.2.3.4:8000\n", "1.2.3.4:8000"}, + {"strips CR/LF", "1.2.3.4\r\n", "1.2.3.4"}, + {"strips tab", "a\tb", "ab"}, + // pasted newline must collapse to one line (TOML injection) + {"blocks toml injection", "1.2.3.4:8000\nadmin_key = \"evil\"", "1.2.3.4:8000admin_key = \"evil\""}, + {"strips ESC/ANSI", "ip\x1b[31mRED\x1b[0m", "ip[31mRED[0m"}, + {"strips DEL and C0", "a\x7fb\x00c", "abc"}, + // bidi/invisible Unicode (Cf) — display-spoofing vectors + {"strips RTL override", "1.2.3.4" + rtl + "X", "1.2.3.4X"}, + {"strips zero-width space", "rpc" + zwsp + ".evil.com", "rpc.evil.com"}, + {"strips BOM", bom + "host:8000", "host:8000"}, + {"empty", "", ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := SanitizeUserInput(c.in); got != c.want { + t.Errorf("SanitizeUserInput(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +func TestSanitizeUserInput_NoControlCharsSurvive(t *testing.T) { + // U+009B is a real C1 control (0xC2 0x9B), exercising the 0x80–0x9f branch. + out := SanitizeUserInput("a\x1bb\nc\rd\te\x00f\x7fg" + string(rune(0x9b)) + "h") + for _, r := range out { + if r < 0x20 || r == 0x7f || (r >= 0x80 && r <= 0x9f) { + t.Fatalf("control char %U survived sanitization in %q", r, out) + } + } + if out != "abcdefgh" { + t.Errorf("got %q, want %q", out, "abcdefgh") + } +} + +func TestSanitizeUserInput_CapsLength(t *testing.T) { + in := strings.Repeat("x", maxUserInputLen+500) + got := SanitizeUserInput(in) + if len([]rune(got)) != maxUserInputLen { + t.Errorf("length = %d, want capped at %d", len([]rune(got)), maxUserInputLen) + } +} diff --git a/pkg/lightbringer/config.go b/pkg/lightbringer/config.go index d53c407c9..55a24a9de 100644 --- a/pkg/lightbringer/config.go +++ b/pkg/lightbringer/config.go @@ -3,6 +3,7 @@ package lightbringer import ( "fmt" "net" + "net/url" "os" "path/filepath" "strconv" @@ -11,6 +12,14 @@ import ( const configFileName = "Lightbringer.toml" +const ( + defaultGossipPort = 65400 + defaultPortRangeStart = 65401 + defaultPortRangeEnd = 65500 + minValidatorPortWidth = 25 + validatorQUICPortOffset = 6 +) + // LightbringerTOML represents the Lightbringer.toml structure that Lightbringer expects. // This mirrors the Rust ConfigRaw struct in the Lightbringer source. type LightbringerTOML struct { @@ -18,6 +27,9 @@ type LightbringerTOML struct { Storage string RpcAddr string GrpcAddr string + GossipPort int + PortRangeStart int + PortRangeEnd int // Optional sections InfluxdbHost string @@ -50,6 +62,15 @@ func (c *LightbringerTOML) Validate() error { return err } } + if err := validateGossipPorts(c.GossipPort, c.PortRangeStart, c.PortRangeEnd); err != nil { + return err + } + if err := validateInfluxDB(c.InfluxdbHost, c.InfluxdbDatabase, c.InfluxdbToken); err != nil { + return err + } + if err := validateBlockConfirmation(c.BlockConfirmRpcHTTP, c.BlockConfirmRpcWS); err != nil { + return err + } return nil } @@ -72,6 +93,111 @@ func validateHostPort(addr, field string) error { return nil } +func effectiveGossipPorts(gossipPort, portRangeStart, portRangeEnd int) (int, int, int) { + if gossipPort == 0 { + gossipPort = defaultGossipPort + } + if portRangeStart == 0 { + portRangeStart = defaultPortRangeStart + } + if portRangeEnd == 0 { + portRangeEnd = defaultPortRangeEnd + } + return gossipPort, portRangeStart, portRangeEnd +} + +// ValidateGossipPorts exposes the runtime gossip-port validation so doctor +// enforces the exact same rules the node applies at startup. +func ValidateGossipPorts(gossipPort, portRangeStart, portRangeEnd int) error { + return validateGossipPorts(gossipPort, portRangeStart, portRangeEnd) +} + +func validateGossipPorts(gossipPort, portRangeStart, portRangeEnd int) error { + effectiveGossipPort, effectiveRangeStart, effectiveRangeEnd := effectiveGossipPorts(gossipPort, portRangeStart, portRangeEnd) + values := []struct { + field string + value int + }{ + {"gossip.gossip_port", effectiveGossipPort}, + {"gossip.port_range_start", effectiveRangeStart}, + {"gossip.port_range_end", effectiveRangeEnd}, + } + for _, item := range values { + if item.value < 1 || item.value > 65535 { + return fmt.Errorf("%s %d is out of range 1-65535", item.field, item.value) + } + } + if effectiveRangeStart > effectiveRangeEnd { + return fmt.Errorf("gossip.port_range_start must be <= gossip.port_range_end") + } + if effectiveRangeEnd-effectiveRangeStart < minValidatorPortWidth { + return fmt.Errorf("gossip.port_range_end - gossip.port_range_start must be at least %d", minValidatorPortWidth) + } + if effectiveRangeEnd+validatorQUICPortOffset > 65535 { + return fmt.Errorf("gossip.port_range_end + %d must fit in 65535", validatorQUICPortOffset) + } + if effectiveGossipPort >= effectiveRangeStart && effectiveGossipPort <= effectiveRangeEnd { + return fmt.Errorf("gossip.gossip_port must not overlap gossip.port_range_start..=gossip.port_range_end") + } + return nil +} + +func validateInfluxDB(host, database, token string) error { + host = strings.TrimSpace(host) + database = strings.TrimSpace(database) + token = strings.TrimSpace(token) + if host == "" && database == "" && token == "" { + return nil + } + if host == "" { + return fmt.Errorf("influxdb.host is required when InfluxDB is configured") + } + if database == "" { + return fmt.Errorf("influxdb.database is required when InfluxDB is configured") + } + if token == "" { + return fmt.Errorf("influxdb.token is required when InfluxDB is configured") + } + if err := validateURLWithSchemes(host, "influxdb.host", "http", "https"); err != nil { + return err + } + return nil +} + +func validateBlockConfirmation(rpcHTTP, rpcWS string) error { + rpcHTTP = strings.TrimSpace(rpcHTTP) + rpcWS = strings.TrimSpace(rpcWS) + if rpcHTTP == "" && rpcWS == "" { + return nil + } + if rpcHTTP == "" { + return fmt.Errorf("block_confirmation.rpc_http is required when block confirmation is configured") + } + if rpcWS == "" { + return fmt.Errorf("block_confirmation.rpc_websocket is required when block confirmation is configured") + } + if err := validateURLWithSchemes(rpcHTTP, "block_confirmation.rpc_http", "http", "https"); err != nil { + return err + } + if err := validateURLWithSchemes(rpcWS, "block_confirmation.rpc_websocket", "ws", "wss"); err != nil { + return err + } + return nil +} + +func validateURLWithSchemes(raw, field string, allowedSchemes ...string) error { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("%s %q is not a valid URL", field, raw) + } + for _, scheme := range allowedSchemes { + if parsed.Scheme == scheme { + return nil + } + } + return fmt.Errorf("%s must use one of: %s", field, strings.Join(allowedSchemes, ", ")) +} + // GenerateTOML produces a valid Lightbringer.toml string from the config. func (c *LightbringerTOML) GenerateTOML() string { var b strings.Builder @@ -81,17 +207,31 @@ func (c *LightbringerTOML) GenerateTOML() string { fmt.Fprintf(&b, "rpc_addr = %q\n", c.RpcAddr) fmt.Fprintf(&b, "grpc_addr = %q\n", c.GrpcAddr) - if c.InfluxdbHost != "" { + if c.GossipPort != 0 || c.PortRangeStart != 0 || c.PortRangeEnd != 0 { + b.WriteString("\n[gossip]\n") + if c.GossipPort != 0 { + fmt.Fprintf(&b, "gossip_port = %d\n", c.GossipPort) + } + if c.PortRangeStart != 0 { + fmt.Fprintf(&b, "port_range_start = %d\n", c.PortRangeStart) + } + if c.PortRangeEnd != 0 { + fmt.Fprintf(&b, "port_range_end = %d\n", c.PortRangeEnd) + } + } + + // Trim to match Validate(): blank means unconfigured; padded values would emit invalid TOML strings. + if host := strings.TrimSpace(c.InfluxdbHost); host != "" { b.WriteString("\n[influxdb]\n") - fmt.Fprintf(&b, "host = %q\n", c.InfluxdbHost) - fmt.Fprintf(&b, "database = %q\n", c.InfluxdbDatabase) - fmt.Fprintf(&b, "token = %q\n", c.InfluxdbToken) + fmt.Fprintf(&b, "host = %q\n", host) + fmt.Fprintf(&b, "database = %q\n", strings.TrimSpace(c.InfluxdbDatabase)) + fmt.Fprintf(&b, "token = %q\n", strings.TrimSpace(c.InfluxdbToken)) } - if c.BlockConfirmRpcHTTP != "" { + if rpcHTTP := strings.TrimSpace(c.BlockConfirmRpcHTTP); rpcHTTP != "" { b.WriteString("\n[block_confirmation]\n") - fmt.Fprintf(&b, "rpc_http = %q\n", c.BlockConfirmRpcHTTP) - fmt.Fprintf(&b, "rpc_websocket = %q\n", c.BlockConfirmRpcWS) + fmt.Fprintf(&b, "rpc_http = %q\n", rpcHTTP) + fmt.Fprintf(&b, "rpc_websocket = %q\n", strings.TrimSpace(c.BlockConfirmRpcWS)) } // Emit [log] section only when quiet mode is enabled. diff --git a/pkg/lightbringer/config_test.go b/pkg/lightbringer/config_test.go index 4f4015cd8..02e1dad24 100644 --- a/pkg/lightbringer/config_test.go +++ b/pkg/lightbringer/config_test.go @@ -3,6 +3,7 @@ package lightbringer import ( "os" "path/filepath" + "runtime" "strings" "testing" @@ -48,6 +49,190 @@ func TestValidate_RequiredFields(t *testing.T) { } } +func TestValidate_GossipPorts(t *testing.T) { + base := LightbringerTOML{ + GossipEntrypoint: "1.2.3.4:8000", + Storage: "/data/shreds", + RpcAddr: "127.0.0.1:3000", + GrpcAddr: "127.0.0.1:3001", + GossipPort: 55000, + PortRangeStart: 55001, + PortRangeEnd: 55100, + } + + tests := []struct { + name string + mutate func(*LightbringerTOML) + wantErr string + }{ + {name: "valid explicit ports"}, + { + name: "narrow range", + mutate: func(cfg *LightbringerTOML) { + cfg.PortRangeEnd = 55010 + }, + wantErr: "at least 25", + }, + { + name: "overlap", + mutate: func(cfg *LightbringerTOML) { + cfg.GossipPort = 55010 + }, + wantErr: "must not overlap", + }, + { + name: "quic overflow", + mutate: func(cfg *LightbringerTOML) { + cfg.PortRangeStart = 65500 + cfg.PortRangeEnd = 65535 + }, + wantErr: "must fit", + }, + { + name: "zero values use lightbringer defaults", + mutate: func(cfg *LightbringerTOML) { + cfg.GossipPort = 0 + cfg.PortRangeStart = 0 + cfg.PortRangeEnd = 0 + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := base + if tt.mutate != nil { + tt.mutate(&cfg) + } + err := cfg.Validate() + if tt.wantErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + +func TestValidate_InfluxDBRequiresCompleteSection(t *testing.T) { + base := LightbringerTOML{ + GossipEntrypoint: "1.2.3.4:8000", + Storage: "/data/shreds", + RpcAddr: "127.0.0.1:3000", + GrpcAddr: "127.0.0.1:3001", + InfluxdbHost: "http://127.0.0.1:18181", + InfluxdbDatabase: "lightbringer", + InfluxdbToken: "token", + } + + tests := []struct { + name string + mutate func(*LightbringerTOML) + wantErr string + }{ + {name: "complete section"}, + { + name: "host only", + mutate: func(cfg *LightbringerTOML) { + cfg.InfluxdbDatabase = "" + cfg.InfluxdbToken = "" + }, + wantErr: "influxdb.database is required", + }, + { + name: "missing token", + mutate: func(cfg *LightbringerTOML) { + cfg.InfluxdbToken = "" + }, + wantErr: "influxdb.token is required", + }, + { + name: "wrong host scheme", + mutate: func(cfg *LightbringerTOML) { + cfg.InfluxdbHost = "ws://127.0.0.1:18181" + }, + wantErr: "influxdb.host must use one of", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := base + if tt.mutate != nil { + tt.mutate(&cfg) + } + err := cfg.Validate() + if tt.wantErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + +func TestValidate_BlockConfirmationRequiresCompleteSection(t *testing.T) { + base := LightbringerTOML{ + GossipEntrypoint: "1.2.3.4:8000", + Storage: "/data/shreds", + RpcAddr: "127.0.0.1:3000", + GrpcAddr: "127.0.0.1:3001", + BlockConfirmRpcHTTP: "http://127.0.0.1:8900", + BlockConfirmRpcWS: "ws://127.0.0.1:8900", + } + + tests := []struct { + name string + mutate func(*LightbringerTOML) + wantErr string + }{ + {name: "complete section"}, + { + name: "missing websocket", + mutate: func(cfg *LightbringerTOML) { + cfg.BlockConfirmRpcWS = "" + }, + wantErr: "block_confirmation.rpc_websocket is required", + }, + { + name: "missing http", + mutate: func(cfg *LightbringerTOML) { + cfg.BlockConfirmRpcHTTP = "" + }, + wantErr: "block_confirmation.rpc_http is required", + }, + { + name: "http field uses ws scheme", + mutate: func(cfg *LightbringerTOML) { + cfg.BlockConfirmRpcHTTP = "ws://127.0.0.1:8900" + }, + wantErr: "block_confirmation.rpc_http must use one of", + }, + { + name: "websocket field uses http scheme", + mutate: func(cfg *LightbringerTOML) { + cfg.BlockConfirmRpcWS = "http://127.0.0.1:8900" + }, + wantErr: "block_confirmation.rpc_websocket must use one of", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := base + if tt.mutate != nil { + tt.mutate(&cfg) + } + err := cfg.Validate() + if tt.wantErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + func TestGenerateTOML_RequiredFieldsOnly(t *testing.T) { cfg := LightbringerTOML{ GossipEntrypoint: "1.2.3.4:8000", @@ -66,6 +251,25 @@ func TestGenerateTOML_RequiredFieldsOnly(t *testing.T) { assert.NotContains(t, toml, "[block_confirmation]") } +func TestGenerateTOML_WithGossipConfig(t *testing.T) { + cfg := LightbringerTOML{ + GossipEntrypoint: "1.2.3.4:8000", + Storage: "/data/shreds", + RpcAddr: "127.0.0.1:3000", + GrpcAddr: "127.0.0.1:3001", + GossipPort: 55000, + PortRangeStart: 55001, + PortRangeEnd: 55100, + } + + toml := cfg.GenerateTOML() + + assert.Contains(t, toml, "[gossip]") + assert.Contains(t, toml, "gossip_port = 55000") + assert.Contains(t, toml, "port_range_start = 55001") + assert.Contains(t, toml, "port_range_end = 55100") +} + func TestGenerateTOML_WithInfluxDB(t *testing.T) { cfg := LightbringerTOML{ GossipEntrypoint: "1.2.3.4:8000", @@ -214,6 +418,15 @@ func TestWriteConfigFile_CreatesValidFile(t *testing.T) { require.NoError(t, err) assert.Contains(t, string(content), `gossip_entrypoint = "10.0.0.1:8000"`) assert.Contains(t, string(content), `grpc_addr = "0.0.0.0:3001"`) + + // The config can carry an InfluxDB token, so the file must be private + // (0600). Permission bits aren't meaningful on Windows. + if runtime.GOOS != "windows" { + info, statErr := os.Stat(path) + require.NoError(t, statErr) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), + "Lightbringer config may hold a secret token; must not be group/world-readable") + } } func TestWriteConfigFile_OverwritesExisting(t *testing.T) { @@ -265,3 +478,11 @@ func TestWriteConfigFile_NoTempFileLeftOnSuccess(t *testing.T) { assert.Len(t, entries, 1) assert.Equal(t, "Lightbringer.toml", entries[0].Name()) } + +// The exported wrapper (used by doctor) must enforce the same rules as runtime. +func TestValidateGossipPorts_ExportedWrapper(t *testing.T) { + require.NoError(t, ValidateGossipPorts(55000, 55001, 55100)) + require.ErrorContains(t, ValidateGossipPorts(55010, 55001, 55100), "must not overlap") + require.ErrorContains(t, ValidateGossipPorts(55000, 55001, 55010), "at least 25") + require.ErrorContains(t, ValidateGossipPorts(70000, 55001, 55100), "out of range") +} diff --git a/pkg/lightbringer/manager.go b/pkg/lightbringer/manager.go index 42a398588..56f9d5c41 100644 --- a/pkg/lightbringer/manager.go +++ b/pkg/lightbringer/manager.go @@ -2,9 +2,9 @@ package lightbringer import ( "bufio" + "context" "fmt" "io" - "net" "os" "os/exec" "path/filepath" @@ -15,6 +15,10 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/overcast" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Manager handles the lifecycle of a Lightbringer child process: @@ -134,6 +138,7 @@ func (m *Manager) Start() error { return } + m.running.Store(true) resultCh <- startResult{cmd: cmd, stdout: stdout, stderr: stderr} // Block this goroutine (and its locked thread) until the child exits. @@ -142,7 +147,10 @@ func (m *Manager) Start() error { // Signal exit to the manager m.running.Store(false) - if waitErr != nil { + deliberateStop := m.stopping.Load() + if waitErr != nil && deliberateStop { + mlog.Log.Infof("lightbringer: process exited after stop: %v", waitErr) + } else if waitErr != nil { mlog.Log.Warnf("lightbringer: process exited with error: %v", waitErr) } else { mlog.Log.Infof("lightbringer: process exited cleanly") @@ -159,7 +167,6 @@ func (m *Manager) Start() error { m.cmd = res.cmd m.done = done - m.running.Store(true) mlog.Log.Infof("lightbringer: started process (pid=%d, binary=%s)", res.cmd.Process.Pid, m.binaryPath) @@ -170,34 +177,102 @@ func (m *Manager) Start() error { return nil } -// WaitReady polls the gRPC endpoint until it accepts a TCP connection, -// or the timeout expires. +// WaitReady polls the gRPC slot-stream service until the real Lightbringer +// protocol responds, or the timeout expires. func (m *Manager) WaitReady(timeout time.Duration) error { - deadline := time.Now().Add(timeout) + return m.WaitReadyContext(context.Background(), timeout) +} + +// WaitReadyContext polls the gRPC slot-stream service until the real +// Lightbringer protocol responds, the timeout expires, or ctx is cancelled. +func (m *Manager) WaitReadyContext(ctx context.Context, timeout time.Duration) error { + if ctx == nil { + ctx = context.Background() + } + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + pollInterval := 500 * time.Millisecond - for time.Now().Before(deadline) { + for { + select { + case <-waitCtx.Done(): + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("gRPC endpoint %s not ready after %s", m.grpcAddr, timeout) + default: + } + if !m.running.Load() { return fmt.Errorf("process exited before becoming ready") } - remaining := time.Until(deadline) dialTimeout := 2 * time.Second - if remaining < dialTimeout { - dialTimeout = remaining + if deadline, ok := waitCtx.Deadline(); ok { + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("gRPC endpoint %s not ready after %s", m.grpcAddr, timeout) + } + if remaining < dialTimeout { + dialTimeout = remaining + } } - conn, err := net.DialTimeout("tcp", m.grpcAddr, dialTimeout) - if err == nil { - _ = conn.Close() - mlog.Log.Infof("lightbringer: gRPC endpoint ready at %s", m.grpcAddr) + if err := probeSlotStreamContext(waitCtx, m.grpcAddr, dialTimeout); err == nil { + mlog.Log.Infof("lightbringer: gRPC slot stream ready at %s", m.grpcAddr) return nil } - time.Sleep(pollInterval) + timer := time.NewTimer(pollInterval) + select { + case <-waitCtx.Done(): + timer.Stop() + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("gRPC endpoint %s not ready after %s", m.grpcAddr, timeout) + case <-timer.C: + } + } +} + +func probeSlotStreamContext(parent context.Context, addr string, timeout time.Duration) error { + if parent == nil { + parent = context.Background() + } + ctx, cancel := context.WithTimeout(parent, timeout) + defer cancel() + + conn, err := grpc.NewClient(addr, grpc.WithInsecure()) + if err != nil { + return err } + defer conn.Close() + + client := overcast.NewSlotStreamClient(conn) + return probeSlotStreamClient(ctx, client) +} - return fmt.Errorf("gRPC endpoint %s not ready after %s", m.grpcAddr, timeout) +func probeSlotStreamClient(ctx context.Context, client overcast.SlotStreamClient) error { + stream, err := client.StreamSlots(ctx, &overcast.SlotStreamRequest{}) + if err != nil { + return err + } + _, err = stream.Recv() + if err == nil { + return nil + } + switch status.Code(err) { + case codes.DeadlineExceeded: + // The service accepted the SlotStream method but did not have a slot + // ready before the probe timeout. That is still a valid readiness + // signal: the target is Lightbringer, not an arbitrary TCP listener or + // unrelated gRPC service. + return nil + default: + return err + } } // captureOutput reads from a pipe line-by-line and writes to the log writer. @@ -282,9 +357,13 @@ func (m *Manager) Pid() int { // with exponential backoff. Stops monitoring when stopCh is closed. // maxRetries=0 means unlimited retries. Returns when stopped or max retries exceeded. func (m *Manager) MonitorAndRestart(stopCh <-chan struct{}, maxRetries int) { - backoff := 2 * time.Second + const initialBackoff = 2 * time.Second + // Uptime past this counts as a healthy run, not a crash loop. + const stabilityWindow = 60 * time.Second + backoff := initialBackoff maxBackoff := 60 * time.Second retries := 0 + lastStartAt := time.Now() // the caller already Start()ed the process we monitor for { done := m.Done() @@ -298,6 +377,12 @@ func (m *Manager) MonitorAndRestart(stopCh <-chan struct{}, maxRetries int) { case <-done: } + // Reset crash-loop counter/backoff after a healthy run so an isolated crash gets the full retry budget. + if time.Since(lastStartAt) >= stabilityWindow { + retries = 0 + backoff = initialBackoff + } + // Process exited — wait for running flag to be cleared // (small window between done closing and running.Store(false)) for i := 0; i < 10 && m.running.Load(); i++ { @@ -314,36 +399,44 @@ func (m *Manager) MonitorAndRestart(stopCh <-chan struct{}, maxRetries int) { default: } - retries++ - if maxRetries > 0 && retries > maxRetries { - mlog.Log.Errorf("lightbringer: exceeded %d restart attempts, giving up", maxRetries) - return - } - - mlog.Log.Warnf("lightbringer: process exited unexpectedly, restarting in %s (attempt %d)", backoff, retries) - - select { - case <-stopCh: - return - case <-time.After(backoff): - } - - // Exponential backoff for next attempt - backoff = backoff * 2 - if backoff > maxBackoff { - backoff = maxBackoff - } - - if _, err := m.WriteConfig(); err != nil { - mlog.Log.Errorf("lightbringer: failed to write config for restart: %v", err) - return - } - - if err := m.Start(); err != nil { - mlog.Log.Errorf("lightbringer: failed to restart: %v — giving up", err) - return + // Retry transient WriteConfig/Start failures within the backoff+maxRetries budget. m.done is refreshed only on successful Start, so don't wait on it after a failed start. + for { + retries++ + if maxRetries > 0 && retries > maxRetries { + mlog.Log.Errorf("lightbringer: exceeded %d restart attempts, giving up", maxRetries) + return + } + + mlog.Log.Warnf("lightbringer: not running; restart attempt %d scheduled after %s", retries, backoff) + select { + case <-stopCh: + mlog.Log.Infof("lightbringer: restart attempt %d cancelled", retries) + return + case <-time.After(backoff): + } + + // Exponential backoff for the next attempt. + backoff = backoff * 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + + if m.stopping.Load() { + return + } + + if _, err := m.WriteConfig(); err != nil { + mlog.Log.Errorf("lightbringer: failed to write config for restart (attempt %d): %v", retries, err) + continue // transient — retry within budget + } + if err := m.Start(); err != nil { + mlog.Log.Errorf("lightbringer: failed to restart (attempt %d): %v", retries, err) + continue // transient — retry within budget + } + + lastStartAt = time.Now() + mlog.Log.Infof("lightbringer: restarted successfully (attempt %d)", retries) + break // success } - mlog.Log.Infof("lightbringer: restarted successfully (attempt %d)", retries) - continue // re-fetch new done channel at top of loop } } diff --git a/pkg/lightbringer/manager_test.go b/pkg/lightbringer/manager_test.go index 5908e445c..cf0ec5071 100644 --- a/pkg/lightbringer/manager_test.go +++ b/pkg/lightbringer/manager_test.go @@ -2,13 +2,19 @@ package lightbringer import ( "bytes" + "context" "os" "path/filepath" "testing" "time" + "github.com/Overclock-Validator/mithril/pkg/overcast" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) // createFakeBinary creates a simple shell script that acts as a fake Lightbringer process. @@ -88,26 +94,22 @@ func TestManager_StartStop(t *testing.T) { LogWriter: &logBuf, }) - // Write config first _, err := mgr.WriteConfig() require.NoError(t, err) - // Start err = mgr.Start() require.NoError(t, err) assert.True(t, mgr.IsRunning()) assert.Greater(t, mgr.Pid(), 0) - // Give it a moment to emit startup message + // wait for startup line time.Sleep(200 * time.Millisecond) - // Stop err = mgr.Stop(5 * time.Second) require.NoError(t, err) assert.False(t, mgr.IsRunning()) assert.Equal(t, 0, mgr.Pid()) - // Verify log capture got the startup message assert.Contains(t, logBuf.String(), "fake lightbringer started") } @@ -150,7 +152,6 @@ func TestManager_DoubleStartFails(t *testing.T) { require.NoError(t, err) defer mgr.Stop(5 * time.Second) - // Second start should fail err = mgr.Start() assert.ErrorContains(t, err, "already running") } @@ -189,7 +190,6 @@ func TestManager_DoneChannelClosesOnExit(t *testing.T) { err = mgr.Start() require.NoError(t, err) - // Process exits quickly — done channel should close select { case <-mgr.Done(): // expected @@ -207,7 +207,6 @@ func TestManager_DoneBeforeStart(t *testing.T) { GrpcAddr: "127.0.0.1:3001", }) - // Done() before Start() returns nil done := mgr.Done() assert.Nil(t, done) } @@ -234,9 +233,94 @@ func TestManager_WaitReadyFailsWhenProcessDies(t *testing.T) { err = mgr.Start() require.NoError(t, err) - // Process exits immediately — WaitReady should detect and fail time.Sleep(200 * time.Millisecond) // let it exit err = mgr.WaitReady(3 * time.Second) assert.ErrorContains(t, err, "process exited before becoming ready") } + +func TestManager_WaitReadyContextCancels(t *testing.T) { + dir := t.TempDir() + binaryPath := createFakeBinary(t, dir) + + mgr := NewManager(ManagerConfig{ + BinaryPath: binaryPath, + ConfigDir: dir, + GrpcAddr: "127.0.0.1:39999", + TOML: LightbringerTOML{ + GossipEntrypoint: "1.2.3.4:8000", + Storage: filepath.Join(dir, "shreds"), + RpcAddr: "127.0.0.1:39998", + GrpcAddr: "127.0.0.1:39999", + }, + }) + + _, err := mgr.WriteConfig() + require.NoError(t, err) + + err = mgr.Start() + require.NoError(t, err) + defer mgr.Stop(5 * time.Second) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err = mgr.WaitReadyContext(ctx, 30*time.Second) + require.ErrorIs(t, err, context.Canceled) +} + +type fakeSlotStreamClient struct { + stream grpc.ServerStreamingClient[overcast.SlotResponse] + err error +} + +func (f fakeSlotStreamClient) StreamSlots(ctx context.Context, in *overcast.SlotStreamRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[overcast.SlotResponse], error) { + return f.stream, f.err +} + +type fakeSlotStream struct { + ctx context.Context + resp *overcast.SlotResponse + recvErr error +} + +func (f fakeSlotStream) Recv() (*overcast.SlotResponse, error) { + if f.resp != nil { + return f.resp, nil + } + return nil, f.recvErr +} + +func (f fakeSlotStream) Header() (metadata.MD, error) { return nil, nil } +func (f fakeSlotStream) Trailer() metadata.MD { return nil } +func (f fakeSlotStream) CloseSend() error { return nil } +func (f fakeSlotStream) Context() context.Context { + if f.ctx != nil { + return f.ctx + } + return context.Background() +} +func (f fakeSlotStream) SendMsg(any) error { return nil } +func (f fakeSlotStream) RecvMsg(any) error { return f.recvErr } + +func TestProbeSlotStreamClientRejectsWrongGRPCService(t *testing.T) { + err := probeSlotStreamClient(context.Background(), fakeSlotStreamClient{ + stream: fakeSlotStream{recvErr: status.Error(codes.Unimplemented, "unknown service")}, + }) + require.Error(t, err) + assert.Equal(t, codes.Unimplemented, status.Code(err)) +} + +func TestProbeSlotStreamClientDeadlineAfterAcceptedMethodIsReady(t *testing.T) { + err := probeSlotStreamClient(context.Background(), fakeSlotStreamClient{ + stream: fakeSlotStream{recvErr: status.Error(codes.DeadlineExceeded, "no slots yet")}, + }) + require.NoError(t, err) +} + +func TestProbeSlotStreamClientResponseIsReady(t *testing.T) { + err := probeSlotStreamClient(context.Background(), fakeSlotStreamClient{ + stream: fakeSlotStream{resp: &overcast.SlotResponse{}}, + }) + require.NoError(t, err) +} diff --git a/pkg/mlog/mlog.go b/pkg/mlog/mlog.go index 21018aad0..c801cd603 100644 --- a/pkg/mlog/mlog.go +++ b/pkg/mlog/mlog.go @@ -150,12 +150,9 @@ func Initialize(cfg LogConfig, runID string) error { Log.writer = nil } - // Create symlink to latest run directory - symlinkPath := filepath.Join(cfg.Dir, "latest") - os.Remove(symlinkPath) // Ignore error if doesn't exist - if err := os.Symlink(runDirName, symlinkPath); err != nil { - // Non-fatal, just log to stdout - fmt.Fprintf(os.Stderr, "warning: failed to create symlink %s: %v\n", symlinkPath, err) + // Point "latest" at this run dir (atomic; non-fatal on failure). + if err := swapLatestSymlink(cfg.Dir, runDirName, shortRunID); err != nil { + fmt.Fprintf(os.Stderr, "warning: %v\n", err) } // Append to runs.log at the base directory level (tracks all runs) @@ -165,12 +162,29 @@ func Initialize(cfg LogConfig, runID string) error { // Start background flush goroutine Log.stopCh = make(chan struct{}) Log.wg.Add(1) - go Log.flushLoop() + go Log.flushLoop(Log.stopCh) Log.initialized = true return nil } +// swapLatestSymlink atomically repoints baseDir/latest via Symlink(tmp)+Rename +// (atomic on POSIX). Failure is non-fatal. +func swapLatestSymlink(baseDir, runDirName, shortRunID string) error { + latestPath := filepath.Join(baseDir, "latest") + tmpName := latestPath + ".tmp." + shortRunID + _ = os.Remove(tmpName) // best-effort: prior crash may have left it + + if err := os.Symlink(runDirName, tmpName); err != nil { + return fmt.Errorf("create temp symlink %s: %w", tmpName, err) + } + if err := os.Rename(tmpName, latestPath); err != nil { + _ = os.Remove(tmpName) + return fmt.Errorf("rename %s -> %s: %w", tmpName, latestPath, err) + } + return nil +} + // appendRunsLogEntry appends an entry to the runs.log file func appendRunsLogEntry(runsLogPath string, ts time.Time, runID, commit, runDir string) { f, err := os.OpenFile(runsLogPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) @@ -187,11 +201,8 @@ func appendRunsLogEntry(runsLogPath string, ts time.Time, runID, commit, runDir } } -// CreateSubprocessWriter returns a writer for a named subprocess (e.g. "lightbringer") -// that routes output to a dedicated log file in the current run directory. Subprocess -// output is not mirrored to the terminal, keeping Mithril's own output readable. -// When file logging is not initialized, output falls back to stderr with a -// "[name] " prefix so developers running without a log directory can still see it. +// CreateSubprocessWriter routes a subprocess's output to its own log file in the +// run directory (not the terminal). Falls back to prefixed stderr when no run dir. func (l *logger) CreateSubprocessWriter(name string) io.Writer { l.mu.Lock() defer l.mu.Unlock() @@ -210,8 +221,7 @@ func (l *logger) CreateSubprocessWriter(name string) io.Writer { } } -// prefixWriter wraps an io.Writer and prepends a prefix to each line. -// Used only as a fallback when subprocess file logging is not available. +// prefixWriter prepends a prefix to each line. Fallback for subprocess stderr. type prefixWriter struct { w io.Writer prefix string @@ -224,7 +234,7 @@ func newPrefixWriter(w io.Writer, prefix string) *prefixWriter { func (pw *prefixWriter) Write(p []byte) (int, error) { lines := bytes.Split(p, []byte("\n")) for i, line := range lines { - // Skip the trailing empty element produced by bytes.Split when input ends in '\n'. + // Skip trailing empty element from a final '\n'. if len(line) == 0 && i == len(lines)-1 { continue } @@ -236,7 +246,7 @@ func (pw *prefixWriter) Write(p []byte) (int, error) { } // flushLoop periodically flushes the buffer and syncs to disk -func (l *logger) flushLoop() { +func (l *logger) flushLoop(stopCh chan struct{}) { defer l.wg.Done() flushTicker := time.NewTicker(2 * time.Second) @@ -246,7 +256,7 @@ func (l *logger) flushLoop() { for { select { - case <-l.stopCh: + case <-stopCh: return case <-flushTicker.C: l.flush() @@ -256,9 +266,23 @@ func (l *logger) flushLoop() { } } +// flushLocked drains and writes the buffer; caller must hold fileMu. +// Lock order: fileMu (outer) -> mu (inner), never reversed. +func (l *logger) flushLocked() { + pending := l.drainPending() + if len(pending) == 0 || l.fileWriter == nil { + return + } + if _, err := l.fileWriter.Write(pending); err != nil { + fmt.Fprintf(os.Stderr, "warning: failed to write mithril log: %v\n", err) + } +} + // flush flushes the buffer without syncing func (l *logger) flush() { - l.writePending(l.drainPending()) + l.fileMu.Lock() + defer l.fileMu.Unlock() + l.flushLocked() } // Flush flushes the log buffer to ensure all pending messages are written. @@ -269,7 +293,9 @@ func Flush() { // flushAndSync flushes and syncs to disk func (l *logger) flushAndSync() { - l.writePending(l.drainPending()) + l.fileMu.Lock() + defer l.fileMu.Unlock() + l.flushLocked() // lumberjack doesn't expose Sync, but draining the in-memory buffer is // sufficient for most cases. } @@ -292,9 +318,10 @@ func Shutdown() { Log.wg.Wait() } - Log.writePending(Log.drainPending()) - + // Drain+write+close under fileMu so a concurrent flush can't write after + // (or interleave with) the final drain. Log.fileMu.Lock() + Log.flushLocked() if Log.fileWriter != nil { Log.fileWriter.Close() } @@ -345,7 +372,7 @@ func SaveRunConfig(configContent []byte) error { } configPath := filepath.Join(runDir, "config.toml") - if err := os.WriteFile(configPath, configContent, 0644); err != nil { + if err := os.WriteFile(configPath, configContent, 0600); err != nil { return fmt.Errorf("failed to save config to run directory: %w", err) } return nil @@ -476,17 +503,6 @@ func (l *logger) drainPendingLocked() []byte { return pending } -func (l *logger) writePending(pending []byte) { - if len(pending) == 0 || l.fileWriter == nil { - return - } - l.fileMu.Lock() - defer l.fileMu.Unlock() - if _, err := l.fileWriter.Write(pending); err != nil { - fmt.Fprintf(os.Stderr, "warning: failed to write mithril log: %v\n", err) - } -} - func (l *logger) Debugf(format string, args ...interface{}) { if l.level > LevelDebug && !l.enableVerbose.Load() { return @@ -533,7 +549,10 @@ func (l *logger) Warnf(format string, args ...interface{}) { func (l *logger) Errorf(format string, args ...interface{}) { msg := fmt.Sprintf("%sERROR: %s\n", relativePrefix(), fmt.Sprintf(format, args...)) - l.writeImmediate(msg) // Errors always flush immediately + l.writeImmediate(msg) + // Error lines often precede fatal returns or panics. Persist them now so a + // process-wide crash in another goroutine does not leave only stdout/stderr. + l.flush() } func (l *logger) EnableInfLogging() { diff --git a/pkg/mlog/mlog_test.go b/pkg/mlog/mlog_test.go new file mode 100644 index 000000000..836a3500e --- /dev/null +++ b/pkg/mlog/mlog_test.go @@ -0,0 +1,157 @@ +package mlog + +import ( + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fresh dir with no "latest" yet: swap creates one pointing at target. +func TestSwapLatestSymlink_CreatesNewSymlink(t *testing.T) { + dir := t.TempDir() + target := "run-20260518-abc123" + require.NoError(t, os.MkdirAll(filepath.Join(dir, target), 0755)) + + err := swapLatestSymlink(dir, target, "abc12345") + require.NoError(t, err) + + got, err := os.Readlink(filepath.Join(dir, "latest")) + require.NoError(t, err) + assert.Equal(t, target, got) +} + +// an existing "latest" symlink is replaced, not errored on. +func TestSwapLatestSymlink_ReplacesExistingSymlink(t *testing.T) { + dir := t.TempDir() + targetA := "run-A" + targetB := "run-B" + require.NoError(t, os.MkdirAll(filepath.Join(dir, targetA), 0755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, targetB), 0755)) + + require.NoError(t, swapLatestSymlink(dir, targetA, "aaaa1111")) + require.NoError(t, swapLatestSymlink(dir, targetB, "bbbb2222")) + + got, err := os.Readlink(filepath.Join(dir, "latest")) + require.NoError(t, err) + assert.Equal(t, targetB, got, "second swap should win") +} + +func TestErrorfFlushesFileBeforeShutdown(t *testing.T) { + Shutdown() + t.Cleanup(Shutdown) + + dir := t.TempDir() + require.NoError(t, Initialize(LogConfig{ + Dir: dir, + Level: "debug", + ToStdout: false, + MaxSizeMB: 100, + MaxAgeDays: 1, + MaxBackups: 1, + }, "run123456")) + + Log.Errorf("panic marker before process abort") + + body, err := os.ReadFile(GetLogPath()) + require.NoError(t, err) + assert.Contains(t, string(body), "panic marker before process abort") +} + +// the .tmp. sentinel is gone after a successful swap (renamed away). +func TestSwapLatestSymlink_CleansUpTempFile(t *testing.T) { + dir := t.TempDir() + target := "run-X" + shortRunID := "xxxx9999" + require.NoError(t, os.MkdirAll(filepath.Join(dir, target), 0755)) + + require.NoError(t, swapLatestSymlink(dir, target, shortRunID)) + + tmpPath := filepath.Join(dir, "latest.tmp."+shortRunID) + _, err := os.Lstat(tmpPath) + assert.True(t, os.IsNotExist(err), "tmp symlink should be renamed away (got err: %v)", err) +} + +// a leftover tmp from a prior crashed swap doesn't block a fresh swap. +func TestSwapLatestSymlink_RecoversFromStaleTmp(t *testing.T) { + dir := t.TempDir() + target := "run-Z" + shortRunID := "zzzz0000" + require.NoError(t, os.MkdirAll(filepath.Join(dir, target), 0755)) + // Simulate a prior crash that left the tmp around + require.NoError(t, os.Symlink("some-old-run", filepath.Join(dir, "latest.tmp."+shortRunID))) + + require.NoError(t, swapLatestSymlink(dir, target, shortRunID)) + + got, err := os.Readlink(filepath.Join(dir, "latest")) + require.NoError(t, err) + assert.Equal(t, target, got) +} + +// Concurrent reader against rapid swaps: the atomic-rename swap must never expose +// ENOENT. On Linux otherErrs must be zero; macOS has a tolerated EINVAL window. +func TestSwapLatestSymlink_NoEnoentWindow(t *testing.T) { + if testing.Short() { + t.Skip("concurrent stress test; skipped in -short mode") + } + + dir := t.TempDir() + targets := []string{"run-A", "run-B", "run-C", "run-D"} + for _, target := range targets { + require.NoError(t, os.MkdirAll(filepath.Join(dir, target), 0755)) + } + // Seed the symlink so the very first read in the loop has something. + require.NoError(t, swapLatestSymlink(dir, targets[0], "seed0000")) + + var ( + stop atomic.Bool + wg sync.WaitGroup + enoent atomic.Int64 + otherErrs atomic.Int64 + reads atomic.Int64 + ) + + // Reader: tight loop reading the symlink. + wg.Add(1) + go func() { + defer wg.Done() + for !stop.Load() { + reads.Add(1) + _, err := os.Readlink(filepath.Join(dir, "latest")) + if err == nil { + continue + } + if os.IsNotExist(err) { + enoent.Add(1) + } else { + otherErrs.Add(1) + } + } + }() + + // Writer: rapid swaps with distinct runIDs to exercise the tmp path. + for i := 0; i < 500; i++ { + require.NoError(t, swapLatestSymlink(dir, targets[i%len(targets)], + "run"+string(rune('0'+i%10))+"000000")) + } + time.Sleep(20 * time.Millisecond) // let reader sample post-last-swap + stop.Store(true) + wg.Wait() + + t.Logf("reads=%d enoent=%d other=%d", reads.Load(), enoent.Load(), otherErrs.Load()) + assert.Equal(t, int64(0), enoent.Load(), + "atomic-rename swap must never expose ENOENT to concurrent readers") + if runtime.GOOS == "darwin" { + // macOS rename-over-symlink briefly exposes a non-symlink (EINVAL); tolerated. + t.Logf("darwin: tolerated %d transient non-ENOENT reads (rename-over-symlink quirk)", otherErrs.Load()) + } else { + assert.Equal(t, int64(0), otherErrs.Load(), + "atomic-rename swap must not expose any broken (non-symlink) reads") + } +} diff --git a/pkg/procctl/atomicwrite.go b/pkg/procctl/atomicwrite.go new file mode 100644 index 000000000..48e42bb9c --- /dev/null +++ b/pkg/procctl/atomicwrite.go @@ -0,0 +1,44 @@ +package procctl + +import ( + "fmt" + "os" + "path/filepath" +) + +// atomicWriteFile writes data via tmp+rename so readers never see partial +// bytes. Chmod runs before rename so perm holds regardless of umask. +func atomicWriteFile(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + + tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp.*") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + tmpPath := tmp.Name() + + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("chmod temp: %w", err) + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("write temp: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpPath) + return fmt.Errorf("sync temp: %w", err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("close temp: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("rename temp: %w", err) + } + return nil +} diff --git a/pkg/procctl/audit.go b/pkg/procctl/audit.go new file mode 100644 index 000000000..f5dc8c694 --- /dev/null +++ b/pkg/procctl/audit.go @@ -0,0 +1,96 @@ +package procctl + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode" +) + +// AuditEntry is one row of the control audit log. Extra values are truncated at +// write time so a line stays under PIPE_BUF (4 KiB) for atomic-append. +type AuditEntry struct { + Action string // "START" | "STOP" | "FORCE_STOP" | "RESTART" + Result string // e.g. "signal_sent" | "signal_failed" | "refused" | "killed" | "ok" | "failed" | "timeout" + Pid int // affected mithril PID (0 if not applicable) + Extra map[string]string // arbitrary key=value context +} + +// maxExtraValueLen caps each Extra value so a long embedded error can't +// push the line past the 4 KiB atomic-append window. +const maxExtraValueLen = 256 + +// AppendAudit writes one line, creating dir (0700) and file (0600) as needed. +// Each call is a single atomic O_APPEND write of <4 KiB; errors are non-fatal. +func AppendAudit(path string, entry AuditEntry) error { + if err := EnsurePidDir(filepath.Dir(path)); err != nil { + return err + } + line := formatAuditLine(time.Now().UTC(), entry, os.Getpid()) + + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("open audit log %s: %w", path, err) + } + defer f.Close() + if _, err := f.WriteString(line + "\n"); err != nil { + return fmt.Errorf("append audit log %s: %w", path, err) + } + return nil +} + +// sanitizeAuditField strips control chars so an untrusted value (run_id, OS +// error string) can't smuggle a newline and forge or split audit lines. +func sanitizeAuditField(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, s) +} + +// auditValue prepares an Extra value for the key=value format. Quoted only if it +// contains space/'='/'"'; plain values stay bare so they're greppable. +func auditValue(s string) string { + s = sanitizeAuditField(s) + if s == "" || strings.ContainsAny(s, " =\"") { + return strconv.Quote(s) + } + return s +} + +// formatAuditLine renders the line without I/O. Pulled out for testability. +func formatAuditLine(ts time.Time, entry AuditEntry, dashboardPid int) string { + var b strings.Builder + b.WriteString(ts.Format(time.RFC3339Nano)) + b.WriteString(" action=") + b.WriteString(sanitizeAuditField(entry.Action)) + b.WriteString(" result=") + b.WriteString(sanitizeAuditField(entry.Result)) + fmt.Fprintf(&b, " dashboard_pid=%d", dashboardPid) + if entry.Pid != 0 { + fmt.Fprintf(&b, " pid=%d", entry.Pid) + } + // Sorted for deterministic output. + if len(entry.Extra) > 0 { + keys := make([]string, 0, len(entry.Extra)) + for k := range entry.Extra { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := sanitizeAuditField(entry.Extra[k]) + if len(v) > maxExtraValueLen { + // ToValidUTF8 drops the partial rune left by the byte cut. + v = strings.ToValidUTF8(v[:maxExtraValueLen], "") + "...(truncated)" + } + fmt.Fprintf(&b, " %s=%s", sanitizeAuditField(k), auditValue(v)) + } + } + return b.String() +} diff --git a/pkg/procctl/audit_test.go b/pkg/procctl/audit_test.go new file mode 100644 index 000000000..dee6f6157 --- /dev/null +++ b/pkg/procctl/audit_test.go @@ -0,0 +1,168 @@ +package procctl + +import ( + "bufio" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Audit log — append-only, greppable key=value forensic record (one line per action). + +// Line format: RFC3339Nano timestamp + core fields + sorted key=value extras. +func TestAppendAudit_BasicLine(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "control.audit") + + entry := AuditEntry{ + Action: "START", + Result: "ok", + Pid: 12345, + Extra: map[string]string{"run_id": "abc123"}, + } + require.NoError(t, AppendAudit(path, entry)) + + lines := readAuditLines(t, path) + require.Len(t, lines, 1) + line := lines[0] + assert.Contains(t, line, "action=START") + assert.Contains(t, line, "result=ok") + assert.Contains(t, line, "pid=12345") + assert.Contains(t, line, "run_id=abc123") + // Leading token must be an RFC3339Nano timestamp. + firstField := strings.SplitN(line, " ", 2)[0] + _, tErr := time.Parse(time.RFC3339Nano, firstField) + require.NoError(t, tErr, "audit line must begin with an RFC3339Nano timestamp, got %q", firstField) +} + +// Multiple calls append rather than overwrite. +func TestAppendAudit_AppendsAcrossCalls(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "control.audit") + + require.NoError(t, AppendAudit(path, AuditEntry{Action: "START", Result: "ok"})) + require.NoError(t, AppendAudit(path, AuditEntry{Action: "STOP", Result: "ok"})) + require.NoError(t, AppendAudit(path, AuditEntry{Action: "FORCE_STOP", Result: "ok"})) + + lines := readAuditLines(t, path) + assert.Len(t, lines, 3) + assert.Contains(t, lines[0], "action=START") + assert.Contains(t, lines[1], "action=STOP") + assert.Contains(t, lines[2], "action=FORCE_STOP") +} + +// File is 0600 — audit holds usernames and TTY identifiers. +func TestAppendAudit_Permissions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "control.audit") + require.NoError(t, AppendAudit(path, AuditEntry{Action: "START", Result: "ok"})) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +// Auditing to a path with no parent dir auto-creates it (0700). +func TestAppendAudit_CreatesParentDir(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "nested", "control.audit") + require.NoError(t, AppendAudit(path, AuditEntry{Action: "START", Result: "ok"})) + _, err := os.Stat(path) + assert.NoError(t, err) +} + +// O_APPEND keeps concurrent short-line writes from interleaving. +func TestAppendAudit_ConcurrentWrites(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "control.audit") + + const writers = 8 + const perWriter = 50 + + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := 0; i < perWriter; i++ { + _ = AppendAudit(path, AuditEntry{ + Action: "START", + Result: "ok", + Extra: map[string]string{"writer": itoaTest(id), "seq": itoaTest(i)}, + }) + } + }(w) + } + wg.Wait() + + lines := readAuditLines(t, path) + assert.Len(t, lines, writers*perWriter, "every Append must produce one line") + for _, line := range lines { + assert.Contains(t, line, "writer=") + assert.Contains(t, line, "seq=") + } +} + +// Extra keys are emitted in sorted order (deterministic output). +func TestAppendAudit_ExtraKeysAreSorted(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "control.audit") + require.NoError(t, AppendAudit(path, AuditEntry{ + Action: "START", + Result: "ok", + Extra: map[string]string{ + "zebra": "1", + "alpha": "2", + "mango": "3", + }, + })) + + lines := readAuditLines(t, path) + require.Len(t, lines, 1) + // alpha < mango < zebra. + a := strings.Index(lines[0], "alpha=") + m := strings.Index(lines[0], "mango=") + z := strings.Index(lines[0], "zebra=") + assert.True(t, a > 0 && a < m && m < z, + "extras should be sorted alphabetically: %s", lines[0]) +} + +func readAuditLines(t *testing.T, path string) []string { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + var out []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + out = append(out, scanner.Text()) + } + require.NoError(t, scanner.Err()) + return out +} + +// itoaTest: minimal int-to-string for test extras. +func itoaTest(v int) string { + if v == 0 { + return "0" + } + neg := v < 0 + if neg { + v = -v + } + buf := make([]byte, 0, 12) + for v > 0 { + buf = append([]byte{byte('0' + v%10)}, buf...) + v /= 10 + } + if neg { + buf = append([]byte{'-'}, buf...) + } + return string(buf) +} diff --git a/pkg/procctl/control.go b/pkg/procctl/control.go new file mode 100644 index 000000000..ca9e46fb4 --- /dev/null +++ b/pkg/procctl/control.go @@ -0,0 +1,237 @@ +package procctl + +import ( + "errors" + "fmt" + "os" + "sync" + "syscall" + "time" +) + +// ErrStopTimeout is returned by WaitStopped when the SIGTERM grace period +// elapses without the target exiting. Callers can offer a Force Stop option. +var ErrStopTimeout = errors.New("stop timed out - process still running") + +// SpawnedByEnv tells a spawned `mithril run` who launched it; the child +// records it in mithril.pid so dashboards can tell apart their own launches. +const SpawnedByEnv = "MITHRIL_SPAWNED_BY" + +// RunOpts is the input to AcquireForRun. +type RunOpts struct { + PidPath string // mithril.pid path (typically DefaultPidFile()) + LockPath string // mithril.lock path (typically DefaultLockFile()) + RunID string // unique run identifier (e.g., from replay.GenerateRunID) + BinaryPath string // absolute path of the running binary (os.Executable()) + ConfigPath string // absolute path of the config in use (may be empty) + LogDir string // mlog run directory (for dashboard log-tail) + SpawnedBy string // "dashboard" | "external" | "cli" +} + +// RunHandle is owned by a started mithril child; Close releases the lock and +// removes the PID file (sync.Once). os.Exit paths must remove it explicitly. +type RunHandle struct { + lock *LockHandle + pidPath string + pidInfo *PidInfo + closeOnce sync.Once + closeErr error +} + +// AcquireForRun takes the single-instance lock, captures identity, and writes the +// PID file. Returns ErrLocked if another mithril runs; Close the handle at exit. +func AcquireForRun(opts RunOpts) (*RunHandle, error) { + lh, err := AcquireLock(opts.LockPath) + if err != nil { + return nil, err + } + + id, err := ReadIdentity(os.Getpid()) + if err != nil { + _ = lh.Release() + return nil, fmt.Errorf("read own identity: %w", err) + } + + info := &PidInfo{ + Pid: os.Getpid(), + StartTimeTicks: id.StartTimeTicks, + ExeInode: id.ExeInode, + BinaryPath: opts.BinaryPath, + RunID: opts.RunID, + ConfigPath: opts.ConfigPath, + LogDir: opts.LogDir, + SpawnedBy: opts.SpawnedBy, + } + if err := WritePidFile(opts.PidPath, info); err != nil { + _ = lh.Release() + return nil, fmt.Errorf("write pid file: %w", err) + } + + return &RunHandle{lock: lh, pidPath: opts.PidPath, pidInfo: info}, nil +} + +// Close releases the lock and removes the PID file. Safe to call repeatedly +// and concurrently; the first call's error is returned by every call. +func (h *RunHandle) Close() error { + if h == nil { + return nil + } + h.closeOnce.Do(func() { + if err := RemovePidFileIfMatches(h.pidPath, h.pidInfo); err != nil { + h.closeErr = err + } + if err := h.lock.Release(); err != nil && h.closeErr == nil { + h.closeErr = err + } + }) + return h.closeErr +} + +// ── Dashboard-side control surface ── + +// SignalStop re-verifies identity, stamps stop_in_progress_by, and sends SIGTERM +// (does not wait — use WaitStopped). Returns ErrPidFileNotFound if nothing to stop. +func SignalStop(pidPath, auditPath string, dashboardPid int) error { + info, err := ReadPidFile(pidPath) + if err != nil { + return err + } + signalHandle, err := OpenSignalHandle(info.Pid) + if err != nil { + // Already gone — report as not-running, not a hard failure. + if errors.Is(err, ErrProcessNotFound) { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "already_exited", + Pid: info.Pid, + }) + return ErrProcessNotFound + } + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "signal_failed", + Pid: info.Pid, + Extra: map[string]string{"error": err.Error()}, + }) + return fmt.Errorf("open signal handle for pid %d: %w", info.Pid, err) + } + defer signalHandle.Close() + + // If the PID was reused by an unrelated process, refuse to signal it. + if ok, reason := Matches(info.Pid, info); !ok { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "refused", + Pid: info.Pid, + Extra: map[string]string{"reason": reason}, + }) + return fmt.Errorf("refuse to signal pid %d: %s", info.Pid, reason) + } + + // Best-effort UI hint; failure here doesn't block the signal. Last-writer + // wins between concurrent dashboards is fine (it's informational). + _ = updateStopInProgressIfMatches(pidPath, dashboardPid, info) + + if err := signalHandle.Signal(syscall.SIGTERM); err != nil { + // Process exited between the identity check and the signal — benign. + if errors.Is(err, syscall.ESRCH) { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "already_exited", + Pid: info.Pid, + }) + return ErrProcessNotFound + } + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "signal_failed", + Pid: info.Pid, + Extra: map[string]string{"error": err.Error()}, + }) + return fmt.Errorf("send SIGTERM to pid %d: %w", info.Pid, err) + } + + _ = AppendAudit(auditPath, AuditEntry{ + Action: "STOP", + Result: "signal_sent", + Pid: info.Pid, + Extra: map[string]string{"run_id": info.RunID}, + }) + return nil +} + +// WaitStopped polls Detect each second until the process stops or timeout +// (ErrStopTimeout). Never auto-escalate to ForceKill — risks corruption. +func WaitStopped(pidPath, lockPath, accountsDir string, timeout time.Duration) (*Detection, error) { + const pollInterval = 1 * time.Second + deadline := time.Now().Add(timeout) + var last *Detection + for { + det, err := Detect(pidPath, lockPath, accountsDir) + if err != nil { + return last, err + } + last = det + if det.Status != StatusRunning { + return det, nil + } + if time.Now().After(deadline) { + return det, ErrStopTimeout + } + time.Sleep(pollInterval) + } +} + +// ForceKill delivers SIGKILL after re-verifying identity (skips shutdown defers). +// Race-free via pidfd on Linux >=5.3; kill(2) fallback has a tiny TOCTOU window. +func ForceKill(pidPath, auditPath string, dashboardPid int) error { + info, err := ReadPidFile(pidPath) + if err != nil { + return err + } + signalHandle, err := OpenSignalHandle(info.Pid) + if err != nil { + // Already gone — not a failure, and no rebuild is needed. + if errors.Is(err, ErrProcessNotFound) { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "FORCE_STOP", + Result: "already_exited", + Pid: info.Pid, + }) + return ErrProcessNotFound + } + _ = AppendAudit(auditPath, AuditEntry{ + Action: "FORCE_STOP", + Result: "signal_failed", + Pid: info.Pid, + Extra: map[string]string{"error": err.Error(), "warning": "accountsdb-rebuild-recommended"}, + }) + return fmt.Errorf("open signal handle for pid %d: %w", info.Pid, err) + } + defer signalHandle.Close() + if ok, reason := Matches(info.Pid, info); !ok { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "FORCE_STOP", + Result: "refused", + Pid: info.Pid, + Extra: map[string]string{"reason": reason}, + }) + return fmt.Errorf("refuse to SIGKILL pid %d: %s", info.Pid, reason) + } + if err := signalHandle.Signal(syscall.SIGKILL); err != nil { + _ = AppendAudit(auditPath, AuditEntry{ + Action: "FORCE_STOP", + Result: "signal_failed", + Pid: info.Pid, + Extra: map[string]string{"error": err.Error(), "warning": "accountsdb-rebuild-recommended"}, + }) + return fmt.Errorf("SIGKILL pid %d: %w", info.Pid, err) + } + _ = AppendAudit(auditPath, AuditEntry{ + Action: "FORCE_STOP", + Result: "killed", + Pid: info.Pid, + Extra: map[string]string{"run_id": info.RunID, "warning": "accountsdb-rebuild-recommended"}, + }) + return nil +} diff --git a/pkg/procctl/control_test.go b/pkg/procctl/control_test.go new file mode 100644 index 000000000..4481ab038 --- /dev/null +++ b/pkg/procctl/control_test.go @@ -0,0 +1,306 @@ +package procctl + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// startStub launches a throwaway child in its own process group, reaped by one +// Wait() goroutine. Cleanup SIGKILLs the group; the channel closes on exit. +func startStub(t *testing.T, args ...string) (*exec.Cmd, <-chan struct{}) { + t.Helper() + cmd := exec.Command(args[0], args[1:]...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + require.NoError(t, cmd.Start()) + done := make(chan struct{}) + go func() { _ = cmd.Wait(); close(done) }() + t.Cleanup(func() { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-done + }) + return cmd, done +} + +// AcquireForRun (called by `mithril run` at startup) acquires the lock + writes +// the PID file, holding the lock until handle.Close. + +func TestAcquireForRun_HappyPath(t *testing.T) { + dir := t.TempDir() + opts := RunOpts{ + PidPath: filepath.Join(dir, "mithril.pid"), + LockPath: filepath.Join(dir, "mithril.lock"), + RunID: "test-run-1", + BinaryPath: "/usr/local/bin/mithril", + ConfigPath: "/etc/mithril.toml", + LogDir: "/var/log/mithril/test", + SpawnedBy: "test", + } + handle, err := AcquireForRun(opts) + require.NoError(t, err) + require.NotNil(t, handle) + defer handle.Close() + + info, err := ReadPidFile(opts.PidPath) + require.NoError(t, err) + assert.Equal(t, os.Getpid(), info.Pid) + assert.Equal(t, "test-run-1", info.RunID) + assert.Equal(t, "test", info.SpawnedBy) + assert.NotZero(t, info.StartTimeTicks) +} + +// Second AcquireForRun returns ErrLocked (another mithril already running). +func TestAcquireForRun_ContentionReturnsErrLocked(t *testing.T) { + dir := t.TempDir() + opts := RunOpts{ + PidPath: filepath.Join(dir, "mithril.pid"), + LockPath: filepath.Join(dir, "mithril.lock"), + RunID: "first", + } + first, err := AcquireForRun(opts) + require.NoError(t, err) + defer first.Close() + + second, err := AcquireForRun(opts) + assert.Nil(t, second) + assert.True(t, errors.Is(err, ErrLocked)) +} + +// Close removes the PID file (leftovers would false-positive on next start). +func TestAcquireForRun_CloseRemovesPidFile(t *testing.T) { + dir := t.TempDir() + opts := RunOpts{ + PidPath: filepath.Join(dir, "mithril.pid"), + LockPath: filepath.Join(dir, "mithril.lock"), + RunID: "to-be-closed", + } + handle, err := AcquireForRun(opts) + require.NoError(t, err) + require.NoError(t, handle.Close()) + + _, err = os.Stat(opts.PidPath) + assert.True(t, os.IsNotExist(err), "PID file should be gone after Close") +} + +// A stale handle's Close must not delete a newer run's PID file. +func TestAcquireForRun_CloseDoesNotRemoveDifferentPidFile(t *testing.T) { + dir := t.TempDir() + opts := RunOpts{ + PidPath: filepath.Join(dir, "mithril.pid"), + LockPath: filepath.Join(dir, "mithril.lock"), + RunID: "old-run", + } + handle, err := AcquireForRun(opts) + require.NoError(t, err) + + replacement := &PidInfo{ + Pid: os.Getpid(), + StartTimeTicks: handle.pidInfo.StartTimeTicks + 1, + ExeInode: handle.pidInfo.ExeInode, + RunID: "new-run", + } + require.NoError(t, WritePidFile(opts.PidPath, replacement)) + require.NoError(t, handle.Close()) + + info, err := ReadPidFile(opts.PidPath) + require.NoError(t, err) + assert.Equal(t, "new-run", info.RunID) +} + +// Double Close is safe (defer + explicit Close). +func TestAcquireForRun_DoubleCloseIsIdempotent(t *testing.T) { + dir := t.TempDir() + opts := RunOpts{ + PidPath: filepath.Join(dir, "mithril.pid"), + LockPath: filepath.Join(dir, "mithril.lock"), + RunID: "double-close", + } + handle, err := AcquireForRun(opts) + require.NoError(t, err) + require.NoError(t, handle.Close()) + assert.NoError(t, handle.Close()) +} + +// SignalStop + WaitStopped — the dashboard's Stop button code path. + +// SignalStop delivers SIGTERM to the recorded process; it must exit. +func TestSignalStop_DeliversToRunningProcess(t *testing.T) { + cmd, done := startStub(t, "sh", "-c", "sleep 30") + time.Sleep(50 * time.Millisecond) + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + auditPath := filepath.Join(dir, "control.audit") + + id, err := ReadIdentity(cmd.Process.Pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: cmd.Process.Pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "stop-target", + })) + + require.NoError(t, SignalStop(pidPath, auditPath, os.Getpid())) + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("SignalStop did not deliver SIGTERM in time") + } +} + +// SignalStop stamps the dashboard PID so other dashboards see the in-flight stop. +func TestSignalStop_RecordsStopInProgress(t *testing.T) { + cmd, _ := startStub(t, "sh", "-c", "sleep 30") + time.Sleep(50 * time.Millisecond) + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + auditPath := filepath.Join(dir, "control.audit") + + id, err := ReadIdentity(cmd.Process.Pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: cmd.Process.Pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "stop-target", + })) + + dashboardPid := os.Getpid() + require.NoError(t, SignalStop(pidPath, auditPath, dashboardPid)) + + info, err := ReadPidFile(pidPath) + require.NoError(t, err) + assert.Equal(t, dashboardPid, info.StopInProgressBy) + assert.False(t, info.StopInProgressAt.IsZero()) +} + +// SignalStop on an absent PID file returns ErrPidFileNotFound. +func TestSignalStop_AbsentPidFile(t *testing.T) { + dir := t.TempDir() + err := SignalStop(filepath.Join(dir, "ghost.pid"), filepath.Join(dir, "audit"), os.Getpid()) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrPidFileNotFound)) +} + +// WaitStopped observes Status != Running once the process exits. +func TestWaitStopped_DetectsExit(t *testing.T) { + cmd, reapDone := startStub(t, "sh", "-c", "sleep 30") + pid := cmd.Process.Pid + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + accountsDir := dir + + id, err := ReadIdentity(pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "wait-target", + })) + + // SIGTERM out of band so WaitStopped only observes exit; startStub's reap + // clears the PID that WaitStopped polls. + require.NoError(t, syscall.Kill(pid, syscall.SIGTERM)) + + det, err := WaitStopped(pidPath, lockPath, accountsDir, 5*time.Second) + require.NoError(t, err) + assert.NotEqual(t, StatusRunning, det.Status) + <-reapDone +} + +// On timeout WaitStopped returns the still-Running Detection plus ErrStopTimeout. +func TestWaitStopped_TimeoutReturnsDetection(t *testing.T) { + cmd, _ := startStub(t, "sh", "-c", "trap '' TERM; sleep 30") // ignore SIGTERM + time.Sleep(100 * time.Millisecond) + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + accountsDir := dir + + id, err := ReadIdentity(cmd.Process.Pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: cmd.Process.Pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "stuck", + })) + + det, err := WaitStopped(pidPath, lockPath, accountsDir, 500*time.Millisecond) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrStopTimeout)) + require.NotNil(t, det) + assert.Equal(t, StatusRunning, det.Status) +} + +// ForceKill — last-resort, user-confirmed. + +// ForceKill kills a SIGTERM-ignoring process. +func TestForceKill_KillsStubbornProcess(t *testing.T) { + cmd, done := startStub(t, "sh", "-c", "trap '' TERM; sleep 30") + pid := cmd.Process.Pid + time.Sleep(100 * time.Millisecond) + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + auditPath := filepath.Join(dir, "control.audit") + + id, err := ReadIdentity(pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "kill-target", + })) + + require.NoError(t, ForceKill(pidPath, auditPath, os.Getpid())) + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("ForceKill did not deliver SIGKILL") + } +} + +// ForceKill's audit entry carries a warning field flagging the corruption risk. +func TestForceKill_RecordsAuditWarning(t *testing.T) { + cmd, _ := startStub(t, "sh", "-c", "sleep 30") + pid := cmd.Process.Pid + time.Sleep(50 * time.Millisecond) + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + auditPath := filepath.Join(dir, "control.audit") + + id, err := ReadIdentity(pid) + require.NoError(t, err) + require.NoError(t, WritePidFile(pidPath, &PidInfo{ + Pid: pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + RunID: "kill-audit", + })) + + require.NoError(t, ForceKill(pidPath, auditPath, os.Getpid())) + + data, err := os.ReadFile(auditPath) + require.NoError(t, err) + body := string(data) + assert.Contains(t, body, "action=FORCE_STOP") + assert.Contains(t, body, "warning=") +} diff --git a/pkg/procctl/detect.go b/pkg/procctl/detect.go new file mode 100644 index 000000000..54399eafe --- /dev/null +++ b/pkg/procctl/detect.go @@ -0,0 +1,160 @@ +package procctl + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Overclock-Validator/mithril/pkg/state" +) + +// Status is the high-level result of Detect — OS truth only. The dashboard +// layers its own transient "starting / stopping / stuck" states on top. +type Status int + +const ( + // StatusStopped — no mithril running and the last shutdown was clean + // (or no state file exists). + StatusStopped Status = iota + + // StatusRunning — a process matching the PID file's identity is alive. + StatusRunning + + // StatusCrashed — no mithril running but the last session didn't record + // a clean shutdown; AccountsDB may need rebuild. + StatusCrashed +) + +func (s Status) String() string { + switch s { + case StatusStopped: + return "Stopped" + case StatusRunning: + return "Running" + case StatusCrashed: + return "Crashed" + default: + return fmt.Sprintf("Status(%d)", int(s)) + } +} + +// Detection is the full result of Detect. All fields are populated on +// success; zero values mean "unknown" or "not applicable to this status." +type Detection struct { + Status Status + Pid int + BinaryPath string + RunID string + SpawnedBy string + ConfigPath string + LogDir string + StdoutPath string + StderrPath string + LockHeld bool + + // Timing — only meaningful when Status == Running. + StartedAt time.Time + Uptime time.Duration + + // Post-mortem — only meaningful when Status ∈ {Stopped, Crashed}. + LastShutdownReason string + LastCleanExit bool + + // Stop-in-progress metadata. Cleared if the dashboard that set it is + // dead or the TTL has elapsed — only a live in-progress stop appears. + StopInProgressBy int + StopInProgressAt time.Time +} + +// stopInProgressTTL bounds how long after a SIGTERM the stop_in_progress_by +// field stays meaningful; beyond it the setting dashboard is presumed gone. +const stopInProgressTTL = 5 * time.Minute + +// Detect inspects the PID, lock, and state files to determine status (pure read). +// Empty accountsDbDir skips post-mortem classification; errors only on a corrupt PID file. +func Detect(pidPath, lockPath, accountsDbDir string) (*Detection, error) { + info, err := ReadPidFile(pidPath) + if err != nil && !errors.Is(err, ErrPidFileNotFound) { + // Corrupt PID file — surface it so the dashboard shows "investigate". + return nil, err + } + + // Branch 1: PID file exists. Check identity and lock. + if info != nil { + if ok, _ := Matches(info.Pid, info); ok { + lockHeld := false + if lockPath != "" { + lockHeld, _ = TestLock(lockPath) + } + det := &Detection{ + Status: StatusRunning, + Pid: info.Pid, + BinaryPath: info.BinaryPath, + RunID: info.RunID, + SpawnedBy: info.SpawnedBy, + ConfigPath: info.ConfigPath, + LogDir: info.LogDir, + StdoutPath: info.StdoutPath, + StderrPath: info.StderrPath, + LockHeld: lockHeld, + } + // StartedAt/Uptime left unset: the dashboard derives run start + // from the timestamped mlog runDir (LogDir) instead. + + // Carry stop-in-progress fields but drop stale ones, so a crashed + // dashboard doesn't permanently grey out Force Stop. + if info.StopInProgressBy != 0 && !info.isStopProgressStale(stopInProgressTTL) { + det.StopInProgressBy = info.StopInProgressBy + det.StopInProgressAt = info.StopInProgressAt + } + return det, nil + } + // PID file exists but the process is gone or its identity has + // changed. Fall through to the post-mortem branch. + } + + // Branch 2: No live process. Classify Stopped vs Crashed via state file. + det := &Detection{Status: StatusStopped} + if accountsDbDir == "" { + return det, nil + } + st, err := state.LoadState(accountsDbDir) + if err != nil { + det.Status = StatusCrashed + det.LastShutdownReason = fmt.Sprintf("state file unreadable: %v", err) + return det, nil + } + if st == nil && accountsDbArtifactsExist(accountsDbDir) { + det.Status = StatusCrashed + det.LastShutdownReason = "incomplete AccountsDB found (state file missing)" + return det, nil + } + clean, reason := state.WasCleanExit(st) + det.LastShutdownReason = reason + det.LastCleanExit = clean + if st != nil && !clean { + det.Status = StatusCrashed + } + return det, nil +} + +func accountsDbArtifactsExist(accountsDbDir string) bool { + // AccountsDB build artifacts that CleanAccountsDbDir removes — present with no + // state file, they mean an interrupted build. Stays a subset of that list: + // bank_hash and mithril_state.history.jsonl aren't removed, so they don't count. + for _, name := range []string{ + "mithril_db", + "bankhash_db", + "accounts", + "largest_file_id", + "manifest", + "mithril_db_log_shards", + } { + if _, err := os.Stat(filepath.Join(accountsDbDir, name)); err == nil { + return true + } + } + return false +} diff --git a/pkg/procctl/detect_test.go b/pkg/procctl/detect_test.go new file mode 100644 index 000000000..07ba9b5ca --- /dev/null +++ b/pkg/procctl/detect_test.go @@ -0,0 +1,254 @@ +package procctl + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Detect combines pidfile + lock + identity + WasCleanExit into one Status (Stopped|Running|Crashed). + +// spawnLiveChild starts a `sleep 30` child and returns its PID + a cleanup func. +func spawnLiveChild(t *testing.T) (int, func()) { + t.Helper() + cmd := exec.Command("sh", "-c", "sleep 30") + require.NoError(t, cmd.Start()) + cleanup := func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + time.Sleep(30 * time.Millisecond) // let it actually start + return cmd.Process.Pid, cleanup +} + +// pidInfoForPid builds a PidInfo matching a live PID by reading its identity. +func pidInfoForPid(t *testing.T, pid int) *PidInfo { + t.Helper() + id, err := ReadIdentity(pid) + require.NoError(t, err) + return &PidInfo{ + Pid: pid, + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + BinaryPath: id.ExePath, + RunID: "test-run-id", + SpawnedBy: "test", + } +} + +// No PID file, no lock → Stopped. +func TestDetect_NoPidFile_IsStopped(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + accountsDir := dir + + det, err := Detect(pidPath, lockPath, accountsDir) + require.NoError(t, err) + assert.Equal(t, StatusStopped, det.Status) + assert.Equal(t, 0, det.Pid) +} + +// Alive PID + matching PID file + held lock → Running. +func TestDetect_AlivePidAndLock_IsRunning(t *testing.T) { + pid, cleanup := spawnLiveChild(t) + defer cleanup() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + require.NoError(t, WritePidFile(pidPath, pidInfoForPid(t, pid))) + + lh, err := AcquireLock(lockPath) + require.NoError(t, err) + defer lh.Release() + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusRunning, det.Status) + assert.Equal(t, pid, det.Pid) + assert.Equal(t, "test-run-id", det.RunID) + assert.True(t, det.LockHeld, "lock is held by this process; Detect should report LockHeld") + // TODO: assert a real Uptime lower bound when populated. + assert.Zero(t, det.Uptime, "Uptime is not populated yet") +} + +// PID gone + no state file → Stopped (can't claim Crashed without evidence). +func TestDetect_PidFileButProcessGone_NoState_IsStopped(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + bogus := &PidInfo{ + Pid: 9999999, + StartTimeTicks: 1, + ExeInode: 1, + RunID: "ghost", + } + require.NoError(t, WritePidFile(pidPath, bogus)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusStopped, det.Status) +} + +// No PID file + state shows session start with no clean shutdown → Crashed. +func TestDetect_PidFileGone_StateSaysCrashed_IsCrashed(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + s := &state.MithrilState{ + StateSchemaVersion: state.CurrentStateSchemaVersion, + Stage: "ready", + SnapshotSlot: 100, + CurrentSessionStartedAt: time.Now().Add(-1 * time.Hour), + // LastShutdownAt is zero → no clean shutdown recorded + } + require.NoError(t, s.Save(dir)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusCrashed, det.Status) + assert.NotEmpty(t, det.LastShutdownReason, "Crashed status should surface a reason") +} + +// No PID file + state shows a clean shutdown → Stopped, LastCleanExit true. +func TestDetect_PidFileGone_StateSaysClean_IsStopped(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + start := time.Now().Add(-2 * time.Hour) + s := &state.MithrilState{ + StateSchemaVersion: state.CurrentStateSchemaVersion, + Stage: "ready", + SnapshotSlot: 100, + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(1 * time.Hour), + LastShutdownReason: state.ShutdownReasonNormal, + } + require.NoError(t, s.Save(dir)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusStopped, det.Status) + assert.True(t, det.LastCleanExit) +} + +func TestDetect_CorruptStateFile_IsCrashed(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, os.WriteFile(filepath.Join(dir, state.StateFileName), []byte("{"), 0600)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusCrashed, det.Status) + assert.Contains(t, det.LastShutdownReason, "state file unreadable") +} + +func TestDetect_MissingStateWithAccountsArtifacts_IsCrashed(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest"), []byte("partial bootstrap artifact"), 0600)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusCrashed, det.Status) + assert.Contains(t, det.LastShutdownReason, "incomplete AccountsDB") +} + +func TestDetect_CleanBuildingStateWithAccountsArtifacts_IsStopped(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest"), []byte("partial bootstrap artifact"), 0600)) + + start := time.Now().Add(-10 * time.Minute) + s := &state.MithrilState{ + StateSchemaVersion: state.CurrentStateSchemaVersion, + Stage: "building", + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(5 * time.Minute), + LastShutdownReason: state.ShutdownReasonNormal, + } + require.NoError(t, s.Save(dir)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusStopped, det.Status) + assert.True(t, det.LastCleanExit) +} + +// Corrupt PID file surfaces an error, not a status. +func TestDetect_CorruptPidFile_SurfacesError(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + require.NoError(t, os.WriteFile(pidPath, []byte("not json"), 0600)) + + _, err := Detect(pidPath, lockPath, dir) + require.Error(t, err) +} + +// stop_in_progress_by pointing at a dead dashboard PID is cleared, not propagated. +func TestDetect_StaleStopInProgress_NotShown(t *testing.T) { + pid, cleanup := spawnLiveChild(t) + defer cleanup() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + + info := pidInfoForPid(t, pid) + info.StopInProgressBy = 9999999 // dead dashboard pid + info.StopInProgressAt = time.Now() + require.NoError(t, WritePidFile(pidPath, info)) + + lh, err := AcquireLock(lockPath) + require.NoError(t, err) + defer lh.Release() + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusRunning, det.Status) + assert.Equal(t, 0, det.StopInProgressBy, "stale stop_in_progress_by must be cleared") +} + +// bank_hash and mithril_state.history.jsonl aren't removed by a clean rebuild, so a +// dir holding only those (no state file) is a cleanly-cleaned AccountsDB, not a crash. +func TestDetect_HistoryAndBankHashOnly_IsStopped(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, os.WriteFile(filepath.Join(dir, state.HistoryFileName), []byte("{}\n"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bank_hash"), []byte("x"), 0600)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusStopped, det.Status) +} + +// A leftover index-shard staging dir (removed by CleanAccountsDbDir) with no state +// file means a genuinely interrupted build. +func TestDetect_LogShardsArtifact_IsCrashed(t *testing.T) { + dir := t.TempDir() + pidPath := filepath.Join(dir, "mithril.pid") + lockPath := filepath.Join(dir, "mithril.lock") + require.NoError(t, os.Mkdir(filepath.Join(dir, "mithril_db_log_shards"), 0700)) + + det, err := Detect(pidPath, lockPath, dir) + require.NoError(t, err) + assert.Equal(t, StatusCrashed, det.Status) + assert.Contains(t, det.LastShutdownReason, "incomplete AccountsDB") +} diff --git a/pkg/procctl/identity.go b/pkg/procctl/identity.go new file mode 100644 index 000000000..d44abb90f --- /dev/null +++ b/pkg/procctl/identity.go @@ -0,0 +1,51 @@ +package procctl + +import ( + "errors" + "fmt" +) + +// ErrProcessNotFound indicates the target PID does not exist (or its +// identity source is unreadable in a way that implies absence). +var ErrProcessNotFound = errors.New("process not found") + +// Identity uniquely identifies a process across PID reuse via (ExeInode, +// StartTimeTicks); ExePath is informational. macOS uses start time alone. +type Identity struct { + Pid int + ExeInode uint64 // 0 on macOS + StartTimeTicks uint64 // USER_HZ clock ticks on Linux; Unix-nano on macOS + ExePath string // resolved binary path (best-effort) +} + +// Matches reports whether pid's live identity matches recorded info. Every signal +// path clears this gate before kill(2); pair with a pidfd on Linux >=5.3. +func Matches(pid int, info *PidInfo) (bool, string) { + if info == nil { + return false, "no recorded identity" + } + if info.Pid != 0 && info.Pid != pid { + return false, fmt.Sprintf("pid mismatch (recorded %d, checking %d)", info.Pid, pid) + } + if info.StartTimeTicks == 0 && info.ExeInode == 0 { + return false, "recorded identity is incomplete" + } + live, err := ReadIdentity(pid) + if err != nil { + if errors.Is(err, ErrProcessNotFound) { + return false, "process not running" + } + return false, fmt.Sprintf("cannot read identity: %v", err) + } + // Strongest, rename/delete-safe check; skipped on macOS (ExeInode zero). + if live.ExeInode != 0 && info.ExeInode != 0 && live.ExeInode != info.ExeInode { + return false, fmt.Sprintf("exe inode mismatch (recorded %d, live %d)", + info.ExeInode, live.ExeInode) + } + // Collision-proof against PID reuse: a new process gets a new starttime. + if info.StartTimeTicks != 0 && live.StartTimeTicks != info.StartTimeTicks { + return false, fmt.Sprintf("start time mismatch (recorded %d, live %d) — pid likely reused", + info.StartTimeTicks, live.StartTimeTicks) + } + return true, "" +} diff --git a/pkg/procctl/identity_darwin.go b/pkg/procctl/identity_darwin.go new file mode 100644 index 000000000..baed42f0b --- /dev/null +++ b/pkg/procctl/identity_darwin.go @@ -0,0 +1,53 @@ +//go:build darwin + +package procctl + +import ( + "errors" + "fmt" + "syscall" + + "golang.org/x/sys/unix" +) + +// ReadIdentity reads process identity on macOS via kern.proc.pid sysctl. ExeInode +// stays zero; StartTimeTicks is Unix-nano start time. Dev-only; production is Linux. +func ReadIdentity(pid int) (*Identity, error) { + // kill(0): ESRCH means gone. + if err := syscall.Kill(pid, 0); err != nil { + if errors.Is(err, syscall.ESRCH) { + return nil, ErrProcessNotFound + } + // EPERM means alive but unsignalable — still a valid identity. + if !errors.Is(err, syscall.EPERM) { + return nil, fmt.Errorf("kill(0) %d: %w", pid, err) + } + } + + kp, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil { + // A missing PID surfaces as EIO (sometimes ESRCH) here; normalize it. + if errors.Is(err, syscall.EIO) || errors.Is(err, syscall.ESRCH) { + return nil, ErrProcessNotFound + } + return nil, fmt.Errorf("sysctl kern.proc.pid %d: %w", pid, err) + } + if kp == nil || kp.Proc.P_pid == 0 { + return nil, ErrProcessNotFound + } + startNsec := kp.Proc.P_starttime.Nano() + if startNsec <= 0 { + return nil, fmt.Errorf("sysctl kern.proc.pid %d returned empty start time", pid) + } + + // ByteSliceToString truncates at the first NUL; P_comm's trailing bytes + // can hold stale kernel data that TrimRight would leave behind. + exePath := unix.ByteSliceToString(kp.Proc.P_comm[:]) + + return &Identity{ + Pid: pid, + ExeInode: 0, // unavailable on macOS + StartTimeTicks: uint64(startNsec), + ExePath: exePath, + }, nil +} diff --git a/pkg/procctl/identity_linux.go b/pkg/procctl/identity_linux.go new file mode 100644 index 000000000..8cf869909 --- /dev/null +++ b/pkg/procctl/identity_linux.go @@ -0,0 +1,87 @@ +//go:build linux + +package procctl + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + "syscall" +) + +// ReadIdentity reads identity from /proc: ExeInode from /proc//exe, start +// time from /proc//stat. ENOENT means reaped → ErrProcessNotFound. +func ReadIdentity(pid int) (*Identity, error) { + // Stat follows the symlink to the underlying inode. + exePath := fmt.Sprintf("/proc/%d/exe", pid) + fi, err := os.Stat(exePath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrProcessNotFound + } + return nil, fmt.Errorf("stat %s: %w", exePath, err) + } + stat, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return nil, errors.New("unexpected Sys() type — not a syscall.Stat_t") + } + inode := stat.Ino + + // Diagnostic only; strip the " (deleted)" suffix for unlinked binaries. + exeTarget, _ := os.Readlink(exePath) + exeTarget = strings.TrimSuffix(exeTarget, " (deleted)") + + startTime, err := readProcStatStartTime(pid) + if err != nil { + return nil, err + } + + return &Identity{ + Pid: pid, + ExeInode: inode, + StartTimeTicks: startTime, + ExePath: exeTarget, + }, nil +} + +// readProcStatStartTime reads /proc//stat and extracts starttime. +// Translates ENOENT to ErrProcessNotFound. +func readProcStatStartTime(pid int) (uint64, error) { + path := fmt.Sprintf("/proc/%d/stat", pid) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return 0, ErrProcessNotFound + } + return 0, fmt.Errorf("read %s: %w", path, err) + } + v, err := parseStatStartTime(data) + if err != nil { + return 0, fmt.Errorf("parse %s: %w", path, err) + } + return v, nil +} + +// parseStatStartTime extracts field 22 (starttime) from /proc//stat, +// splitting after the last ')' since comm can contain spaces/parens. +func parseStatStartTime(data []byte) (uint64, error) { + line := string(data) + closeParen := strings.LastIndex(line, ")") + if closeParen < 0 { + return 0, fmt.Errorf("malformed stat line (no closing paren): %q", line) + } + tail := strings.TrimSpace(line[closeParen+1:]) + fields := strings.Fields(tail) + const startTimeIndex = 19 // whole-line field 22, minus the 3 pre-state fields + if len(fields) <= startTimeIndex { + return 0, fmt.Errorf("stat line has %d post-comm fields, need at least %d", + len(fields), startTimeIndex+1) + } + v, err := strconv.ParseUint(fields[startTimeIndex], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse starttime %q: %w", fields[startTimeIndex], err) + } + return v, nil +} diff --git a/pkg/procctl/identity_linux_test.go b/pkg/procctl/identity_linux_test.go new file mode 100644 index 000000000..bb358d50b --- /dev/null +++ b/pkg/procctl/identity_linux_test.go @@ -0,0 +1,115 @@ +//go:build linux + +package procctl + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// /proc//stat parser. The comm field can hold spaces and embedded parens +// (kernel writes the filename verbatim), so a naive whitespace split breaks. + +func TestParseStatStartTime_TypicalLine(t *testing.T) { + line := buildStatLine("bash", 1234567) + v, err := parseStatStartTime([]byte(line)) + require.NoError(t, err) + assert.Equal(t, uint64(1234567), v) +} + +// comm with a space must not shift the state field. +func TestParseStatStartTime_CommWithSpaces(t *testing.T) { + line := buildStatLine("my program", 9999999) + v, err := parseStatStartTime([]byte(line)) + require.NoError(t, err) + assert.Equal(t, uint64(9999999), v) +} + +// Embedded parens in comm: parser must skip to the LAST ')'. +func TestParseStatStartTime_CommWithEmbeddedParens(t *testing.T) { + line := buildStatLine("(weird) name)", 42) + v, err := parseStatStartTime([]byte(line)) + require.NoError(t, err) + assert.Equal(t, uint64(42), v) +} + +// Kernel emits a trailing '\n'; parser tolerates it. +func TestParseStatStartTime_TrailingNewline(t *testing.T) { + line := buildStatLine("bash", 100) + "\n" + v, err := parseStatStartTime([]byte(line)) + require.NoError(t, err) + assert.Equal(t, uint64(100), v) +} + +func TestParseStatStartTime_MalformedNoParen(t *testing.T) { + _, err := parseStatStartTime([]byte("totally not a stat line")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no closing paren") +} + +// Truncated post-comm tail errors cleanly (no panic). +func TestParseStatStartTime_TooFewFields(t *testing.T) { + _, err := parseStatStartTime([]byte("1 (bash) R 0 0")) + require.Error(t, err) + assert.Contains(t, err.Error(), "fields") +} + +// buildStatLine builds a synthetic /proc//stat line. Non-starttime fields +// are zero so they can't accidentally match the value under test. +func buildStatLine(comm string, startTime uint64) string { + // Whole-line field 22 (starttime) is post-paren index 19 (zero-based). + post := []string{ + "R", // 3 state + "0", // 4 ppid + "0", // 5 pgrp + "0", // 6 session + "0", // 7 tty_nr + "0", // 8 tpgid + "0", // 9 flags + "0", // 10 minflt + "0", // 11 cminflt + "0", // 12 majflt + "0", // 13 cmajflt + "0", // 14 utime + "0", // 15 stime + "0", // 16 cutime + "0", // 17 cstime + "0", // 18 priority + "0", // 19 nice + "0", // 20 num_threads + "0", // 21 itrealvalue + "", // 22 starttime — filled below + } + post[19] = itoa(startTime) + // Pad so a future bump in expected field count doesn't break these tests. + for i := 0; i < 30; i++ { + post = append(post, "0") + } + tail := joinSpaces(post) + return "12345 (" + comm + ") " + tail +} + +func itoa(v uint64) string { + if v == 0 { + return "0" + } + buf := make([]byte, 0, 20) + for v > 0 { + buf = append([]byte{byte('0' + v%10)}, buf...) + v /= 10 + } + return string(buf) +} + +func joinSpaces(parts []string) string { + if len(parts) == 0 { + return "" + } + out := parts[0] + for _, p := range parts[1:] { + out += " " + p + } + return out +} diff --git a/pkg/procctl/identity_test.go b/pkg/procctl/identity_test.go new file mode 100644 index 000000000..7e7838f48 --- /dev/null +++ b/pkg/procctl/identity_test.go @@ -0,0 +1,128 @@ +package procctl + +import ( + "errors" + "os" + "os/exec" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Cross-platform identity tests (Linux + macOS). /proc parser edge cases live +// in identity_linux_test.go. + +// Reading our own identity works on every platform. +func TestReadIdentity_Self(t *testing.T) { + id, err := ReadIdentity(os.Getpid()) + require.NoError(t, err) + require.NotNil(t, id) + assert.Equal(t, os.Getpid(), id.Pid) + assert.NotZero(t, id.StartTimeTicks, "start time must be populated on every platform") + // ExeInode is Linux-only; macOS leaves it zero. + if runtime.GOOS == "linux" { + assert.NotZero(t, id.ExeInode, "Linux must populate ExeInode") + } +} + +// Absent PID (past pid_max) returns ErrProcessNotFound. +func TestReadIdentity_NonExistentPid(t *testing.T) { + _, err := ReadIdentity(9999999) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrProcessNotFound), + "expected ErrProcessNotFound, got %v", err) +} + +// Matches against our own live identity is true. +func TestMatches_SelfHappyPath(t *testing.T) { + id, err := ReadIdentity(os.Getpid()) + require.NoError(t, err) + + info := &PidInfo{ + Pid: os.Getpid(), + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks, + } + ok, reason := Matches(os.Getpid(), info) + assert.True(t, ok, "matches own identity: %s", reason) +} + +// PID-reuse case: right PID + exe but different start time → no match. +func TestMatches_StartTimeMismatchFails(t *testing.T) { + id, err := ReadIdentity(os.Getpid()) + require.NoError(t, err) + + info := &PidInfo{ + Pid: os.Getpid(), + ExeInode: id.ExeInode, + StartTimeTicks: id.StartTimeTicks + 999_999_999, // synthetic mismatch + } + ok, reason := Matches(os.Getpid(), info) + assert.False(t, ok) + assert.Contains(t, reason, "start time") +} + +// Different binary at the same PID → no match. Skipped on macOS (ExeInode is zero). +func TestMatches_ExeInodeMismatchFails(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("ExeInode is Linux-only") + } + id, err := ReadIdentity(os.Getpid()) + require.NoError(t, err) + + info := &PidInfo{ + Pid: os.Getpid(), + ExeInode: id.ExeInode + 1, // synthetic mismatch + StartTimeTicks: id.StartTimeTicks, + } + ok, reason := Matches(os.Getpid(), info) + assert.False(t, ok) + assert.Contains(t, reason, "inode") +} + +// Recorded PID != live PID bails out before any /proc read. +func TestMatches_PidMismatch(t *testing.T) { + info := &PidInfo{ + Pid: 99, + StartTimeTicks: 12345, + } + ok, reason := Matches(100, info) + assert.False(t, ok) + assert.Contains(t, reason, "pid mismatch") +} + +func TestMatches_IncompleteIdentityFailsClosed(t *testing.T) { + info := &PidInfo{Pid: os.Getpid()} + ok, reason := Matches(os.Getpid(), info) + assert.False(t, ok) + assert.Contains(t, reason, "incomplete") +} + +// Nil PidInfo returns (false, reason), no crash. +func TestMatches_NilInfo(t *testing.T) { + ok, reason := Matches(os.Getpid(), nil) + assert.False(t, ok) + assert.NotEmpty(t, reason) +} + +// Matches against a reaped PID is false. +func TestMatches_DeadPidReturnsFalse(t *testing.T) { + cmd := exec.Command("sh", "-c", "exit 0") + require.NoError(t, cmd.Start()) + pid := cmd.Process.Pid + require.NoError(t, cmd.Wait()) // reaps the zombie + + // Let the kernel flush /proc on Linux (macOS reaps via Wait()). + time.Sleep(10 * time.Millisecond) + + info := &PidInfo{ + Pid: pid, + StartTimeTicks: 1, // arbitrary; ReadIdentity fails first + } + ok, reason := Matches(pid, info) + assert.False(t, ok) + assert.NotEmpty(t, reason) +} diff --git a/pkg/procctl/lockfile.go b/pkg/procctl/lockfile.go new file mode 100644 index 000000000..3d86ffffa --- /dev/null +++ b/pkg/procctl/lockfile.go @@ -0,0 +1,85 @@ +package procctl + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" +) + +// ErrLocked means the lock is held by another process (or another OFD). +// Callers use errors.Is to detect the "already running" case. +var ErrLocked = errors.New("lock already held") + +// LockHandle owns an active flock. Release is idempotent. NOT safe for +// concurrent use: acquire and release on the same goroutine. +type LockHandle struct { + f *os.File + released sync.Once +} + +// AcquireLock opens path (0600 if absent) and takes a non-blocking exclusive +// flock, returning ErrLocked if another OFD holds it. The lock is per-OFD. +func AcquireLock(path string) (*LockHandle, error) { + if err := EnsurePidDir(filepath.Dir(path)); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("open lockfile %s: %w", path, err) + } + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, fmt.Errorf("%s: %w", path, ErrLocked) + } + return nil, fmt.Errorf("flock %s: %w", path, err) + } + return &LockHandle{f: f}, nil +} + +// Release unlocks and closes the descriptor. Only the first call acts; only +// it can return an error. +func (lh *LockHandle) Release() error { + if lh == nil || lh.f == nil { + return nil + } + var releaseErr error + lh.released.Do(func() { + // Unflock then close; close alone would also release the lock. + if err := syscall.Flock(int(lh.f.Fd()), syscall.LOCK_UN); err != nil { + releaseErr = fmt.Errorf("unflock: %w", err) + } + if err := lh.f.Close(); err != nil && releaseErr == nil { + releaseErr = fmt.Errorf("close lockfile: %w", err) + } + lh.f = nil + }) + return releaseErr +} + +// TestLock probes whether path is flock-held: (true, nil) held, (false, nil) +// free, (false, err) on I/O failure. An absent lockfile reads as unlocked. +func TestLock(path string) (bool, error) { + f, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("open lockfile %s for probe: %w", path, err) + } + defer f.Close() + + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return true, nil // held by someone else + } + return false, fmt.Errorf("probe flock %s: %w", path, err) + } + // Free — release immediately; a probe-only release error is benign. + _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN) + return false, nil +} diff --git a/pkg/procctl/lockfile_test.go b/pkg/procctl/lockfile_test.go new file mode 100644 index 000000000..18b642af8 --- /dev/null +++ b/pkg/procctl/lockfile_test.go @@ -0,0 +1,116 @@ +package procctl + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Lockfile — flock-based single-instance guard. Contention is exercised +// within-process via a second fd (different OFD) on the same path. + +func TestAcquireLock_HappyPath(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + lh, err := AcquireLock(path) + require.NoError(t, err) + require.NotNil(t, lh) + require.NoError(t, lh.Release()) + + // flock is advisory; the file persists after Release. + _, err = os.Stat(path) + assert.NoError(t, err) +} + +// Lockfile (and parent dir) is created on demand; callers needn't MkdirAll. +func TestAcquireLock_CreatesLockFileIfMissing(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "nested", "mithril.lock") + + lh, err := AcquireLock(path) + require.NoError(t, err) + defer lh.Release() + + info, err := os.Stat(path) + require.NoError(t, err) + // flock is on the inode, not content — file stays empty. + assert.Equal(t, int64(0), info.Size()) +} + +// Second AcquireLock on a held path returns ErrLocked — the single-instance guard. +func TestAcquireLock_ContentionFails(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + first, err := AcquireLock(path) + require.NoError(t, err) + defer first.Release() + + second, err := AcquireLock(path) + assert.Nil(t, second) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrLocked), "second acquire should return ErrLocked, got %v", err) +} + +// Lock is re-acquirable after Release (stop-then-start-again path). +func TestAcquireLock_ReleaseAllowsReacquire(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + first, err := AcquireLock(path) + require.NoError(t, err) + require.NoError(t, first.Release()) + + second, err := AcquireLock(path) + require.NoError(t, err, "after Release the lock should be re-acquirable") + require.NoError(t, second.Release()) +} + +// Double-Release is a silent no-op, so deferred Release() can't double-error. +func TestRelease_IsIdempotent(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + lh, err := AcquireLock(path) + require.NoError(t, err) + require.NoError(t, lh.Release()) + assert.NoError(t, lh.Release(), "second Release should be a silent no-op") +} + +// TestLock reports false on an unheld lockfile — Detect uses this to spot stale PID files. +func TestTestLock_ReportsUnlocked(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + locked, err := TestLock(path) + require.NoError(t, err) + assert.False(t, locked) +} + +func TestTestLock_ReportsLocked(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.lock") + + lh, err := AcquireLock(path) + require.NoError(t, err) + defer lh.Release() + + locked, err := TestLock(path) + require.NoError(t, err) + assert.True(t, locked) +} + +// Probing an absent lockfile reports unlocked, not an error. +func TestTestLock_AbsentFileIsUnlocked(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "never-created.lock") + + locked, err := TestLock(path) + require.NoError(t, err) + assert.False(t, locked) +} diff --git a/pkg/procctl/path.go b/pkg/procctl/path.go new file mode 100644 index 000000000..0b2d90de1 --- /dev/null +++ b/pkg/procctl/path.go @@ -0,0 +1,89 @@ +// Package procctl provides Mithril's process-control primitives: single-instance +// enforcement (lock + PID file), stale-PID detection, signalling, and an audit log. +package procctl + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + // dirName is the per-user subdirectory holding the PID file, lock file, + // and audit log. Created with 0700. + dirName = "mithril" + + // pidFileBase is the JSON file recording the running process's identity. + pidFileBase = "mithril.pid" + + // lockFileBase holds the flock. Separate from the PID file so the PID + // file can be rewritten without disturbing the lock. + lockFileBase = "mithril.lock" + + // auditLogBase logs every control action as key=value lines. Never rotated. + auditLogBase = "control.audit" + + // envOverride points all three files at one path; its directory holds + // the lock and audit log too. + envOverride = "MITHRIL_PID_FILE" +) + +// DefaultPidFile resolves the PID file path, in precedence order: +// 1. $MITHRIL_PID_FILE +// 2. $XDG_STATE_HOME/mithril/mithril.pid +// 3. $HOME/.local/state/mithril/mithril.pid (default) +// 4. $XDG_RUNTIME_DIR/mithril/mithril.pid (only when HOME is unset) +// 5. /tmp/mithril/mithril.pid (last resort) +// +// $XDG_RUNTIME_DIR is avoided by default (logind wipes it on logout). Stale +// files are safe — Matches() rejects them. +func DefaultPidFile() string { + if v := os.Getenv(envOverride); v != "" { + return v + } + if v := os.Getenv("XDG_STATE_HOME"); v != "" { + return filepath.Join(v, dirName, pidFileBase) + } + if home, _ := os.UserHomeDir(); home != "" { + return filepath.Join(home, ".local", "state", dirName, pidFileBase) + } + // No HOME (some containers/init): runtime dir if present, else /tmp. Both are + // ephemeral, so MITHRIL_PID_FILE is preferred there. + if v := os.Getenv("XDG_RUNTIME_DIR"); v != "" { + return filepath.Join(v, dirName, pidFileBase) + } + return filepath.Join("/tmp", dirName, pidFileBase) +} + +// DefaultLockFile returns the lock file path co-located with DefaultPidFile. +// The lock file is always empty; only its flock state matters. +func DefaultLockFile() string { + return filepath.Join(filepath.Dir(DefaultPidFile()), lockFileBase) +} + +// DefaultAuditLog returns the audit log path co-located with DefaultPidFile. +// The audit log is append-only and never rotated. +func DefaultAuditLog() string { + return filepath.Join(filepath.Dir(DefaultPidFile()), auditLogBase) +} + +// EnsurePidDir creates dir as 0700 when missing. It never chmods an existing +// dir (MITHRIL_PID_FILE may point at a shared dir like /tmp); files stay 0600. +func EnsurePidDir(dir string) error { + if info, err := os.Stat(dir); err == nil { + if !info.IsDir() { + return fmt.Errorf("pid path parent %s is not a directory", dir) + } + return nil + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat pid dir %s: %w", dir, err) + } + + if err := os.MkdirAll(dir, 0700); err != nil { + return fmt.Errorf("create pid dir %s: %w", dir, err) + } + if err := os.Chmod(dir, 0700); err != nil { + return fmt.Errorf("chmod pid dir %s to 0700: %w", dir, err) + } + return nil +} diff --git a/pkg/procctl/path_test.go b/pkg/procctl/path_test.go new file mode 100644 index 000000000..51a60adfb --- /dev/null +++ b/pkg/procctl/path_test.go @@ -0,0 +1,151 @@ +package procctl + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// PID-file path resolution precedence (first set wins): +// MITHRIL_PID_FILE > $XDG_STATE_HOME > $HOME/.local/state > $XDG_RUNTIME_DIR. + +// withEnv sets env vars for the test, restoring prior values (incl. unset) on cleanup. +func withEnv(t *testing.T, kvs map[string]string) { + t.Helper() + for k, v := range kvs { + prior, had := os.LookupEnv(k) + require.NoError(t, os.Setenv(k, v)) + t.Cleanup(func() { + if had { + _ = os.Setenv(k, prior) + } else { + _ = os.Unsetenv(k) + } + }) + } +} + +// withUnsetEnv unsets env vars for the test, restoring on cleanup (for fallback chains). +func withUnsetEnv(t *testing.T, keys ...string) { + t.Helper() + for _, k := range keys { + prior, had := os.LookupEnv(k) + require.NoError(t, os.Unsetenv(k)) + t.Cleanup(func() { + if had { + _ = os.Setenv(k, prior) + } + }) + } +} + +// MITHRIL_PID_FILE overrides every XDG layer. +func TestDefaultPidFile_HonorsExplicitEnvOverride(t *testing.T) { + dir := t.TempDir() + override := filepath.Join(dir, "custom", "my.pid") + withEnv(t, map[string]string{"MITHRIL_PID_FILE": override}) + + got := DefaultPidFile() + assert.Equal(t, override, got, "explicit env override must be returned verbatim") +} + +// XDG_STATE_HOME (persistent) wins over XDG_RUNTIME_DIR (logout-wiped). +func TestDefaultPidFile_PrefersXdgStateHomeOverRuntimeDir(t *testing.T) { + runtimeDir := t.TempDir() + stateDir := t.TempDir() + withUnsetEnv(t, "MITHRIL_PID_FILE") + withEnv(t, map[string]string{ + "XDG_RUNTIME_DIR": runtimeDir, + "XDG_STATE_HOME": stateDir, + }) + + got := DefaultPidFile() + assert.Equal(t, filepath.Join(stateDir, "mithril", "mithril.pid"), got, + "persistent XDG_STATE_HOME must win over logout-wiped XDG_RUNTIME_DIR") +} + +func TestDefaultPidFile_UsesXdgStateHome(t *testing.T) { + stateDir := t.TempDir() + withUnsetEnv(t, "MITHRIL_PID_FILE", "XDG_RUNTIME_DIR") + withEnv(t, map[string]string{"XDG_STATE_HOME": stateDir}) + + got := DefaultPidFile() + assert.Equal(t, filepath.Join(stateDir, "mithril", "mithril.pid"), got) +} + +// $HOME/.local/state (persistent) wins over XDG_RUNTIME_DIR when XDG_STATE_HOME is unset. +func TestDefaultPidFile_PrefersHomeOverRuntimeDir(t *testing.T) { + home := t.TempDir() + runtimeDir := t.TempDir() + withUnsetEnv(t, "MITHRIL_PID_FILE", "XDG_STATE_HOME") + withEnv(t, map[string]string{"HOME": home, "XDG_RUNTIME_DIR": runtimeDir}) + + got := DefaultPidFile() + assert.Equal(t, filepath.Join(home, ".local", "state", "mithril", "mithril.pid"), got, + "persistent $HOME/.local/state must win over logout-wiped XDG_RUNTIME_DIR") +} + +// With no HOME and no XDG_STATE_HOME, falls back to XDG_RUNTIME_DIR. +func TestDefaultPidFile_FallsBackToRuntimeDirWhenNoHome(t *testing.T) { + runtimeDir := t.TempDir() + withUnsetEnv(t, "MITHRIL_PID_FILE", "XDG_STATE_HOME", "HOME") + withEnv(t, map[string]string{"XDG_RUNTIME_DIR": runtimeDir}) + + got := DefaultPidFile() + assert.Equal(t, filepath.Join(runtimeDir, "mithril", "mithril.pid"), got) +} + +// Lock file shares the PID file's directory. +func TestDefaultLockFile_LivesAlongsidePidFile(t *testing.T) { + withUnsetEnv(t, "MITHRIL_PID_FILE") + pidPath := DefaultPidFile() + lockPath := DefaultLockFile() + assert.Equal(t, filepath.Dir(pidPath), filepath.Dir(lockPath), + "lock file and PID file must be in the same directory") + assert.Equal(t, "mithril.lock", filepath.Base(lockPath)) +} + +// Audit log shares the PID file's directory — must not sit under rotating storage.logs. +func TestDefaultAuditLog_LivesAlongsidePidFile(t *testing.T) { + withUnsetEnv(t, "MITHRIL_PID_FILE") + pidPath := DefaultPidFile() + auditPath := DefaultAuditLog() + assert.Equal(t, filepath.Dir(pidPath), filepath.Dir(auditPath), + "audit log and PID file must be in the same directory") + assert.Equal(t, "control.audit", filepath.Base(auditPath)) +} + +// EnsurePidDir creates the dir 0700 (per-user state). +func TestEnsurePidDir_CreatesDirWith0700(t *testing.T) { + base := t.TempDir() + target := filepath.Join(base, "nested", "dir") + require.NoError(t, EnsurePidDir(target)) + + info, err := os.Stat(target) + require.NoError(t, err) + assert.True(t, info.IsDir()) + assert.Equal(t, os.FileMode(0700), info.Mode().Perm()) +} + +func TestEnsurePidDir_IdempotentOnExisting(t *testing.T) { + dir := t.TempDir() + require.NoError(t, EnsurePidDir(dir)) + require.NoError(t, EnsurePidDir(dir)) +} + +// Existing dirs aren't chmod'd — protects shared dirs like /tmp. +func TestEnsurePidDir_DoesNotChmodExistingDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "shared") + require.NoError(t, os.Mkdir(dir, 0755)) + // Chmod explicitly: os.Mkdir applies mode &^ umask, so a hardened umask + // would otherwise strip bits and make the assertion flaky. + require.NoError(t, os.Chmod(dir, 0755)) + require.NoError(t, EnsurePidDir(dir)) + + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0755), info.Mode().Perm()) +} diff --git a/pkg/procctl/pidfd_linux.go b/pkg/procctl/pidfd_linux.go new file mode 100644 index 000000000..e92d27384 --- /dev/null +++ b/pkg/procctl/pidfd_linux.go @@ -0,0 +1,90 @@ +//go:build linux + +package procctl + +import ( + "errors" + "sync" + "syscall" + + "golang.org/x/sys/unix" +) + +type SignalHandle struct { + pid int + fd int + usePidfd bool +} + +var ( + pidfdProbeOnce sync.Once + pidfdSupported bool +) + +// PidfdAvailable reports whether the kernel supports pidfd (>=5.3), probing +// once via pidfd_open(getpid()) and caching the result. +func PidfdAvailable() bool { + pidfdProbeOnce.Do(func() { + fd, err := unix.PidfdOpen(unix.Getpid(), 0) + switch { + case err == nil: + _ = unix.Close(fd) + pidfdSupported = true + case errors.Is(err, unix.ENOSYS): + pidfdSupported = false // pre-5.3, use kill(2) + default: + // Transient failure (e.g. fd exhaustion) — assume supported; per-call + // open errors surface in OpenSignalHandle. + pidfdSupported = true + } + }) + return pidfdSupported +} + +// OpenSignalHandle opens a reusable signal target. With pidfd, open it before +// Matches() and signal the same handle after, closing the PID-reuse window. +func OpenSignalHandle(pid int) (*SignalHandle, error) { + if PidfdAvailable() { + fd, err := unix.PidfdOpen(pid, 0) + if err != nil { + // Already gone — treat as benign. + if errors.Is(err, unix.ESRCH) { + return nil, ErrProcessNotFound + } + return nil, err + } + return &SignalHandle{pid: pid, fd: fd, usePidfd: true}, nil + } + return &SignalHandle{pid: pid}, nil +} + +func (h *SignalHandle) Close() error { + if h == nil || !h.usePidfd { + return nil + } + err := unix.Close(h.fd) + h.usePidfd = false + h.fd = -1 + return err +} + +func (h *SignalHandle) Signal(sig syscall.Signal) error { + if h == nil { + return errors.New("nil signal handle") + } + if h.usePidfd { + return unix.PidfdSendSignal(h.fd, unix.Signal(sig), nil, 0) + } + return syscall.Kill(h.pid, sig) +} + +// SendSignal delivers sig via pidfd_send_signal when available, else kill(2) +// (signal 0 = liveness probe). The kill(2) fallback needs a Matches() re-verify. +func SendSignal(pid int, sig syscall.Signal) error { + handle, err := OpenSignalHandle(pid) + if err != nil { + return err + } + defer handle.Close() + return handle.Signal(sig) +} diff --git a/pkg/procctl/pidfd_other.go b/pkg/procctl/pidfd_other.go new file mode 100644 index 000000000..5f54eaffd --- /dev/null +++ b/pkg/procctl/pidfd_other.go @@ -0,0 +1,34 @@ +//go:build !linux + +package procctl + +import "syscall" + +type SignalHandle struct { + pid int +} + +// PidfdAvailable is always false off Linux (pidfd is Linux-only). +func PidfdAvailable() bool { return false } + +func OpenSignalHandle(pid int) (*SignalHandle, error) { + return &SignalHandle{pid: pid}, nil +} + +func (h *SignalHandle) Close() error { + return nil +} + +func (h *SignalHandle) Signal(sig syscall.Signal) error { + return syscall.Kill(h.pid, sig) +} + +// SendSignal sends sig via kill(2) off Linux. +func SendSignal(pid int, sig syscall.Signal) error { + handle, err := OpenSignalHandle(pid) + if err != nil { + return err + } + defer handle.Close() + return handle.Signal(sig) +} diff --git a/pkg/procctl/pidfd_test.go b/pkg/procctl/pidfd_test.go new file mode 100644 index 000000000..10e15eee4 --- /dev/null +++ b/pkg/procctl/pidfd_test.go @@ -0,0 +1,72 @@ +package procctl + +import ( + "os/exec" + "runtime" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// SendSignal — race-free delivery (pidfd on Linux ≥5.3, syscall.Kill fallback +// elsewhere). Tests run against a real sleep subprocess. + +// SIGTERM to a sleeping process makes it exit well before the sleep ends. +func TestSendSignal_DeliversSigtermToSleep(t *testing.T) { + cmd := exec.Command("sh", "-c", "sleep 30") + require.NoError(t, cmd.Start()) + pid := cmd.Process.Pid + + // Give the shell + sleep a tick to set up signal handlers. + time.Sleep(50 * time.Millisecond) + + require.NoError(t, SendSignal(pid, syscall.SIGTERM)) + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + select { + case <-done: + // Expected — process exited promptly. + case <-time.After(5 * time.Second): + // If we timed out, force-kill so the test binary doesn't hang. + _ = cmd.Process.Kill() + <-done + t.Fatal("process did not exit within 5s of SIGTERM") + } +} + +// Signaling an absent PID errors rather than silently succeeding. +func TestSendSignal_NonExistentPidReturnsErr(t *testing.T) { + err := SendSignal(9999999, syscall.SIGTERM) + require.Error(t, err) + // Skip the errno check: pidfd and Kill paths wrap ESRCH differently. + t.Logf("SendSignal to absent PID returned: %v", err) +} + +// Signal 0 is a liveness/permission check (used by Detect and audit logging). +func TestSendSignal_Sig0IsLivenessCheck(t *testing.T) { + cmd := exec.Command("sh", "-c", "sleep 5") + require.NoError(t, cmd.Start()) + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + time.Sleep(20 * time.Millisecond) + + require.NoError(t, SendSignal(cmd.Process.Pid, syscall.Signal(0))) +} + +// PidfdAvailable is stable across calls and false on non-Linux. +func TestPidfdAvailable_DocsRuntime(t *testing.T) { + a := PidfdAvailable() + b := PidfdAvailable() + assert.Equal(t, a, b, "PidfdAvailable should be stable") + if runtime.GOOS != "linux" { + assert.False(t, a, "PidfdAvailable must be false on non-Linux") + } + t.Logf("PidfdAvailable on %s: %v", runtime.GOOS, a) +} diff --git a/pkg/procctl/pidfile.go b/pkg/procctl/pidfile.go new file mode 100644 index 000000000..4537b726e --- /dev/null +++ b/pkg/procctl/pidfile.go @@ -0,0 +1,203 @@ +package procctl + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" +) + +// pidFileSchemaVersion is the PID file format version; bump only on +// incompatible changes (readers reject any other value). +const pidFileSchemaVersion = 1 + +// maxPidFileSize caps ReadPidFile (content is ~500 bytes) as a DoS guard +// against a huge file in the PID-file slot. +const maxPidFileSize = 64 * 1024 + +// ErrPidFileNotFound is returned when the PID file is absent. Callers use +// errors.Is to distinguish "not running" from "corrupt — investigate." +var ErrPidFileNotFound = errors.New("pid file not found") + +// PidInfo is the JSON content of mithril.pid. The (Pid, StartTimeTicks, +// ExeInode) triple is collision-proof for stale detection across PID reuse. +type PidInfo struct { + SchemaVersion int `json:"schema_version"` + + // Process identity (stable for the process's lifetime) + Pid int `json:"pid"` + StartTimeTicks uint64 `json:"start_time_ticks,omitempty"` // /proc//stat field 22 on Linux + ExeInode uint64 `json:"exe_inode,omitempty"` // st_ino of /proc//exe + BinaryPath string `json:"binary_path"` // resolved absolute path + + // Session metadata + RunID string `json:"run_id"` + ConfigPath string `json:"config_path,omitempty"` + SpawnedBy string `json:"spawned_by"` // "dashboard" | "external" | "cli" + LogDir string `json:"log_dir,omitempty"` + StdoutPath string `json:"stdout_path,omitempty"` // dashboard-spawned child stdout tail + StderrPath string `json:"stderr_path,omitempty"` // dashboard-spawned child stderr tail + + // Control-state metadata (overwritten by signal/stop helpers) + StopInProgressBy int `json:"stop_in_progress_by,omitempty"` // dashboard PID currently stopping; 0 = none + StopInProgressAt time.Time `json:"stop_in_progress_at"` +} + +// isStopProgressStale reports whether the recorded stopping dashboard is gone or +// older than ttl. Zero means "no stop in progress" (never stale). +func (p *PidInfo) isStopProgressStale(ttl time.Duration) bool { + if p == nil || p.StopInProgressBy == 0 { + return false + } + if p.StopInProgressBy <= 1 { + // PID <=1 is never a valid owner; -1 would let kill(-1, 0) broadcast. + return true + } + if !p.StopInProgressAt.IsZero() && time.Since(p.StopInProgressAt) > ttl { + return true + } + // signal-0 ESRCH means the dashboard is gone. + err := syscall.Kill(p.StopInProgressBy, 0) + return errors.Is(err, syscall.ESRCH) +} + +// ReadPidFile loads PidInfo, returning ErrPidFileNotFound when absent. The +// read is capped at maxPidFileSize so a huge file can't exhaust memory. +func ReadPidFile(path string) (*PidInfo, error) { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrPidFileNotFound + } + return nil, fmt.Errorf("open pid file %s: %w", path, err) + } + defer f.Close() + + data, err := io.ReadAll(io.LimitReader(f, maxPidFileSize)) + if err != nil { + return nil, fmt.Errorf("read pid file %s: %w", path, err) + } + + var info PidInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, fmt.Errorf("parse pid file %s: %w", path, err) + } + if info.SchemaVersion != pidFileSchemaVersion { + return nil, fmt.Errorf("pid file %s has unsupported schema version %d (expected %d)", + path, info.SchemaVersion, pidFileSchemaVersion) + } + return &info, nil +} + +// WritePidFile writes PidInfo atomically (tmp+rename, dir 0700, file 0600). +// SchemaVersion is forced to the current value. +func WritePidFile(path string, info *PidInfo) error { + if info == nil { + return fmt.Errorf("nil pid info") + } + info.SchemaVersion = pidFileSchemaVersion + + if err := EnsurePidDir(filepath.Dir(path)); err != nil { + return err + } + + data, err := json.MarshalIndent(info, "", " ") + if err != nil { + return fmt.Errorf("marshal pid info: %w", err) + } + if err := atomicWriteFile(path, data, 0600); err != nil { + return fmt.Errorf("write pid file %s: %w", path, err) + } + return nil +} + +// RemovePidFile deletes path. Absent files are not an error — cleanup +// paths should not have to special-case the "already gone" case. +func RemovePidFile(path string) error { + err := os.Remove(path) + if err == nil || os.IsNotExist(err) { + return nil + } + return fmt.Errorf("remove pid file %s: %w", path, err) +} + +// RemovePidFileIfMatches deletes path only when it still describes the expected +// run, so Close() can't delete a newer run's PID file. +func RemovePidFileIfMatches(path string, expected *PidInfo) error { + if expected == nil { + return fmt.Errorf("nil expected pid info") + } + current, err := ReadPidFile(path) + if err != nil { + if errors.Is(err, ErrPidFileNotFound) { + return nil + } + return err + } + if !sameRunIdentity(current, expected) { + return nil + } + return RemovePidFile(path) +} + +func sameRunIdentity(a, b *PidInfo) bool { + if a == nil || b == nil { + return false + } + return a.Pid == b.Pid && + a.StartTimeTicks == b.StartTimeTicks && + a.ExeInode == b.ExeInode && + a.RunID == b.RunID +} + +// UpdatePidLogDir records the per-run log dir after mlog init, via a second +// rewrite while still holding the lock. +func UpdatePidLogDir(path, logDir string) error { + info, err := ReadPidFile(path) + if err != nil { + return err + } + info.LogDir = logDir + return WritePidFile(path, info) +} + +// UpdatePidOutputPaths records the dashboard-owned stdout/stderr files for a child. +// The PID check keeps a racing dashboard from annotating a different run's file. +func UpdatePidOutputPaths(path string, pid int, stdoutPath, stderrPath string) error { + info, err := ReadPidFile(path) + if err != nil { + return err + } + if info.Pid != pid { + return nil + } + info.StdoutPath = stdoutPath + info.StderrPath = stderrPath + return WritePidFile(path, info) +} + +// updateStopInProgressIfMatches sets (pid != 0) or clears (0) stop_in_progress_by, +// only when the file still describes expected. Idempotent. +func updateStopInProgressIfMatches(path string, dashboardPid int, expected *PidInfo) error { + if expected == nil { + return fmt.Errorf("nil expected pid info") + } + info, err := ReadPidFile(path) + if err != nil { + return err + } + if !sameRunIdentity(info, expected) { + return nil + } + info.StopInProgressBy = dashboardPid + if dashboardPid == 0 { + info.StopInProgressAt = time.Time{} + } else { + info.StopInProgressAt = time.Now() + } + return WritePidFile(path, info) +} diff --git a/pkg/procctl/pidfile_test.go b/pkg/procctl/pidfile_test.go new file mode 100644 index 000000000..8603e2022 --- /dev/null +++ b/pkg/procctl/pidfile_test.go @@ -0,0 +1,252 @@ +package procctl + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// samplePidInfo builds a deterministic PidInfo for write/read tests. +func samplePidInfo() *PidInfo { + return &PidInfo{ + SchemaVersion: pidFileSchemaVersion, + Pid: 12345, + StartTimeTicks: 1723456789, + ExeInode: 5827319, + BinaryPath: "/home/operator/mithril/mithril", + RunID: "20260518-143209Z_abc1234_def56789", + ConfigPath: "/home/operator/mithril/config.toml", + SpawnedBy: "dashboard", + LogDir: "/mnt/mithril-logs/20260518-143209Z_abc1234_def56789", + StdoutPath: "/tmp/mithril-dashboard-spawn-stdout-123.log", + StderrPath: "/tmp/mithril-dashboard-spawn-stderr-123.log", + StopInProgressBy: 0, + } +} + +// All fields survive a write→read JSON roundtrip. +func TestPidInfo_WriteRead_Roundtrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + want := samplePidInfo() + + require.NoError(t, WritePidFile(path, want)) + + got, err := ReadPidFile(path) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, want.SchemaVersion, got.SchemaVersion) + assert.Equal(t, want.Pid, got.Pid) + assert.Equal(t, want.StartTimeTicks, got.StartTimeTicks) + assert.Equal(t, want.ExeInode, got.ExeInode) + assert.Equal(t, want.BinaryPath, got.BinaryPath) + assert.Equal(t, want.RunID, got.RunID) + assert.Equal(t, want.ConfigPath, got.ConfigPath) + assert.Equal(t, want.SpawnedBy, got.SpawnedBy) + assert.Equal(t, want.LogDir, got.LogDir) + assert.Equal(t, want.StdoutPath, got.StdoutPath) + assert.Equal(t, want.StderrPath, got.StderrPath) +} + +// Absent file returns ErrPidFileNotFound, distinct from corrupt. +func TestReadPidFile_NotFound(t *testing.T) { + dir := t.TempDir() + got, err := ReadPidFile(filepath.Join(dir, "absent.pid")) + assert.Nil(t, got) + assert.ErrorIs(t, err, ErrPidFileNotFound) +} + +// Corrupt JSON returns a wrapped error, not ErrPidFileNotFound. +func TestReadPidFile_CorruptJSON(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.pid") + require.NoError(t, os.WriteFile(path, []byte("{not json"), 0600)) + + got, err := ReadPidFile(path) + assert.Nil(t, got) + require.Error(t, err) + assert.NotErrorIs(t, err, ErrPidFileNotFound, "corrupt JSON must not pretend to be 'not found'") +} + +// Unknown schema versions are rejected. +func TestReadPidFile_WrongSchemaVersion(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.pid") + body, _ := json.Marshal(map[string]any{"schema_version": 999, "pid": 1}) + require.NoError(t, os.WriteFile(path, body, 0600)) + + got, err := ReadPidFile(path) + assert.Nil(t, got) + require.Error(t, err) + assert.Contains(t, err.Error(), "schema") +} + +// Write is atomic via tmp+rename; no .tmp.* leftover after success. +func TestWritePidFile_AtomicViaTmpRename(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + require.NoError(t, WritePidFile(path, samplePidInfo())) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "mithril.pid", entries[0].Name()) +} + +// File is 0600 — JSON holds config paths and run IDs. +func TestWritePidFile_Permissions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + require.NoError(t, WritePidFile(path, samplePidInfo())) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestWritePidFile_CreatesParentDir(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "nested", "mithril.pid") + require.NoError(t, WritePidFile(path, samplePidInfo())) + + _, err := os.Stat(path) + assert.NoError(t, err) +} + +func TestRemovePidFile_DeletesExisting(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + require.NoError(t, WritePidFile(path, samplePidInfo())) + require.NoError(t, RemovePidFile(path)) + + _, err := os.Stat(path) + assert.True(t, os.IsNotExist(err)) +} + +// Removing an absent file must not propagate ENOENT. +func TestRemovePidFile_AbsentIsNotError(t *testing.T) { + dir := t.TempDir() + require.NoError(t, RemovePidFile(filepath.Join(dir, "never_existed.pid"))) +} + +// UpdatePidLogDir sets the log dir without disturbing identity metadata. +func TestUpdatePidLogDir(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + require.NoError(t, WritePidFile(path, samplePidInfo())) + + newLogDir := "/mnt/mithril-logs/run-after-init" + require.NoError(t, UpdatePidLogDir(path, newLogDir)) + + got, err := ReadPidFile(path) + require.NoError(t, err) + assert.Equal(t, newLogDir, got.LogDir) + assert.Equal(t, samplePidInfo().Pid, got.Pid) + assert.Equal(t, samplePidInfo().RunID, got.RunID) +} + +func TestUpdatePidOutputPaths(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + info := samplePidInfo() + info.StdoutPath = "" + info.StderrPath = "" + require.NoError(t, WritePidFile(path, info)) + + require.NoError(t, UpdatePidOutputPaths(path, info.Pid, "/tmp/stdout.log", "/tmp/stderr.log")) + + got, err := ReadPidFile(path) + require.NoError(t, err) + assert.Equal(t, "/tmp/stdout.log", got.StdoutPath) + assert.Equal(t, "/tmp/stderr.log", got.StderrPath) +} + +func TestUpdatePidOutputPaths_IgnoresDifferentPid(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + info := samplePidInfo() + info.StdoutPath = "" + info.StderrPath = "" + require.NoError(t, WritePidFile(path, info)) + + require.NoError(t, UpdatePidOutputPaths(path, info.Pid+1, "/tmp/stdout.log", "/tmp/stderr.log")) + + got, err := ReadPidFile(path) + require.NoError(t, err) + assert.Empty(t, got.StdoutPath) + assert.Empty(t, got.StderrPath) +} + +// updateStopInProgressIfMatches sets/clears stop_in_progress_by when the run still matches. +func TestUpdateStopInProgressSetClear(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + expected := samplePidInfo() + require.NoError(t, WritePidFile(path, expected)) + + // Set. + require.NoError(t, updateStopInProgressIfMatches(path, 99999, expected)) + got, err := ReadPidFile(path) + require.NoError(t, err) + assert.Equal(t, 99999, got.StopInProgressBy) + assert.False(t, got.StopInProgressAt.IsZero(), "stop_in_progress_at should be stamped on set") + + // Clear. + require.NoError(t, updateStopInProgressIfMatches(path, 0, got)) + got, err = ReadPidFile(path) + require.NoError(t, err) + assert.Equal(t, 0, got.StopInProgressBy) + assert.True(t, got.StopInProgressAt.IsZero(), "stop_in_progress_at should be zeroed on clear") +} + +func TestUpdateStopInProgressIfMatchesSkipsDifferentRun(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "mithril.pid") + oldRun := samplePidInfo() + newRun := samplePidInfo() + newRun.RunID = "new-run" + newRun.StartTimeTicks++ + require.NoError(t, WritePidFile(path, newRun)) + + require.NoError(t, updateStopInProgressIfMatches(path, 99999, oldRun)) + + got, err := ReadPidFile(path) + require.NoError(t, err) + assert.Equal(t, "new-run", got.RunID) + assert.Equal(t, 0, got.StopInProgressBy) +} + +// isStopProgressStale staleness rules across PID sentinel, security gate, TTL, and liveness. +func TestPidInfo_StopStaleByDeadDashboard(t *testing.T) { + // PID 0 is the "no stop in progress" sentinel — not stale. + none := &PidInfo{StopInProgressBy: 0} + assert.False(t, none.isStopProgressStale(time.Minute)) + + // PID 1 (init) is never a real dashboard — always stale. + pidOne := &PidInfo{StopInProgressBy: 1, StopInProgressAt: time.Now()} + assert.True(t, pidOne.isStopProgressStale(time.Minute), + "PID 1 should always be stale (not a legitimate dashboard PID)") + + // Negative PID must be stale before syscall.Kill sees it (kill(-1, 0) broadcast). + negPid := &PidInfo{StopInProgressBy: -1, StopInProgressAt: time.Now()} + assert.True(t, negPid.isStopProgressStale(time.Minute), + "negative PID must be rejected as stale to prevent kill(-1, 0) broadcast") + + // Bogus high PID is almost certainly dead → stale. + dead := &PidInfo{StopInProgressBy: 999999, StopInProgressAt: time.Now()} + assert.True(t, dead.isStopProgressStale(time.Minute)) + + // Old timestamp is stale via TTL regardless of liveness. + staleByAge := &PidInfo{StopInProgressBy: 99999, StopInProgressAt: time.Now().Add(-10 * time.Minute)} + assert.True(t, staleByAge.isStopProgressStale(time.Minute)) + + // Live PID with a fresh timestamp — not stale (another dashboard mid-stop). + livePid := &PidInfo{StopInProgressBy: os.Getpid(), StopInProgressAt: time.Now()} + assert.False(t, livePid.isStopProgressStale(time.Minute), + "live dashboard PID with fresh timestamp must not be stale") +} diff --git a/pkg/progress/jsonl.go b/pkg/progress/jsonl.go new file mode 100644 index 000000000..e0a6a9e43 --- /dev/null +++ b/pkg/progress/jsonl.go @@ -0,0 +1,74 @@ +package progress + +import ( + "encoding/json" + "fmt" + "os" + "sync" + "time" +) + +// JSONLEmitter appends one-line JSON progress events for external tools to tail. +// Append-only, mutex-serialized, fail-soft; empty path and nil receiver are no-ops. +type JSONLEmitter struct { + mu sync.Mutex + f *os.File +} + +const JSONLFileName = "progress.jsonl" + +// NewJSONLEmitter opens path in append+create mode. Empty path returns a no-op +// emitter. Returns a wrapped error if the file cannot be opened. +func NewJSONLEmitter(path string) (*JSONLEmitter, error) { + if path == "" { + return &JSONLEmitter{}, nil + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return nil, fmt.Errorf("open jsonl %s: %w", path, err) + } + return &JSONLEmitter{f: f}, nil +} + +// Emit appends one JSON line, stamping "ts" (RFC3339Nano) and overwriting any +// caller-supplied "ts". Nil/disabled emitters are no-ops; errors go to stderr. +func (e *JSONLEmitter) Emit(event map[string]any) { + if e == nil { + return + } + if event == nil { + event = map[string]any{} + } + event["ts"] = time.Now().UTC().Format(time.RFC3339Nano) + + data, err := json.Marshal(event) + if err != nil { + fmt.Fprintf(os.Stderr, "progress jsonl: marshal failed: %v\n", err) + return + } + data = append(data, '\n') + + // Check e.f under the lock — Close() nils it under the same lock. + e.mu.Lock() + defer e.mu.Unlock() + if e.f == nil { + return + } + if _, err := e.f.Write(data); err != nil { + fmt.Fprintf(os.Stderr, "progress jsonl: write failed: %v\n", err) + } +} + +// Close closes the underlying file. Safe on a nil/disabled emitter; later Emit +// calls become no-ops. +func (e *JSONLEmitter) Close() { + if e == nil { + return + } + e.mu.Lock() + defer e.mu.Unlock() + if e.f != nil { + _ = e.f.Close() + e.f = nil + } +} diff --git a/pkg/progress/jsonl_test.go b/pkg/progress/jsonl_test.go new file mode 100644 index 000000000..d9dd259a4 --- /dev/null +++ b/pkg/progress/jsonl_test.go @@ -0,0 +1,160 @@ +package progress + +import ( + "bufio" + "encoding/json" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// empty path -> usable no-op emitter (feature disabled). +func TestJSONLEmitter_NilPathIsNoOp(t *testing.T) { + e, err := NewJSONLEmitter("") + require.NoError(t, err) + require.NotNil(t, e) + e.Emit(map[string]any{"phase": "test", "n": 1}) + e.Close() +} + +// nil receiver methods must not panic (zero value = disabled). +func TestJSONLEmitter_NilReceiver(t *testing.T) { + var e *JSONLEmitter + e.Emit(map[string]any{"x": 1}) + e.Close() +} + +// each Emit produces one well-formed JSON line. +func TestJSONLEmitter_AppendsOneLinePerEmit(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "progress.jsonl") + + e, err := NewJSONLEmitter(path) + require.NoError(t, err) + defer e.Close() + + e.Emit(map[string]any{"phase": "snapshot_download", "done": 100, "total": 1000}) + e.Emit(map[string]any{"phase": "snapshot_download", "done": 500, "total": 1000}) + e.Emit(map[string]any{"phase": "ready"}) + e.Close() + + lines := readAllLines(t, path) + require.Len(t, lines, 3) + + for i, line := range lines { + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &ev), + "line %d should be valid JSON: %s", i, line) + assert.NotEmpty(t, ev["phase"], "line %d should have 'phase' field", i) + assert.NotEmpty(t, ev["ts"], "emitter should stamp 'ts' on every event") + } +} + +// emitter stamps a 'ts' (RFC3339) on every event automatically. +func TestJSONLEmitter_StampsTimestamp(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "progress.jsonl") + + e, err := NewJSONLEmitter(path) + require.NoError(t, err) + defer e.Close() + + e.Emit(map[string]any{"phase": "x"}) + e.Close() + + lines := readAllLines(t, path) + require.Len(t, lines, 1) + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &ev)) + tsRaw, ok := ev["ts"].(string) + require.True(t, ok, "ts should be a string") + assert.Contains(t, tsRaw, "T") // RFC3339 date/time separator +} + +// emitter's own 'ts' wins over a caller-supplied one. +func TestJSONLEmitter_CallerCannotOverrideTimestamp(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "progress.jsonl") + + e, err := NewJSONLEmitter(path) + require.NoError(t, err) + defer e.Close() + + e.Emit(map[string]any{"phase": "x", "ts": "1999-01-01T00:00:00Z"}) + e.Close() + + lines := readAllLines(t, path) + require.Len(t, lines, 1) + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &ev)) + assert.NotEqual(t, "1999-01-01T00:00:00Z", ev["ts"]) +} + +// concurrent Emits produce well-formed lines (mutex prevents interleaving). +func TestJSONLEmitter_ConcurrentEmitsSerialize(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "progress.jsonl") + + e, err := NewJSONLEmitter(path) + require.NoError(t, err) + + var wg sync.WaitGroup + const writers = 8 + const perWriter = 50 + for i := 0; i < writers; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for j := 0; j < perWriter; j++ { + e.Emit(map[string]any{"phase": "x", "writer": id, "seq": j}) + } + }(i) + } + wg.Wait() + e.Close() + + lines := readAllLines(t, path) + assert.Equal(t, writers*perWriter, len(lines)) + for i, line := range lines { + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &ev), + "line %d garbled: %q", i, line) + } +} + +// reopening appends (not truncates) so prior-run history survives restart. +func TestJSONLEmitter_AppendsToExistingFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "progress.jsonl") + + e1, err := NewJSONLEmitter(path) + require.NoError(t, err) + e1.Emit(map[string]any{"phase": "first"}) + e1.Close() + + e2, err := NewJSONLEmitter(path) + require.NoError(t, err) + e2.Emit(map[string]any{"phase": "second"}) + e2.Close() + + lines := readAllLines(t, path) + require.Len(t, lines, 2) +} + +func readAllLines(t *testing.T, path string) []string { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + var out []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + out = append(out, scanner.Text()) + } + require.NoError(t, scanner.Err()) + return out +} diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index bc3f9da75..64f2d5da3 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "errors" "fmt" "io" "os" @@ -459,6 +460,14 @@ func readTar( if err == io.EOF { break } else if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil || errors.Is(err, context.Canceled) { + if ctxErr == nil { + ctxErr = err + } + mlog.Log.Infof("Snapshot unpack stopped during shutdown: %v", ctxErr) + cleanupPartial("canceled") + return ctxErr + } mlog.Log.Errorf("reading next tar: %s\n", err) cleanupPartial("read error") return err @@ -471,6 +480,14 @@ func readTar( writer := bytes.NewBuffer(make([]byte, 0, header.Size)) tarBytesRead, err := io.Copy(writer, tarReader) if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil || errors.Is(err, context.Canceled) { + if ctxErr == nil { + ctxErr = err + } + mlog.Log.Infof("Snapshot unpack stopped during shutdown: %v", ctxErr) + cleanupPartial("canceled") + return ctxErr + } mlog.Log.Errorf("err copying data to reader: %s\n", err) cleanupPartial("copy error") return err diff --git a/pkg/state/state.go b/pkg/state/state.go index 4ef42b9c9..c4f687af6 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/Overclock-Validator/mithril/pkg/mlog" @@ -22,27 +23,24 @@ const CurrentStateSchemaVersion uint32 = 2 // The state file serves as an atomic marker of validity - AccountsDB is valid // if and only if this file exists with Stage == "ready". type MithrilState struct { - // ========================================================================= - // Schema & Run Lineage - // ========================================================================= + // Schema and run lineage. StateSchemaVersion uint32 `json:"state_schema_version"` // Run lineage - tracks the chain of sessions that have used this AccountsDB RootRunID string `json:"root_run_id,omitempty"` // Run that built AccountsDB from snapshot (never changes) ParentRunID string `json:"parent_run_id,omitempty"` // Run we resumed from (empty if fresh start) CurrentRunID string `json:"current_run_id,omitempty"` // Run that last wrote this file - // ========================================================================= - // Writer Metadata (who last wrote this file and why they stopped) - // ========================================================================= + // Writer metadata: who last wrote this file and why they stopped. LastWriterVersion string `json:"last_writer_version,omitempty"` // Semver tag (e.g., "v0.1.0" or "dev") LastWriterCommit string `json:"last_writer_commit,omitempty"` // Git commit hash of writer binary LastWriterBranch string `json:"last_writer_branch,omitempty"` // Git branch name (may be empty) LastShutdownReason string `json:"last_shutdown_reason,omitempty"` // human-readable reason LastShutdownAt time.Time `json:"last_shutdown_at,omitempty"` // when shutdown occurred - // ========================================================================= - // AccountsDB Origin (snapshot info - set once, never changes) - // ========================================================================= + // Anchors clean-exit detection. Zero means legacy file (see WasCleanExit). + CurrentSessionStartedAt time.Time `json:"current_session_started_at,omitempty"` + + // AccountsDB origin: snapshot info set once and never changed. Stage string `json:"stage"` // "ready", "downloading", "building", "corrupted" SnapshotSlot uint64 `json:"snapshot_slot"` // Slot of the snapshot used to build AccountsDB SnapshotEpoch uint64 `json:"snapshot_epoch,omitempty"` // Epoch of the snapshot @@ -58,11 +56,8 @@ type MithrilState struct { CorruptionReason string `json:"corruption_reason,omitempty"` CorruptionDetectedAt time.Time `json:"corruption_detected_at,omitempty"` - // ========================================================================= - // Manifest Seed Data (copied from manifest at snapshot build time) - // Used ONLY for fresh-start replay. Resume uses Last* fields instead. - // ========================================================================= - + // Manifest seed data is copied at snapshot build time. Fresh-start replay + // uses these fields; resume uses the Last* fields instead. // Block configuration seed ManifestParentSlot uint64 `json:"manifest_parent_slot,omitempty"` ManifestParentBankhash string `json:"manifest_parent_bankhash,omitempty"` // base58 @@ -106,18 +101,13 @@ type MithrilState struct { // Cleared after first replayed slot to save space. ManifestEpochStakes map[uint64]string `json:"manifest_epoch_stakes,omitempty"` - // ========================================================================= - // Current Position (where we left off) - // ========================================================================= + // Current position: where replay left off. LastSlot uint64 `json:"last_slot,omitempty"` // Last successfully replayed slot LastEpoch uint64 `json:"last_epoch,omitempty"` // Epoch of last replayed slot LastBankhash string `json:"last_bankhash,omitempty"` // Bankhash of last replayed slot (base58) LastBlockHeight uint64 `json:"last_block_height,omitempty"` // Block height of last replayed slot - // ========================================================================= - // Resume Context (everything needed to continue replay from LastSlot) - // These fields capture state at the end of the last successfully replayed slot - // ========================================================================= + // Resume context captured at the end of the last successfully replayed slot. // LtHash and fee state LastAcctsLtHash string `json:"last_accts_lt_hash,omitempty"` // base64 encoded cumulative LtHash @@ -147,10 +137,7 @@ type MithrilState struct { // These are NOT loaded from manifest - they are computed during replay and persisted. ComputedEpochStakes map[uint64]string `json:"computed_epoch_stakes,omitempty"` - // ========================================================================= - // Legacy fields - kept for backwards compatibility - // ========================================================================= - // TODO: Remove after v1.0 release + // Legacy fields kept for backwards compatibility. LastCommit string `json:"last_commit,omitempty"` // Deprecated: use last_writer_commit LastRunID string `json:"last_run_id,omitempty"` // Deprecated: use current_run_id LastRunAt time.Time `json:"last_run_at,omitempty"` // Deprecated: tracked via last_shutdown_at @@ -164,6 +151,7 @@ const ( ShutdownReasonLeaderSchedule = "leader schedule fetch failed from all RPC endpoints" ShutdownReasonError = "replay error" // Will be suffixed with actual error ShutdownReasonCompleted = "replay completed - reached end slot" + ShutdownReasonStartupFailed = "startup failed before replay" ) // BlockhashEntry represents a single entry in the RecentBlockhashes sysvar @@ -183,9 +171,8 @@ type ManifestFeeRateGovernorSeed struct { BurnPercent byte `json:"burn_percent"` } -// ManifestEpochScheduleSeed contains the bank epoch schedule serialized in the -// snapshot manifest. Some clusters can expose a divergent EpochSchedule sysvar -// account, so replay uses this bank schedule for epoch/leader/rewards logic. +// ManifestEpochScheduleSeed is the bank epoch schedule from the snapshot manifest. +// Replay uses it (not the sysvar account, which can diverge) for epoch/leader/rewards. type ManifestEpochScheduleSeed struct { SlotsPerEpoch uint64 `json:"slots_per_epoch"` LeaderScheduleSlotOffset uint64 `json:"leader_schedule_slot_offset"` @@ -245,7 +232,7 @@ func (s *MithrilState) Save(accountsDbDir string) error { // Write to temp file first, then rename for atomicity tmpFile := stateFile + ".tmp" - if err := os.WriteFile(tmpFile, data, 0644); err != nil { + if err := os.WriteFile(tmpFile, data, 0600); err != nil { return fmt.Errorf("failed to write state file: %w", err) } @@ -267,6 +254,43 @@ func (s *MithrilState) IsCorrupted() bool { return s != nil && s.Stage == "corrupted" } +// StartSession stamps CurrentSessionStartedAt and persists. Called once at +// startup (runLive, after LoadState) so WasCleanExit only trusts a shutdown +// recorded after this session began. +func (s *MithrilState) StartSession(accountsDbDir string) error { + s.CurrentSessionStartedAt = time.Now() + return s.Save(accountsDbDir) +} + +// WasCleanExit reports (clean, reason) for the prior shutdown; legacy files (zero start time) trust the reason directly. +func WasCleanExit(s *MithrilState) (bool, string) { + if s == nil { + return false, "no state file" + } + if s.CurrentSessionStartedAt.IsZero() { + if s.LastShutdownReason == "" { + return false, "legacy state file: no shutdown reason recorded" + } + return isCleanShutdownReason(s.LastShutdownReason), + "legacy state file: " + s.LastShutdownReason + } + if s.LastShutdownAt.IsZero() || s.LastShutdownAt.Before(s.CurrentSessionStartedAt) { + return false, "session crashed (no shutdown recorded after start)" + } + if s.LastShutdownReason == "" { + return false, "shutdown recorded without reason" + } + return isCleanShutdownReason(s.LastShutdownReason), s.LastShutdownReason +} + +// isCleanShutdownReason returns true when reason matches a constant that +// represents an orderly exit (no AccountsDB rebuild required). +func isCleanShutdownReason(reason string) bool { + return reason == ShutdownReasonNormal || + reason == ShutdownReasonCompleted || + strings.HasPrefix(reason, ShutdownReasonStartupFailed) +} + // MarkCorrupted updates the state file to indicate AccountsDB is corrupted. // This persists the corruption status so the next startup knows to rebuild. func (s *MithrilState) MarkCorrupted(accountsDbDir string, reason string) error { @@ -413,6 +437,36 @@ func (s *MithrilState) UpdateOnShutdown(accountsDbDir string, slot uint64, bankh return s.Save(accountsDbDir) } +// RecordSessionShutdown records a shutdown for a session that never persisted a +// new replay slot (stopped during startup/catchup). It leaves the prior replay +// position intact while marking the session so it isn't misread as a crash. +func (s *MithrilState) RecordSessionShutdown(accountsDbDir string, ctx *ShutdownContext) error { + if s == nil { + return fmt.Errorf("nil state") + } + if ctx != nil { + if s.RootRunID == "" { + s.RootRunID = ctx.RunID + } + if s.CurrentRunID != "" && s.CurrentRunID != ctx.RunID { + s.ParentRunID = s.CurrentRunID + } + s.CurrentRunID = ctx.RunID + s.LastWriterVersion = ctx.WriterVersion + s.LastWriterCommit = ctx.WriterCommit + s.LastWriterBranch = ctx.WriterBranch + s.LastCommit = ctx.WriterCommit + if ctx.ShutdownReason != "" { + s.LastShutdownReason = ctx.ShutdownReason + s.LastShutdownAt = time.Now() + } + s.LastRunID = ctx.RunID + s.LastRunAt = time.Now() + } + s.StateSchemaVersion = CurrentStateSchemaVersion + return s.Save(accountsDbDir) +} + // HasResumeData returns true if the state has resume context stored. // This indicates the state was saved during a graceful shutdown with full context. func (s *MithrilState) HasResumeData() bool { @@ -526,18 +580,19 @@ func NewReadyState(snapshotSlot uint64, snapshotEpoch uint64, fullSnapshotPath s // NewReadyStateWithOpts creates a new state with full options including cluster and version info. func NewReadyStateWithOpts(opts NewReadyStateOpts) *MithrilState { state := &MithrilState{ - StateSchemaVersion: CurrentStateSchemaVersion, - Stage: "ready", - SnapshotSlot: opts.SnapshotSlot, - SnapshotEpoch: opts.SnapshotEpoch, - BuildCompleted: time.Now(), - BuildStartedAt: opts.BuildStartedAt, - BuildMode: opts.BuildMode, - Cluster: opts.Cluster, - GenesisHash: opts.GenesisHash, - LastWriterVersion: opts.WriterVersion, - LastWriterCommit: opts.WriterCommit, - LastCommit: opts.WriterCommit, // Also set legacy field + StateSchemaVersion: CurrentStateSchemaVersion, + Stage: "ready", + SnapshotSlot: opts.SnapshotSlot, + SnapshotEpoch: opts.SnapshotEpoch, + BuildCompleted: time.Now(), + BuildStartedAt: opts.BuildStartedAt, + BuildMode: opts.BuildMode, + Cluster: opts.Cluster, + GenesisHash: opts.GenesisHash, + LastWriterVersion: opts.WriterVersion, + LastWriterCommit: opts.WriterCommit, + LastCommit: opts.WriterCommit, // Also set legacy field + CurrentSessionStartedAt: time.Now(), // session that built this state } if opts.FullSnapshotPath != "" { diff --git a/pkg/state/state_test.go b/pkg/state/state_test.go new file mode 100644 index 000000000..195ff254f --- /dev/null +++ b/pkg/state/state_test.go @@ -0,0 +1,218 @@ +package state + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// minimalValidState populates the fields LoadState and Save require. +func minimalValidState() *MithrilState { + return &MithrilState{ + StateSchemaVersion: CurrentStateSchemaVersion, + Stage: "ready", + SnapshotSlot: 100, + } +} + +// StartSession sets CurrentSessionStartedAt within [before, after]. +func TestStartSession_StampsCurrentTime(t *testing.T) { + dir := t.TempDir() + s := minimalValidState() + + before := time.Now() + require.NoError(t, s.StartSession(dir)) + after := time.Now() + + assert.False(t, s.CurrentSessionStartedAt.IsZero(), "timestamp should be set") + assert.False(t, s.CurrentSessionStartedAt.Before(before), "timestamp should be >= before") + assert.False(t, s.CurrentSessionStartedAt.After(after), "timestamp should be <= after") +} + +// CurrentSessionStartedAt survives a Save -> LoadState round-trip. +func TestStartSession_PersistsAcrossLoad(t *testing.T) { + dir := t.TempDir() + s := minimalValidState() + require.NoError(t, s.StartSession(dir)) + saved := s.CurrentSessionStartedAt + + loaded, err := LoadState(dir) + require.NoError(t, err) + require.NotNil(t, loaded) + assert.True(t, loaded.CurrentSessionStartedAt.Equal(saved), + "loaded timestamp should equal saved (got %v, want %v)", + loaded.CurrentSessionStartedAt, saved) +} + +// a second StartSession replaces the prior timestamp (one session per process). +func TestStartSession_OverwritesPriorValue(t *testing.T) { + dir := t.TempDir() + s := minimalValidState() + require.NoError(t, s.StartSession(dir)) + first := s.CurrentSessionStartedAt + time.Sleep(2 * time.Millisecond) + require.NoError(t, s.StartSession(dir)) + second := s.CurrentSessionStartedAt + + assert.True(t, second.After(first), "second StartSession should advance the timestamp") +} + +// a file without current_session_started_at loads with a zero-value field. +func TestLoadState_LegacyFileWithoutSessionField(t *testing.T) { + dir := t.TempDir() + legacyJSON := []byte(`{ + "state_schema_version": 2, + "stage": "ready", + "snapshot_slot": 100, + "last_shutdown_reason": "graceful shutdown (Ctrl+C)", + "last_shutdown_at": "2026-05-01T12:00:00Z" + }`) + require.NoError(t, os.WriteFile(filepath.Join(dir, StateFileName), legacyJSON, 0644)) + + loaded, err := LoadState(dir) + require.NoError(t, err) + require.NotNil(t, loaded) + assert.True(t, loaded.CurrentSessionStartedAt.IsZero(), + "legacy file should leave CurrentSessionStartedAt zero") + assert.Equal(t, "graceful shutdown (Ctrl+C)", loaded.LastShutdownReason) +} + +// TestWasCleanExit_NilState returns (false, reason) without panicking. +func TestWasCleanExit_NilState(t *testing.T) { + clean, reason := WasCleanExit(nil) + assert.False(t, clean) + assert.NotEmpty(t, reason) +} + +// happy path: clean shutdown, consistent timestamps, clean reason constant. +func TestWasCleanExit_FreshSession_CleanShutdown(t *testing.T) { + start := time.Now().Add(-1 * time.Hour) + s := &MithrilState{ + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(30 * time.Minute), + LastShutdownReason: ShutdownReasonNormal, + } + clean, _ := WasCleanExit(s) + assert.True(t, clean) +} + +// the other clean constant (replay completed) is also accepted. +func TestWasCleanExit_FreshSession_CompletedReason(t *testing.T) { + start := time.Now().Add(-1 * time.Hour) + s := &MithrilState{ + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(30 * time.Minute), + LastShutdownReason: ShutdownReasonCompleted, + } + clean, _ := WasCleanExit(s) + assert.True(t, clean) +} + +func TestWasCleanExit_FreshSession_StartupFailedBeforeReplay(t *testing.T) { + start := time.Now().Add(-1 * time.Hour) + s := &MithrilState{ + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(30 * time.Minute), + LastShutdownReason: ShutdownReasonStartupFailed + ": invalid port", + } + clean, reason := WasCleanExit(s) + assert.True(t, clean) + assert.Contains(t, reason, ShutdownReasonStartupFailed) +} + +// session started but died before recording a shutdown (crash, OOM, SIGKILL). +func TestWasCleanExit_FreshSession_NoShutdownRecorded(t *testing.T) { + s := &MithrilState{ + CurrentSessionStartedAt: time.Now().Add(-30 * time.Minute), + // LastShutdownAt zero, LastShutdownReason empty + } + clean, reason := WasCleanExit(s) + assert.False(t, clean) + assert.NotEmpty(t, reason) +} + +// shutdown timestamp predates the session start: stale, current session crashed. +func TestWasCleanExit_ShutdownBeforeSessionStart(t *testing.T) { + priorShutdown := time.Now().Add(-2 * time.Hour) + thisStart := time.Now().Add(-1 * time.Hour) + s := &MithrilState{ + CurrentSessionStartedAt: thisStart, + LastShutdownAt: priorShutdown, + LastShutdownReason: ShutdownReasonNormal, // stale value from prior session + } + clean, reason := WasCleanExit(s) + assert.False(t, clean, "stale shutdown timestamp must not be treated as clean") + assert.NotEmpty(t, reason) +} + +// non-clean reasons (stall, etc.) stay non-clean even with correct timestamps. +func TestWasCleanExit_FreshSession_StallReason(t *testing.T) { + start := time.Now().Add(-1 * time.Hour) + s := &MithrilState{ + CurrentSessionStartedAt: start, + LastShutdownAt: start.Add(30 * time.Minute), + LastShutdownReason: ShutdownReasonStall, + } + clean, _ := WasCleanExit(s) + assert.False(t, clean, "stall is a recovery-required reason, not clean") +} + +// legacy files (no session start) take the shutdown reason at face value. +func TestWasCleanExit_LegacyFile_CleanReason(t *testing.T) { + s := &MithrilState{ + // CurrentSessionStartedAt is zero + LastShutdownAt: time.Now().Add(-30 * time.Minute), + LastShutdownReason: ShutdownReasonNormal, + } + clean, reason := WasCleanExit(s) + assert.True(t, clean, "legacy file with a clean reason should still be treated as clean") + assert.Contains(t, reason, "legacy") +} + +// legacy file with no reason returns false: cleanness can't be inferred. +func TestWasCleanExit_LegacyFile_EmptyReason(t *testing.T) { + s := &MithrilState{ + // Everything zero/empty + } + clean, reason := WasCleanExit(s) + assert.False(t, clean) + assert.NotEmpty(t, reason) +} + +func TestRecordSessionShutdown_PreservesReplayPosition(t *testing.T) { + dir := t.TempDir() + start := time.Now().Add(-1 * time.Minute) + s := &MithrilState{ + StateSchemaVersion: CurrentStateSchemaVersion, + Stage: "ready", + SnapshotSlot: 100, + LastSlot: 123, + LastBankhash: "prior-bankhash", + CurrentRunID: "previous-run", + CurrentSessionStartedAt: start, + } + + require.NoError(t, s.RecordSessionShutdown(dir, &ShutdownContext{ + RunID: "clean-early-stop", + WriterVersion: "test", + WriterCommit: "abcdef", + WriterBranch: "branch", + ShutdownReason: ShutdownReasonNormal, + })) + + loaded, err := LoadState(dir) + require.NoError(t, err) + require.NotNil(t, loaded) + assert.Equal(t, uint64(123), loaded.LastSlot) + assert.Equal(t, "prior-bankhash", loaded.LastBankhash) + assert.Equal(t, "previous-run", loaded.ParentRunID) + assert.Equal(t, "clean-early-stop", loaded.CurrentRunID) + + clean, reason := WasCleanExit(loaded) + assert.True(t, clean) + assert.Equal(t, ShutdownReasonNormal, reason) +}