diff --git a/Cargo.lock b/Cargo.lock index 9d76107..7f719a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1129,7 +1129,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "eldrun" -version = "0.1.31" +version = "0.1.32" dependencies = [ "arboard", "base64 0.22.1", diff --git a/package.json b/package.json index a3727b3..c5b149e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "eldrun", "private": true, - "version": "0.1.32", + "version": "0.1.33", "license": "MIT OR Apache-2.0", "type": "module", "scripts": { diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index e1c5304..933eaa8 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "eldrun" -version = "0.1.32" +version = "0.1.33" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/src-tauri/src/commands/git.rs b/src-tauri/src/commands/git.rs index b9ffada..c25a68c 100644 --- a/src-tauri/src/commands/git.rs +++ b/src-tauri/src/commands/git.rs @@ -396,6 +396,20 @@ fn git_file_statuses_blocking( } } + // When `rel_path` is itself wholly ignored, EVERY entry under it is ignored — + // but git's porcelain only ever emits *file* lines, so a child that is an + // empty directory (or a tree of only empty directories, e.g. freshly scaffolded + // experiment-output folders) produces no line at all and would silently drop + // its "ignored" marker, landing in the regular section. git also can't be + // coaxed into listing such dirs: it collapses a wholly-ignored subtree to its + // topmost ignored ancestor and never enumerates empty descendants. Emit a + // sentinel under the reserved key "." (never a real listing entry) so the + // frontend can default every child of this directory to ignored, letting any + // explicit per-child status above (a force-added tracked file) still win. + if wholly_ignored { + map.insert(".".to_string(), "ignored".to_string()); + } + Ok(map) } diff --git a/src-tauri/src/commands/projects.rs b/src-tauri/src/commands/projects.rs index 471b5da..8dfb155 100644 --- a/src-tauri/src/commands/projects.rs +++ b/src-tauri/src/commands/projects.rs @@ -1017,6 +1017,56 @@ fn patch_compute_hosts( Ok(hosts) } +/// Persist which machine shells launched from this project run on — the choice +/// made in the `RunHostPicker`. Mirrors `set_project_python`: it writes BOTH the +/// `projects.json` entry's `extra["run_host"]` (what the frontend seeds its live +/// preference store from on load) and the `project.json` `run_host` field (keeps +/// it with the project). `location` is a `TabLocation` string (`"local"` / +/// `"remote"` / `"host:"`); `None` or empty clears it (back to the primary +/// default). Returns the stored value. +#[tauri::command] +pub fn set_project_run_host( + project_id: String, + location: Option, +) -> Result, String> { + let value = location + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + let list_path = storage::state_dir().join("projects.json"); + let mut list: ProjectsList = if list_path.exists() { + storage::read_json(&list_path).map_err(|e| e.to_string())? + } else { + Vec::new() + }; + let entry = list + .iter_mut() + .find(|p| p.id == project_id) + .ok_or_else(|| format!("project '{project_id}' not found"))?; + + match &value { + Some(v) => { + entry + .extra + .insert("run_host".into(), serde_json::Value::String(v.clone())); + } + None => { + entry.extra.remove("run_host"); + } + } + let local_file = entry.local_file.clone(); + storage::write_json(&list_path, &list).map_err(|e| e.to_string())?; + + let proj_path = PathBuf::from(&local_file); + if proj_path.exists() { + if let Ok(mut project) = storage::read_json::(&proj_path) { + project.run_host = value.clone(); + storage::write_json(&proj_path, &project).map_err(|e| e.to_string())?; + } + } + Ok(value) +} + /// Add an extra "worker" host to a remote project (the pill's "Add machine"). /// Mints a stable `id` when the incoming spec has none, then appends it. Returns /// the full updated host list. The primary (`Project.remote`) is untouched. diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8aeb19a..264efe1 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -601,6 +601,7 @@ pub fn run() { commands::projects::add_compute_host, commands::projects::remove_compute_host, commands::projects::patch_compute_host, + commands::projects::set_project_run_host, commands::projects::set_project_categories, commands::projects::set_project_git_disabled, commands::projects::save_tab_layout, diff --git a/src-tauri/src/schema/project.rs b/src-tauri/src/schema/project.rs index 4765726..43c5040 100644 --- a/src-tauri/src/schema/project.rs +++ b/src-tauri/src/schema/project.rs @@ -339,6 +339,15 @@ pub struct Project { /// entry's `extra["python_interpreter"]`. #[serde(skip_serializing_if = "Option::is_none")] pub python_interpreter: Option, + /// Which machine shells launched from this project run on — the choice made in + /// the `RunHostPicker`, persisted so it survives a relaunch (a Run/Debug or a + /// new shell tab lands on it instead of the primary). A `TabLocation` string: + /// `"local"` (the mirror), `"remote"` (the primary host), or `"host:"` (a + /// worker). Absent = the shell default (the primary). Mirrored into the + /// `projects.json` entry's `extra["run_host"]`, which is what the frontend + /// seeds its live preference store from on load. + #[serde(skip_serializing_if = "Option::is_none")] + pub run_host: Option, #[serde(flatten)] pub extra: HashMap, } diff --git a/src-tauri/src/services/openvpn.rs b/src-tauri/src/services/openvpn.rs index 515b95d..da5805f 100644 --- a/src-tauri/src/services/openvpn.rs +++ b/src-tauri/src/services/openvpn.rs @@ -507,7 +507,7 @@ pub fn openvpn_available() -> bool { /// encrypted private key. /// /// Shape: `--config [--auth-user-pass ] [--askpass ] --auth-nocache -/// --writepid --connect-timeout 20 --connect-retry-max 3 +/// --writepid --connect-timeout 20 --persist-tun /// --verb 3 --mute 0`. pub fn openvpn_args( config: &str, @@ -535,8 +535,27 @@ pub fn openvpn_args( pidfile.to_string_lossy().into_owned(), "--connect-timeout".to_string(), "20".to_string(), - "--connect-retry-max".to_string(), - "3".to_string(), + // NO `--connect-retry-max`. That option is *not* an initial-connect + // guard — it caps reconnect attempts for the tunnel's whole lifetime, so + // the `3` this used to pass meant any mid-session blip (a rekey failure, + // a suspended laptop, a server restart) burned the three retries and + // OpenVPN **exited for good**, leaving the machine's routing changed and + // every SSH/SFTP session on top of it stalled against a peer that would + // never answer. Unlimited (the OpenVPN default) is what makes a tunnel + // survive a blip. The *initial* connect is still bounded — from the + // outside, by `CONNECT_TIMEOUT` + the `child.kill()` on its Err — so a + // doomed connect cannot retry forever behind the user's back. + // + // `--persist-tun` keeps the tun device and its routes across those + // reconnects, which is what lets a connection on top of the tunnel ride + // one out instead of breaking. It also makes the failure mode + // fail-*closed*: while OpenVPN is re-handshaking, traffic is black-holed + // rather than leaking out the physical default route. That is the right + // trade for a VPN, and it is only safe now that the tunnel no longer + // gives up permanently (above) and that a tunnel dying is *detected* + // rather than left showing a green lamp (`stores/vpnStatus`'s drop + // reconcile). + "--persist-tun".to_string(), // Readiness is detected by watching OpenVPN's output for READY_MARKER, // so logging must be deterministic no matter what the config says: a // config's `mute N` suppresses consecutive same-category lines — the @@ -2543,6 +2562,31 @@ mod tests { assert!(args.iter().any(|a| a == "--auth-nocache")); } + /// A tunnel must survive a blip. `--connect-retry-max` caps reconnects for + /// the tunnel's whole *lifetime*, not just the first connect, so passing it + /// meant a momentary drop exhausted the retries and OpenVPN exited for good + /// — leaving the machine's routing changed and every SSH/SFTP session on top + /// stalled against a peer that would never answer. It must stay absent, with + /// `--persist-tun` holding the device and routes across a reconnect. + #[test] + fn openvpn_args_let_a_dropped_tunnel_reconnect_forever() { + let args = openvpn_args( + "/home/u/work.ovpn", + None, + Some(Path::new("/run/eldrun/openvpn/x.pass")), + Path::new("/run/eldrun/openvpn/x.pid"), + ) + .unwrap(); + assert!( + !args.iter().any(|a| a == "--connect-retry-max"), + "a retry cap makes a blip kill the tunnel permanently: {args:?}" + ); + assert!(args.iter().any(|a| a == "--persist-tun")); + // The initial connect stays bounded by CONNECT_TIMEOUT, not by openvpn. + let ti = args.iter().position(|a| a == "--connect-timeout").unwrap(); + assert_eq!(args[ti + 1], "20"); + } + /// Readiness is detected by scanning OpenVPN's output for the /// "Initialization Sequence Completed" marker, so the argv must pin the /// logging knobs a config could otherwise sabotage it with: a `mute 16` diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index a72d31e..412c483 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,6 +1,6 @@ { "productName": "Eldrun", - "version": "0.1.32", + "version": "0.1.33", "identifier": "io.github.fseiffarth.eldrun", "build": { "frontendDist": "../dist", diff --git a/src/__tests__/DirSizeGuard.test.ts b/src/__tests__/DirSizeGuard.test.ts new file mode 100644 index 0000000..a5676d8 --- /dev/null +++ b/src/__tests__/DirSizeGuard.test.ts @@ -0,0 +1,145 @@ +/** + * The file tree's folder sizes must never be able to freeze the app. + * + * They are a display aid — the tree renders fine without them — but for a remote + * project each one is a `du` over SSH, and the tree fires one per visible folder + * at once. When the network under the project goes away (a dropped VPN tunnel, + * a suspended laptop) those calls do not *fail*: a black-holed socket returns no + * error, so each blocks until ssh's keepalive gives up ~45s later, holding a + * backend blocking-pool thread the whole time. Opening one folder of thirty + * subfolders then meant thirty stalled threads and a frozen UI. + * + * These lock the three limits that prevent that, and — just as importantly — the + * cases where the guard must stay out of the way. + */ +import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; + +import { + DirSizeUnavailable, + FAILURES_TO_OPEN, + MAX_CONCURRENT, + OPEN_MS, + TIMEOUT_MS, + guardedDirSize, + isHostTimeout, + resetDirSizeGuard, +} from "../lib/dirSizeGuard"; + +const PROJECT = "/home/u/eldrun/projects/demo"; + +/** A call that never settles — what a black-holed SSH round trip looks like. */ +const hangs = () => new Promise(() => {}); + +beforeEach(() => { + resetDirSizeGuard(); + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); + resetDirSizeGuard(); +}); + +describe("timeout", () => { + it("gives up on a call that never answers instead of hanging forever", async () => { + const settled = vi.fn(); + const p = guardedDirSize(PROJECT, hangs).then(settled, settled); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS - 1); + expect(settled).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2); + await p; + expect(settled).toHaveBeenCalledOnce(); + expect(isHostTimeout(settled.mock.calls[0][0])).toBe(true); + }); + + it("passes a prompt answer straight through", async () => { + await expect(guardedDirSize(PROJECT, () => Promise.resolve(4096))).resolves.toBe(4096); + }); +}); + +describe("concurrency cap", () => { + it("holds calls beyond the cap back rather than dispatching them all at once", async () => { + const started = vi.fn(); + for (let i = 0; i < MAX_CONCURRENT + 3; i += 1) { + void guardedDirSize(PROJECT, () => { + started(); + return hangs(); + }).catch(() => {}); + } + await vi.advanceTimersByTimeAsync(0); + // The point: a wide folder must not fire one call per subfolder into a + // network that isn't answering. + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT); + }); + + it("hands a freed slot to the next waiter", async () => { + const started = vi.fn(); + for (let i = 0; i < MAX_CONCURRENT + 1; i += 1) { + void guardedDirSize(PROJECT, () => { + started(); + return hangs(); + }).catch(() => {}); + } + await vi.advanceTimersByTimeAsync(0); + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT); + // First in flight times out → its slot goes to the queued call. + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + expect(started).toHaveBeenCalledTimes(MAX_CONCURRENT + 1); + }); +}); + +describe("circuit breaker", () => { + /** Time out `n` calls back to back. */ + async function timeOut(n: number) { + for (let i = 0; i < n; i += 1) { + const p = guardedDirSize(PROJECT, hangs).catch((e) => e); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + await p; + } + } + + it("refuses instantly once the host has repeatedly failed to answer", async () => { + await timeOut(FAILURES_TO_OPEN); + // This is the whole point: the 30th folder costs nothing at all, instead of + // each one paying the timeout in turn. + const run = vi.fn(hangs); + await expect(guardedDirSize(PROJECT, run)).rejects.toBeInstanceOf(DirSizeUnavailable); + expect(run).not.toHaveBeenCalled(); + }); + + it("does not open on failures that are not timeouts", async () => { + // An unreadable folder answered fine — it says nothing about the host, so it + // must not push the project toward a trip. + for (let i = 0; i < FAILURES_TO_OPEN + 2; i += 1) { + await guardedDirSize(PROJECT, () => Promise.reject(new Error("permission denied"))).catch( + () => {}, + ); + } + await expect(guardedDirSize(PROJECT, () => Promise.resolve(7))).resolves.toBe(7); + }); + + it("lets a retry through once the open window elapses", async () => { + await timeOut(FAILURES_TO_OPEN); + await vi.advanceTimersByTimeAsync(OPEN_MS + 1); + await expect(guardedDirSize(PROJECT, () => Promise.resolve(11))).resolves.toBe(11); + }); + + it("closes again on the first success, so one good answer restores the tree", async () => { + await timeOut(FAILURES_TO_OPEN - 1); + await expect(guardedDirSize(PROJECT, () => Promise.resolve(1))).resolves.toBe(1); + // The earlier timeouts are forgiven — the next timeout starts a fresh count + // rather than tripping the breaker immediately. + const p = guardedDirSize(PROJECT, hangs).catch((e) => e); + await vi.advanceTimersByTimeAsync(TIMEOUT_MS + 1); + await p; + const run = vi.fn(() => Promise.resolve(2)); + await expect(guardedDirSize(PROJECT, run)).resolves.toBe(2); + expect(run).toHaveBeenCalledOnce(); + }); + + it("is per project — one dead host does not stop another project's tree", async () => { + await timeOut(FAILURES_TO_OPEN); + await expect(guardedDirSize(PROJECT, hangs)).rejects.toBeInstanceOf(DirSizeUnavailable); + await expect(guardedDirSize("/home/u/other", () => Promise.resolve(99))).resolves.toBe(99); + }); +}); diff --git a/src/__tests__/ShellScriptRun.test.ts b/src/__tests__/ShellScriptRun.test.ts index 6ea5006..4ee1d49 100644 --- a/src/__tests__/ShellScriptRun.test.ts +++ b/src/__tests__/ShellScriptRun.test.ts @@ -51,6 +51,46 @@ describe("shell script run planning", () => { }); }); + it("runs on the chosen worker machine when a run-host preference is set", () => { + const plan = shellScriptRunPlan({ + project: remoteProject, + treeRoot: "/state/simplegnn", + syncSource: "remote", + scriptPath: "/home/alice/simplegnn/train.sh", + interp: "bash", + runHostPref: "host:mlai20", + }); + + // The chosen machine wins over the browsed side; the script path stays + // project-relative so it resolves against that host's own project root (the + // backend re-cds into the worker's remote_path). + expect(plan).toMatchObject({ + cwd: "/home/alice/simplegnn", + scriptRel: "train.sh", + initialInput: "bash 'train.sh'", + location: "host:mlai20", + }); + }); + + it("sends a mirror-browsed script to the chosen remote machine", () => { + // Browsing the local mirror but with a worker chosen: the run still lands on + // the worker, and scriptRel (project-relative) resolves there. + const plan = shellScriptRunPlan({ + project: remoteProject, + treeRoot: "/state/simplegnn/mirror", + syncSource: "local", + scriptPath: "/state/simplegnn/mirror/train.sh", + interp: "bash", + runHostPref: "host:mlai20", + }); + + expect(plan).toMatchObject({ + cwd: "/home/alice/simplegnn", + scriptRel: "train.sh", + location: "host:mlai20", + }); + }); + it("refuses to build bash with an empty script path", () => { expect( shellScriptRunPlan({ diff --git a/src/__tests__/VpnDropGating.test.ts b/src/__tests__/VpnDropGating.test.ts new file mode 100644 index 0000000..6b85f52 --- /dev/null +++ b/src/__tests__/VpnDropGating.test.ts @@ -0,0 +1,115 @@ +/** + * A tunnel that dies on its own must be *noticed*, and noticing it must gate the + * SSH/SFTP work belonging to the projects that were riding it. + * + * This is the failure this exists for: OpenVPN exits (it used to do so + * permanently after three failed reconnects — see `openvpn_args`), the machine's + * routing is left pointing into a tunnel with nothing behind it, and every + * pooled-SSH call a remote project makes now aims at a peer that will never + * answer. Those calls are synchronous Tauri commands, and a black-holed socket + * yields no error, so each blocks ~45s until ssh's keepalive gives up. The file + * tree issues one per visible folder. The app froze. + * + * `refresh()` is the only thing that reconciles against the backend, so the drop + * has to be detected *there* — and it has to be distinguishable from a + * deliberate disconnect, which must not be reported as a drop. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +import { invoke } from "@tauri-apps/api/core"; + +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn(() => Promise.resolve()) })); + +import { useVpnStatusStore, markVpnConnected, disconnectVpnTunnel } from "../stores/vpnStatus"; +import { useRemoteStatusStore } from "../stores/remoteStatus"; + +const invokeMock = vi.mocked(invoke); +const CONFIG = "/store/office.ovpn"; +const PROJECT = "p-remote"; + +/** What the backend reports as live on the next reconcile. */ +function backendReports(configs: string[]) { + invokeMock.mockImplementation((cmd: string) => { + if (cmd === "openvpn_active") return Promise.resolve(configs); + return Promise.resolve(); + }); +} + +beforeEach(() => { + invokeMock.mockReset(); + useVpnStatusStore.setState({ byConfig: {}, holders: {} }); + useRemoteStatusStore.setState({ byProject: {}, byHost: {} }); +}); + +describe("a tunnel that dies unasked", () => { + beforeEach(() => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + }); + + it("is detected by the reconcile and clears the holder's remote status", async () => { + backendReports([]); // the tunnel is simply gone + await useVpnStatusStore.getState().refresh(); + + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBeUndefined(); + // The gate: with the project's status cleared, `useRemoteBlocked` stops the + // SFTP/git probes being dispatched at all, rather than each paying ~45s. + expect(useRemoteStatusStore.getState().byProject[PROJECT]).toBeUndefined(); + }); + + it("releases the claims, so a later reconnect is not seen as already held", async () => { + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useVpnStatusStore.getState().holders[CONFIG]).toBeUndefined(); + }); + + it("leaves a project that never claimed the tunnel alone", async () => { + // It reaches its host by some other route; this tunnel is none of its business. + useRemoteStatusStore.getState().setSsh("p-other", "connected"); + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject["p-other"]?.ssh).toBe("connected"); + }); +}); + +describe("everything that is not a drop", () => { + it("does not fire for a tunnel that is still up", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + backendReports([CONFIG]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); + + it("does not fire for a deliberate disconnect", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + // A UI teardown forgets the config first, so the next reconcile sees neither + // side and must not mistake it for a death. + disconnectVpnTunnel(CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + backendReports([]); + await useVpnStatusStore.getState().refresh(); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); + + it("does not fire for a tunnel still mid-handshake", async () => { + useVpnStatusStore.getState().setState(CONFIG, "connecting"); + useVpnStatusStore.getState().acquire(CONFIG, PROJECT); + useRemoteStatusStore.getState().setSsh(PROJECT, "connecting"); + backendReports([]); // not in the registry yet — that is normal, not a death + await useVpnStatusStore.getState().refresh(); + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBe("connecting"); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connecting"); + }); + + it("does nothing when the backend cannot be reached at all", async () => { + markVpnConnected(PROJECT, CONFIG); + useRemoteStatusStore.getState().setSsh(PROJECT, "connected"); + invokeMock.mockImplementation(() => Promise.reject(new Error("backend gone"))); + await useVpnStatusStore.getState().refresh(); + // No answer is not evidence the tunnel died — the state must be left alone. + expect(useVpnStatusStore.getState().byConfig[CONFIG]).toBe("connected"); + expect(useRemoteStatusStore.getState().byProject[PROJECT]?.ssh).toBe("connected"); + }); +}); diff --git a/src/components/embed/FileViewerPane.tsx b/src/components/embed/FileViewerPane.tsx index 5fb47da..5b831c6 100644 --- a/src/components/embed/FileViewerPane.tsx +++ b/src/components/embed/FileViewerPane.tsx @@ -30,6 +30,7 @@ import { PresentationOverlay } from "./PresentationOverlay"; import { renderMarkdown } from "../../lib/viewers/markdown"; import { enrichMarkdownDom } from "../../lib/viewers/markdownEnrich"; import { highlight, languageForPath, escapeHtml } from "../../lib/viewers/highlight"; +import { useOllamaStatus } from "../../lib/ollamaStatus"; import { printDocument, printHtmlBody, @@ -64,6 +65,7 @@ import { snapBreakpointLine, } from "../../lib/viewers/python"; import { debugPythonFile, runCwd, runPythonFile, placeForFocused } from "../../lib/pythonRun"; +import { RunHostPicker } from "../tabs/TabLocalityBadges"; import { isSlurmScript, parseSbatchDirectives, @@ -4143,30 +4145,16 @@ function useTabAiPrefs(tabKey: string | undefined, type: InternalViewer): TabAiP * Both AI-assist features the controls expose (autocomplete + grammar) run only * against a resident model, so the controls hide themselves entirely when none * is loaded. Mirrors the lamp logic in `LocalModelMenu`: `ollama_status` is - * `"loaded"` iff `/api/ps` reports a resident model. Polled on the same 5s - * cadence as the 🧠 menu so the controls appear/disappear within a few seconds - * of a model being warmed or unloaded out of band. + * `"loaded"` iff `/api/ps` reports a resident model. + * + * Rides the app-wide shared poller (`lib/ollamaStatus`) rather than owning a + * timer. This hook runs **per editable viewer tab**, so a private 5s interval + * made the `/api/ps` request rate a function of how many tabs happened to be + * open — asking the same machine-wide question N times over. One timer now + * serves every surface, and they all flip on the same observation. */ function useLocalModelLoaded(): boolean { - const [loaded, setLoaded] = useState(false); - useEffect(() => { - let cancelled = false; - const check = () => - invoke<"stopped" | "idle" | "loaded">("ollama_status") - .then((s) => { - if (!cancelled) setLoaded(s === "loaded"); - }) - .catch(() => { - if (!cancelled) setLoaded(false); - }); - void check(); - const id = window.setInterval(check, 5000); - return () => { - cancelled = true; - window.clearInterval(id); - }; - }, []); - return loaded; + return useOllamaStatus() === "loaded"; } /** @@ -5253,6 +5241,21 @@ function TextView({ /> )} + {/* Which machine a Run/Debug lands on — shown right next to the Run button + so the choice is co-located with running, and keyed by THIS viewer's + `projectId`, the exact scope `onRun` reads the preference back under (a + picker in the file panel is keyed by the *active* project, which can + differ from the viewed file's project → the choice would be dropped and + the run would fall back to the primary). Only for a remote project with + extra worker machines; a lone-primary project has no machine to pick. */} + {showEditor && pyRun && isRemoteProject && projectId && + (project?.compute_hosts?.length ?? 0) > 0 && ( + + )} {showEditor && pyRun && ( { + if (gitStatuses["."] !== "ignored") return gitStatuses; + const m: GitStatusMap = {}; + for (const e of entries) m[e.name] = gitStatuses[e.name] ?? "ignored"; + return m; + }, [gitStatuses, entries]); + // The three on-screen sections in render order (regular, then the collapsible // scaffold + gitignored groups). Shared by the renderer and the selection // click handler so a shift-range spans exactly what the user sees. Mirrors the @@ -567,10 +584,10 @@ export function FileTree({ const isRoot = !relPath && separateScaffold; const nonStandard = isRoot ? entries.filter((e) => !STANDARD_PROJECT_FILES.has(e.name)) : entries; const standard = isRoot ? entries.filter((e) => STANDARD_PROJECT_FILES.has(e.name)) : []; - const regular = nonStandard.filter((e) => gitStatuses[e.name] !== "ignored"); - const gitignored = nonStandard.filter((e) => gitStatuses[e.name] === "ignored"); + const regular = nonStandard.filter((e) => displayStatuses[e.name] !== "ignored"); + const gitignored = nonStandard.filter((e) => displayStatuses[e.name] === "ignored"); return { regular, standard, gitignored }; - }, [entries, relPath, separateScaffold, gitStatuses]); + }, [entries, relPath, separateScaffold, displayStatuses]); // Flat list of visible row paths in on-screen order — the axis a shift-range // spans. Collapsed sections contribute no rows (they aren't rendered). @@ -664,7 +681,13 @@ export function FileTree({ pending.forEach((e) => requestedSizes.current.add(e.path)); const gen = sizeGeneration.current; for (const e of pending) { - invoke("dir_size", { projectDir, relPath: relForEntry(e), excluded: scanExcludedList }) + guardedDirSize(projectDir, () => + invoke("dir_size", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then((bytes) => { if (sizeGeneration.current === gen) setDirSizes((m) => ({ ...m, [e.path]: bytes })); }) @@ -693,22 +716,34 @@ export function FileTree({ pending.forEach((e) => requestedSizes.current.add(e.path)); const gen = sizeGeneration.current; for (const e of pending) { - invoke<{ total: number; ignored: number }>("dir_size_breakdown", { - projectDir, - relPath: relForEntry(e), - excluded: scanExcludedList, - }) + guardedDirSize(projectDir, () => + invoke<{ total: number; ignored: number }>("dir_size_breakdown", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then(({ total, ignored }) => { if (sizeGeneration.current !== gen) return; setDirSizes((m) => ({ ...m, [e.path]: total })); if (ignored > 0) setDirIgnoredBytes((m) => ({ ...m, [e.path]: ignored })); }) - .catch(() => { - // Fall back to the plain total rather than showing no size at all — - // `dir_size_breakdown` can fail for reasons `dir_size` wouldn't (a - // backend that hasn't picked up this new command yet, no `git` on - // PATH), and a folder's size is more important than its ignored split. - invoke("dir_size", { projectDir, relPath: relForEntry(e), excluded: scanExcludedList }) + .catch((err) => { + // The fallback is for a `dir_size_breakdown` that failed for a reason + // `dir_size` wouldn't (a backend that hasn't picked up this newer + // command, no `git` on PATH) — there, a folder's size matters more than + // its ignored split. It is NOT for a host that stopped answering: a + // timed-out or breaker-refused call says nothing about the *command*, + // so retrying with the other one would just queue a second doomed call + // per folder — twice the pile-up the guard exists to prevent. + if (err instanceof DirSizeUnavailable || isHostTimeout(err)) return; + guardedDirSize(projectDir, () => + invoke("dir_size", { + projectDir, + relPath: relForEntry(e), + excluded: scanExcludedList, + }), + ) .then((bytes) => { if (sizeGeneration.current === gen) setDirSizes((m) => ({ ...m, [e.path]: bytes })); }) @@ -2151,6 +2186,11 @@ export function FileTree({ syncSource, scriptPath: entry.path, interp, + // Run on the machine chosen in the RunHostPicker (the same preference the + // Python Run honours), so "pick machine X ⇒ all shells run on X". + runHostPref: projectId + ? useRunHostPrefStore.getState().byProject[projectId] + : undefined, }); if (!plan) { setError(`Run failed: ${entry.path} is outside the current project tree`); @@ -2570,7 +2610,7 @@ export function FileTree({ const { regular, standard, gitignored } = sections; function renderEntry(e: FileEntry, isScaffold = false, isGitignored = false) { - const status = isGitignored ? undefined : gitStatuses[e.name]; + const status = isGitignored ? undefined : displayStatuses[e.name]; const sizeClass = !e.is_dir ? sizeCategory(e.size) : ""; // A folder's headline number is its non-ignored weight — the part // that's actually "yours" — with the full total + ignored split @@ -2997,7 +3037,7 @@ export function FileTree({ ); } - const status = gitStatuses[entry.name]; + const status = displayStatuses[entry.name]; const isTex = !entry.is_dir && entry.extension === ".tex"; const isZip = !entry.is_dir && entry.extension === ".zip"; const entryRel = relForEntry(entry); @@ -3344,7 +3384,7 @@ export function FileTree({ {/* Same rule as the row: no ignored/total split for a folder that is itself ignored — "412 MB of 412 MB ignored" says nothing. */} {tooltip.entry.is_dir - && gitStatuses[tooltip.entry.name] !== "ignored" + && displayStatuses[tooltip.entry.name] !== "ignored" && dirIgnoredBytes[tooltip.entry.path] !== undefined && (
Ignored diff --git a/src/components/files/ProjectFilesView.tsx b/src/components/files/ProjectFilesView.tsx index aabef9b..389ee29 100644 --- a/src/components/files/ProjectFilesView.tsx +++ b/src/components/files/ProjectFilesView.tsx @@ -11,7 +11,6 @@ import { useRemoteBlocked, } from "./ProjectFilesPane"; import { RunHostPicker } from "../tabs/TabLocalityBadges"; -import { runHostPickerApplies } from "../../stores/runHostPref"; import { ProjectFilesSettingsDialog, useProjectFileFilters } from "./ProjectFilesSettings"; import { useImportDrop } from "./importDrop"; import { logoutRemote } from "../../stores/projects"; @@ -789,10 +788,11 @@ export function ProjectFilesView({ {/* Run-host picker — which machine scripts/shells launched from this project run on (primary or a worker), distinct from the source - switch's read side. Shown only while the switch is on Remote (no - machine axis on Local), and hidden once a worker keeps its own synced - code copy — then the run target is the tab's picked locality. */} - {source === "remote" && runHostPickerApplies(project.compute_hosts) && ( + switch's read side. Shown whenever the switch is on Remote (no machine + axis on Local), including a multi-machine project with a synced-code + worker: a Python Run opens a fresh tab, so this project-wide picker is + the only control that can send that run to a worker. */} + {source === "remote" && ( - {source === "remote" && runHostPickerApplies(project.compute_hosts) && ( + {source === "remote" && ( { void refresh(); const onFocus = () => void refresh(); window.addEventListener("focus", onFocus); - return () => window.removeEventListener("focus", onFocus); + const id = window.setInterval(() => void refresh(), 10_000); + return () => { + window.removeEventListener("focus", onFocus); + window.clearInterval(id); + }; }, [refresh]); // The stored `.ovpn` list is only needed once the menu is open — and with it, which diff --git a/src/components/layout/LocalModelMenu.tsx b/src/components/layout/LocalModelMenu.tsx index de45241..e3d3dfd 100644 --- a/src/components/layout/LocalModelMenu.tsx +++ b/src/components/layout/LocalModelMenu.tsx @@ -3,6 +3,7 @@ import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; import { useSettingsStore } from "../../stores/settings"; import { useEnergySaver, saverInterval } from "../../stores/power"; +import { useOllamaStatus } from "../../lib/ollamaStatus"; import { formatBytes, gpuAdapterTooltip, @@ -62,7 +63,12 @@ export function LocalModelMenu() { // Three-state Ollama health for the status lamp: "stopped" (server down, red), // "idle" (server up, no model in memory, yellow), "loaded" (a model is loaded // in memory, green). - const [status, setStatus] = useState<"stopped" | "idle" | "loaded">("stopped"); + // Once Ollama is installed, the server's health is polled so the button shows a + // live lamp without the user opening the menu. The poll itself is the app-wide + // shared one (`lib/ollamaStatus`) — it is a machine-wide fact, and the file + // viewer asks the same question per open tab, so a timer here as well meant the + // same `/api/ps` round trip several times over. + const status = useOllamaStatus(installed, saverInterval(5000, energySaver)); const [open, setOpen] = useState(false); // Every installed model (from list_ollama_models_detailed). Resident ones are // selectable as the active local model; the rest can be loaded into memory. @@ -163,27 +169,6 @@ export function LocalModelMenu() { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - // Once Ollama is installed, poll the server's health so the button can show a - // live stopped/idle/loaded lamp without the user opening the menu. - useEffect(() => { - if (!installed) { - setStatus("stopped"); - return; - } - let cancelled = false; - const check = () => - invoke<"stopped" | "idle" | "loaded">("ollama_status") - .then((s) => { - if (!cancelled) setStatus(s); - }) - .catch(() => {}); - void check(); - const id = window.setInterval(check, saverInterval(5000, energySaver)); - return () => { - cancelled = true; - window.clearInterval(id); - }; - }, [installed, energySaver]); // The GPU's own memory, polled only while the menu is open: the question this // menu raises is "will the next model fit?", which each model's `size_vram` diff --git a/src/lib/dirSizeGuard.ts b/src/lib/dirSizeGuard.ts new file mode 100644 index 0000000..a7d1abc --- /dev/null +++ b/src/lib/dirSizeGuard.ts @@ -0,0 +1,152 @@ +/** + * The guard around the file tree's per-folder size calls. + * + * Folder sizes are a **best-effort display aid** — the tree renders without them + * and simply shows no size when one doesn't arrive. That is what makes them safe + * to bound aggressively, and bounding them matters because of where they run: for + * a remote project `dir_size`/`dir_size_breakdown` are a `du` over SSH, and the + * tree fires **one per visible folder, all at once**. + * + * When the network under a remote project goes away — a dropped VPN tunnel, a + * suspended laptop — those calls do not fail. A black-holed socket produces no + * error, so each one blocks until ssh's own keepalive gives up (~45 s), and each + * blocked call is holding a backend blocking-pool thread the whole time. Opening + * one folder of thirty subfolders then means thirty stalled threads and a UI with + * nothing to render, which is exactly how a lost tunnel turned into a frozen app. + * + * Three limits, each aimed at a different part of that: + * + * - **Timeout** — a size that hasn't arrived in {@link TIMEOUT_MS} is treated as + * one that isn't coming. This does *not* cancel the backend call (a Tauri + * command in flight cannot be recalled); it frees the caller, which is the + * part that was stuck. + * - **Concurrency cap** — at most {@link MAX_CONCURRENT} in flight per project, + * so a wide folder queues instead of dispatching thirty calls into a network + * that is not answering. + * - **Circuit breaker** — {@link FAILURES_TO_OPEN} consecutive timeouts mean the + * host, not the folder, is the problem. The breaker then opens for + * {@link OPEN_MS} and every further request for that project is refused + * *immediately*, so the other twenty-nine folders cost nothing at all rather + * than each paying the timeout in turn. One success closes it again. + * + * Keyed by `projectDir`, because "is this host answering?" is a property of the + * project's remote, not of any one folder. A local project's walk never times out + * in practice, so the breaker stays shut and this is pure overhead-free + * pass-through for it. + */ + +/** Longest a folder size may take before it is written off. */ +export const TIMEOUT_MS = 15_000; +/** Most size calls in flight at once, per project. */ +export const MAX_CONCURRENT = 4; +/** Consecutive timeouts that mean the host is gone, not the folder slow. */ +export const FAILURES_TO_OPEN = 3; +/** How long the breaker stays open before one request is let through to retry. */ +export const OPEN_MS = 30_000; + +/** Thrown when the breaker refuses a call outright. Callers treat it like any + * other failure — the size is simply left unresolved. */ +export class DirSizeUnavailable extends Error { + constructor(reason: string) { + super(reason); + this.name = "DirSizeUnavailable"; + } +} + +interface ProjectGate { + /** Consecutive timeouts since the last success. */ + failures: number; + /** Epoch ms until which the breaker is open; 0 when shut. */ + openUntil: number; + /** Calls currently in flight. */ + active: number; + /** Waiters parked on the concurrency cap, released FIFO. */ + queue: Array<() => void>; +} + +const gates = new Map(); + +function gateFor(projectDir: string): ProjectGate { + let g = gates.get(projectDir); + if (!g) { + g = { failures: 0, openUntil: 0, active: 0, queue: [] }; + gates.set(projectDir, g); + } + return g; +} + +function acquire(g: ProjectGate): Promise { + if (g.active < MAX_CONCURRENT) { + g.active += 1; + return Promise.resolve(); + } + return new Promise((resolve) => g.queue.push(resolve)); +} + +function release(g: ProjectGate): void { + const next = g.queue.shift(); + if (next) { + // Hand the slot straight to the next waiter — `active` never drops. + next(); + return; + } + g.active -= 1; +} + +/** + * Run one folder-size call under the timeout, the concurrency cap and the + * breaker. Rejects (with {@link DirSizeUnavailable} when the breaker is open, or + * a timeout error) rather than hanging; the caller leaves the size unresolved. + * + * `run` is a thunk, not a promise, so a queued call is not *started* until it has + * a slot — passing an already-running promise would defeat the cap. + */ +export async function guardedDirSize(projectDir: string, run: () => Promise): Promise { + const g = gateFor(projectDir); + if (g.openUntil > Date.now()) { + throw new DirSizeUnavailable("folder sizes paused: the project's host is not responding"); + } + await acquire(g); + let timer: number | undefined; + try { + const result = await Promise.race([ + run(), + new Promise((_, reject) => { + timer = window.setTimeout( + () => reject(new Error(`folder size timed out after ${TIMEOUT_MS}ms`)), + TIMEOUT_MS, + ); + }), + ]); + // Any answer at all proves the host is talking — shut the breaker. + g.failures = 0; + g.openUntil = 0; + return result; + } catch (e) { + // Only a *timeout* is evidence about the host. A call that came back with an + // error (an unreadable folder, a backend that lacks the command) answered + // fine, and must not push the project toward a breaker trip. + if (!(e instanceof DirSizeUnavailable) && timer !== undefined && isHostTimeout(e)) { + g.failures += 1; + if (g.failures >= FAILURES_TO_OPEN) g.openUntil = Date.now() + OPEN_MS; + } + throw e; + } finally { + if (timer !== undefined) window.clearTimeout(timer); + release(g); + } +} + +/** Whether a rejection means *the host stopped answering* rather than the call + * itself failing — the distinction that decides whether a retry could ever + * help. Only this guard's own timeout qualifies. */ +export function isHostTimeout(e: unknown): boolean { + return e instanceof Error && e.message.includes("timed out"); +} + +/** Drop a project's gate — on project close, so a stale open breaker can't + * outlive the tree that tripped it. Exported for tests too. */ +export function resetDirSizeGuard(projectDir?: string): void { + if (projectDir === undefined) gates.clear(); + else gates.delete(projectDir); +} diff --git a/src/lib/ollamaStatus.ts b/src/lib/ollamaStatus.ts new file mode 100644 index 0000000..35d9e15 --- /dev/null +++ b/src/lib/ollamaStatus.ts @@ -0,0 +1,122 @@ +import { useEffect, useState } from "react"; +import { invoke } from "@tauri-apps/api/core"; + +/** + * **The** poll of Ollama's `stopped | idle | loaded` state — one timer for the + * whole app, however many components ask. + * + * `ollama_status` is a `GET /api/ps` round trip, and it had two independent + * callers on 5 s timers: the header's 🧠 `LocalModelMenu` lamp, and + * `useLocalModelLoaded` in the file viewer — the second of which is **per + * editable viewer tab**, so the request rate grew with the number of open tabs + * rather than staying a property of the app. The observable result was a steady + * stream of paired `/api/ps` hits a couple of seconds apart, forever, asking the + * same question and getting the same answer. + * + * The state is genuinely global — whether a model is resident in Ollama's memory + * is a fact about the machine, not about a tab — so a single shared poller is not + * just cheaper, it is the more correct model: every surface now flips on the same + * observation instead of each discovering it up to 5 s apart. + * + * Shape: subscribers register an interval they'd be happy with, the timer runs at + * the **shortest** one asked for, and it only exists while at least one component + * is subscribed. A tick that lands while a request is still in flight is skipped + * rather than queued, so a slow or wedged Ollama can never accumulate a backlog. + */ +export type OllamaStatus = "stopped" | "idle" | "loaded"; + +/** Default cadence, matching what each caller polled at individually before. */ +export const DEFAULT_INTERVAL_MS = 5000; + +interface Subscriber { + notify: (s: OllamaStatus) => void; + intervalMs: number; +} + +const subscribers = new Set(); +let timer: number | undefined; +let timerInterval = 0; +let inFlight = false; +/** Last observed value, replayed to a new subscriber so it renders the known + * state immediately instead of flashing `stopped` until the next tick. */ +let current: OllamaStatus = "stopped"; + +async function poll(): Promise { + if (inFlight) return; // a wedged server must not build a queue of retries + inFlight = true; + try { + const next = await invoke("ollama_status"); + current = next; + for (const s of subscribers) s.notify(next); + } catch { + // Unreachable backend — report the same thing an unreachable Ollama does. + current = "stopped"; + for (const s of subscribers) s.notify("stopped"); + } finally { + inFlight = false; + } +} + +/** (Re)start the shared timer at the shortest interval any subscriber wants, or + * stop it entirely when the last one leaves. */ +function retime(): void { + if (subscribers.size === 0) { + if (timer !== undefined) window.clearInterval(timer); + timer = undefined; + timerInterval = 0; + return; + } + const wanted = Math.min(...[...subscribers].map((s) => s.intervalMs)); + if (timer !== undefined && timerInterval === wanted) return; + if (timer !== undefined) window.clearInterval(timer); + timerInterval = wanted; + timer = window.setInterval(() => void poll(), wanted); +} + +/** + * Subscribe to the shared Ollama status. + * + * `enabled` false unsubscribes (and reports `"stopped"`), so a caller gated on + * something else — the menu's "is Ollama even installed?" check — contributes no + * polling at all while it has nothing to show. + */ +export function useOllamaStatus(enabled = true, intervalMs = DEFAULT_INTERVAL_MS): OllamaStatus { + const [status, setStatus] = useState(enabled ? current : "stopped"); + + useEffect(() => { + if (!enabled) { + setStatus("stopped"); + return; + } + let cancelled = false; + const sub: Subscriber = { + notify: (s) => { + if (!cancelled) setStatus(s); + }, + intervalMs, + }; + subscribers.add(sub); + retime(); + // Seat from the last known value, then ask straight away — a component that + // mounts between ticks must not wait a whole interval for its first answer. + setStatus(current); + void poll(); + return () => { + cancelled = true; + subscribers.delete(sub); + retime(); + }; + }, [enabled, intervalMs]); + + return status; +} + +/** Reset the module between tests (no component is subscribed by then). */ +export function resetOllamaStatusPoller(): void { + subscribers.clear(); + if (timer !== undefined) window.clearInterval(timer); + timer = undefined; + timerInterval = 0; + inFlight = false; + current = "stopped"; +} diff --git a/src/lib/pythonRun.ts b/src/lib/pythonRun.ts index 285bc69..9d5b6df 100644 --- a/src/lib/pythonRun.ts +++ b/src/lib/pythonRun.ts @@ -24,8 +24,7 @@ import { invoke } from "@tauri-apps/api/core"; import { basename, dirname } from "./paths"; import { useTabsStore, type TabEntry } from "../stores/tabs"; -import { useProjectsStore } from "../stores/projects"; -import { useRunHostPrefStore, runHostPickerApplies } from "../stores/runHostPref"; +import { useRunHostPrefStore } from "../stores/runHostPref"; /** How a run/debug tab is inserted into the layout. Given the built (keyless) * tab, place it and return the created entry — or null when it streamed the tab @@ -194,15 +193,15 @@ export function openPythonTab(opts: { if (prior) store.removeTab(prior.key); // Which machine to run on: the project's run-host preference (set from the file - // viewer's picker), keyed by the owning project. `scope` is the project id for a - // project tab ("root" has no pref). Honoured only while the picker applies — - // once a worker keeps its own synced code copy the picker is hidden and the run - // target is the tab's picked locality, so we ignore any stale pref and omit - // `location` (⇒ the shell default, the primary on a remote project). - const project = useProjectsStore.getState().projects.find((p) => p.id === scope); - const runHost = runHostPickerApplies(project?.compute_hosts) - ? useRunHostPrefStore.getState().byProject[scope] - : undefined; + // viewer's `RunHostPicker`), keyed by the owning project. `scope` is the project + // id for a project tab ("root" has no pref). Honoured for EVERY remote project, + // including a genuinely multi-machine one with a synced-code worker — a Python + // Run creates a fresh tab, so the per-tab locality badge can't pre-target it and + // the project-wide picker is the only control that can send a run to a worker. + // Unset ⇒ `location` is omitted (⇒ the shell default, the primary). A pref naming + // a since-removed worker is harmless: CenterPanel/`wrap_pty_options` fall the tab + // back to the primary. + const runHost = useRunHostPrefStore.getState().byProject[scope]; const tab: Omit = { label: pyTabLabel(mode, file), cmd: "", // the host's default shell diff --git a/src/lib/shellScriptRun.ts b/src/lib/shellScriptRun.ts index d1b15b4..a87ccc3 100644 --- a/src/lib/shellScriptRun.ts +++ b/src/lib/shellScriptRun.ts @@ -1,5 +1,5 @@ import { relativePathWithin } from "./paths"; -import type { TabLocation } from "../stores/tabs"; +import { isRemoteLocation, type TabLocation } from "../stores/tabs"; import type { ProjectEntry } from "../types"; export type ScriptShell = "bash" | "zsh" | "fish" | "ksh" | "powershell" | "cmd"; @@ -63,24 +63,45 @@ export function scriptRelFromRoot(root: string, absPath: string): string | null * user is browsing. Mount-free remote listings return host paths, while the * tree's `projectDir` is the local state dir; using that dir to relativize a * host path produced `bash ''`. Use the host `remote_path` for remote-source - * rows, and explicitly pin the tab locality to match the selected side. */ + * rows to compute the script's project-relative path. + * + * WHERE the script runs is the project's run-host preference (`runHostPref`, the + * machine chosen in the `RunHostPicker`) when set — so "pick machine X ⇒ shells + * run on X" holds for a script Run exactly as for a Python Run — falling back to + * the browsed side. `scriptRel` is project-relative, so it resolves against the + * chosen host's own project root (the backend re-cds into the target host's + * `remote_path`) or the local mirror, whichever the run location names — the two + * sides mirror the same tree. */ export function shellScriptRunPlan(opts: { project: ProjectEntry | null | undefined; treeRoot: string; syncSource?: "remote" | "local"; scriptPath: string; interp: ScriptShell; + /** The project's run-host preference (`useRunHostPrefStore`), if any. */ + runHostPref?: TabLocation; }): ShellScriptRunPlan | null { const remote = opts.project?.remote; const isRemoteProject = !!remote; - const runsRemote = isRemoteProject && opts.syncSource !== "local"; - const root = runsRemote ? remote.remote_path : opts.treeRoot; - const scriptRel = scriptRelFromRoot(root, opts.scriptPath); + const browsedRemote = isRemoteProject && opts.syncSource !== "local"; + // The script's project-relative path, computed from the side it was browsed on + // (that is the side `scriptPath` belongs to). + const browsedRoot = browsedRemote ? remote.remote_path : opts.treeRoot; + const scriptRel = scriptRelFromRoot(browsedRoot, opts.scriptPath); if (!scriptRel) return null; + // The run location: the chosen machine wins; unset ⇒ the browsed side. + const location: TabLocation | undefined = isRemoteProject + ? (opts.runHostPref ?? (browsedRemote ? "remote" : "local")) + : undefined; + // The tab cwd must match the run side so `scriptRel` resolves: the host project + // root for a remote run (the backend re-cds into the *target* host's remote_path + // anyway), the local mirror for a local run. + const runsRemote = location ? isRemoteLocation(location) : false; + const cwd = runsRemote && remote ? remote.remote_path : opts.treeRoot; return { - cwd: root, + cwd, scriptRel, initialInput: shellRunCommand(opts.interp, scriptRel), - location: isRemoteProject ? (runsRemote ? "remote" : "local") : undefined, + location, }; } diff --git a/src/stores/projects.ts b/src/stores/projects.ts index 5fd8789..9cbcbd5 100644 --- a/src/stores/projects.ts +++ b/src/stores/projects.ts @@ -19,8 +19,10 @@ import { useTabsStore, type SavedLayoutTree, type TabKind, + type TabLocation, type ViewerState, } from "./tabs"; +import { useRunHostPrefStore } from "./runHostPref"; import { type AgentMode } from "../components/tabs/agentModes"; import { useTimerStore } from "./timer"; import { useVpnPromptStore } from "./vpnPrompt"; @@ -690,6 +692,16 @@ export const useProjectsStore = create((set, get) => ({ activeId, rightPanelFolderByProject, }); + // Re-hydrate the run-host preference (which machine shells run on) from each + // project's persisted `run_host`, so a choice made in a previous session still + // sends this session's Run/Debug and "+" shells to that machine. Merge-only, so + // a change made this session before a reload is never clobbered. + useRunHostPrefStore.getState().seed( + projects.map((p) => ({ + projectId: p.id, + location: p.run_host as TabLocation | undefined, + })), + ); // Fire-and-forget: sniff each local repo's `origin` host so pills can badge // a hosting provider (GitHub/GitLab) and the hover can show the git address, // even when the repo was pushed outside Eldrun's Publish flow. Host-only, no diff --git a/src/stores/runHostPref.ts b/src/stores/runHostPref.ts index 7f1aa83..bb0f92b 100644 --- a/src/stores/runHostPref.ts +++ b/src/stores/runHostPref.ts @@ -1,38 +1,58 @@ import { create } from "zustand"; -import type { LocalityHost, TabLocation } from "./tabs"; +import { invoke } from "@tauri-apps/api/core"; +import type { TabLocation } from "./tabs"; /** - * Whether the per-project run-host picker applies at all. It is meant for the - * simple case (a lone primary, or shared-fs workers that are just one shared - * tree). The moment a worker keeps its OWN **synced code copy** the project is - * genuinely multi-machine, and the run target is chosen **per tab** (the locality - * badge) — so the picker is hidden and a Run/Debug falls back to the tab's - * locality (the primary by default) rather than a project-wide preference. - * Shared-fs workers (`shared_fs`) don't trip this — they leave the picker in place. - */ -export function runHostPickerApplies(computeHosts?: LocalityHost[]): boolean { - return !(computeHosts ?? []).some((h) => !h.shared_fs && h.sync_code !== false); -} - -/** - * Ephemeral, non-persisted "which machine should scripts and shells launched - * from this project's file view run on" preference, keyed by project id - * (`docs/multi_host_remote_plan.md`). Set from the file viewer's run-host picker - * (`RunHostPicker`), read at launch time by `lib/pythonRun` so a Run/Debug lands - * on the chosen host (primary or a worker) instead of the shell default. + * "Which machine should scripts and shells launched from this project run on" + * preference, keyed by project id (`docs/multi_host_remote_plan.md`). Set from the + * `RunHostPicker`, read at launch time by `lib/pythonRun`, `lib/shellScriptRun`, + * and `stores/tabs`' new-shell-tab funnel so a Run/Debug or a "+" shell lands on + * the chosen host (primary or a worker) instead of the shell default. + * + * The picker + this preference apply to EVERY remote project, including a + * genuinely multi-machine one with a synced-code worker: a Run opens a fresh tab, + * so the per-tab locality badge can't pre-target it — the project-wide picker is + * the only control that can send a run to a worker. * - * Unset ⇒ the tab's own kind default (a shell defaults to the primary on a remote - * project) — so a project that never touches the picker behaves exactly as before. - * A value is a `TabLocation` (`"local" | "remote" | "host:"`), the same axis a - * tab's locality badge sets, so the run tab carries it verbatim. + * This live store is the read cache; the value is **persisted** per project (in + * `project.json`'s `run_host`, mirrored into `projects.json`) so the choice + * survives a relaunch. `set` writes through to the backend; `seed` re-hydrates the + * cache from the loaded projects on startup (see `stores/projects` `load`). Unset ⇒ + * the tab's own kind default (a shell defaults to the primary on a remote project), + * so a project that never touches the picker behaves exactly as before. A value is + * a `TabLocation` (`"local" | "remote" | "host:"`), the same axis a tab's + * locality badge sets, so the run tab carries it verbatim. */ interface RunHostPrefStore { byProject: Record; + /** Set (and persist) the run host for a project. The store updates immediately; + * the disk write is fire-and-forget (a failure only means it won't survive a + * relaunch — never worth blocking the click or the run). */ set: (projectId: string, location: TabLocation) => void; + /** Re-hydrate the cache from the persisted per-project values on load. Merges + * (never clobbers a choice already made this session), so it is safe to call on + * every projects reload. */ + seed: (entries: { projectId: string; location: TabLocation | undefined }[]) => void; } export const useRunHostPrefStore = create((set) => ({ byProject: {}, - set: (projectId, location) => - set((s) => ({ byProject: { ...s.byProject, [projectId]: location } })), + set: (projectId, location) => { + set((s) => ({ byProject: { ...s.byProject, [projectId]: location } })); + void invoke("set_project_run_host", { projectId, location }).catch((e) => + console.error(`persist run host for ${projectId} failed`, e), + ); + }, + seed: (entries) => + set((s) => { + const byProject = { ...s.byProject }; + for (const { projectId, location } of entries) { + // A choice already made this session wins over the persisted one (they + // can only differ if the user just changed it before this reload). + if (location && byProject[projectId] === undefined) { + byProject[projectId] = location; + } + } + return { byProject }; + }), })); diff --git a/src/stores/tabs.ts b/src/stores/tabs.ts index 7f1fcdb..fa069de 100644 --- a/src/stores/tabs.ts +++ b/src/stores/tabs.ts @@ -9,6 +9,7 @@ import { METRIC, agentMetricLeaf, sub } from "../lib/usageMetrics"; import { useLinkRoutingStore } from "./linkRouting"; import { bumpUsage } from "./usage"; import { newTmuxSessionName } from "../lib/tmuxSession"; +import { useRunHostPrefStore } from "./runHostPref"; /** A shell tab gets a stable persisted tmux session name at creation (TODO #85), * so a persistent remote run reattaches after a relaunch instead of forking a @@ -21,6 +22,24 @@ function withTmuxSession(tab: Omit): Omit { return tab; } +/** A new SHELL tab launched from a project inherits that project's run-host + * preference (the machine chosen in the `RunHostPicker`) as its `location`, so + * "pick machine X ⇒ ALL shells run on X" — the "+" → Shell tab, not just a + * Python/script Run. Scoped tightly: only a `shell` tab, only when it did NOT + * already pin a `location` (a script/tmux/SLURM/git-resolve tab sets its own and + * must keep it), and only when the owning scope has a stored preference (a local + * project, the root scope, and a box scope have none → unchanged). Baked in at + * creation because `effectiveTabLocation`'s per-kind default has no project + * context to consult the preference. */ +function withRunHostDefault( + scope: string, + tab: Omit, +): Omit { + if (tab.kind !== "shell" || tab.location !== undefined) return tab; + const pref = useRunHostPrefStore.getState().byProject[scope]; + return pref ? { ...tab, location: pref } : tab; +} + export type TabKind = | "agent" | "local_agent" @@ -1689,7 +1708,10 @@ export const useTabsStore = create((set, get) => ({ addTab: (tab, opts) => { const key = nextKey(tab.kind); // Spread first so a stray `key` on the payload can't shadow the minted one. - const entry: TabEntry = { ...withTmuxSession(tab), key }; + const entry: TabEntry = { + ...withTmuxSession(withRunHostDefault(get().scope, tab)), + key, + }; if (!opts?.seeded) countTabOpen(get().scope, entry); set((s) => { const { tabs, layout, focusedGroupId } = currentScopeState(s); @@ -1722,7 +1744,11 @@ export const useTabsStore = create((set, get) => ({ addTabToScope: (scope, tab) => { const key = nextKey(tab.kind); - const entry: TabEntry = { ...withTmuxSession(tab), key, scope }; + const entry: TabEntry = { + ...withTmuxSession(withRunHostDefault(scope, tab)), + key, + scope, + }; countTabOpen(scope, entry); set((s) => { const tabs = s.tabsByScope[scope] ?? []; diff --git a/src/stores/vpnStatus.ts b/src/stores/vpnStatus.ts index 5da9ed2..31a3358 100644 --- a/src/stores/vpnStatus.ts +++ b/src/stores/vpnStatus.ts @@ -108,6 +108,7 @@ export const useVpnStatusStore = create((set) => ({ refresh: async () => { const active = await invoke("openvpn_active").catch(() => null); if (!active) return; + let dropped: string[] = []; set((s) => { const byConfig: Record = {}; // What the backend says is up... @@ -117,11 +118,52 @@ export const useVpnStatusStore = create((set) => ({ for (const [config, state] of Object.entries(s.byConfig)) { if (state === "connecting" && !(config in byConfig)) byConfig[config] = state; } + // A tunnel we believed was UP that the backend no longer reports has + // **died on its own** — it was not disconnected from the UI, because every + // deliberate teardown (`disconnectVpnTunnel`, `releaseAll`, `setState off`) + // forgets the config here first, so it is already absent by the time the + // next reconcile runs. Only an unplanned death reaches this branch. + dropped = Object.keys(s.byConfig).filter( + (config) => s.byConfig[config] === "connected" && !(config in byConfig), + ); return { byConfig }; }); + for (const config of dropped) { + onTunnelDropped(config); + // Drop the claims too: the tunnel they were holding no longer exists, so + // leaving them behind would make a later reconnect look already-held. + useVpnStatusStore.getState().releaseAll(config); + } }, })); +/** + * React to a tunnel that died without being asked to (see `refresh`). + * + * The point is not the lamp — it is that **every SSH/SFTP call belonging to a + * project that was riding this tunnel is now aimed at a peer that will never + * answer**. Those calls are synchronous Tauri commands over a pooled + * ControlMaster: nothing about a black-holed socket produces an error, so each + * one blocks until ssh's own keepalive gives up (~45 s), and the file tree + * issues one per visible folder. That is the freeze this exists to prevent — + * clearing the project's status flips `useRemoteBlocked`, so the probes are + * never dispatched in the first place rather than each paying the timeout. + * + * Scoped to the tunnel's **holders**: a project that never claimed this config + * reaches its host by some other route and is none of this tunnel's business. + * + * The backend's pooled session is deliberately NOT torn down here. Reaping it is + * already the pooled reader's job (`services::remote::pooled_sftp_host` evicts a + * child whose ssh keepalive killed it), and a teardown issued from this path + * would itself be a call over the dead connection — the exact thing being + * avoided. + */ +function onTunnelDropped(config: string): void { + const holders = useVpnStatusStore.getState().holders[config] ?? []; + const remote = useRemoteStatusStore.getState(); + for (const projectId of holders) remote.clear(projectId); +} + /** True when at least one tunnel is up or coming up — i.e. the machine's routing * is (or is about to be) Eldrun's doing. */ export function anyVpnLive(byConfig: Record): boolean { diff --git a/src/types/index.ts b/src/types/index.ts index 0eb8546..5dabff2 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -463,6 +463,12 @@ export interface ProjectEntry { * environments auto-detect cannot see (a conda env, a Poetry venv outside the * tree, a second venv). Set from the pill's "Python interpreter…" dialog. */ python_interpreter?: string; + /** Which machine shells launched from this project run on — the persisted + * `RunHostPicker` choice (a `TabLocation`: "local" | "remote" | "host:"). + * Seeds the live `useRunHostPrefStore` on load so the choice survives a + * relaunch. Mirrored from project.json's `run_host` into the entry's flattened + * `extra`. Absent = the shell default (the primary). */ + run_host?: string; /** Denormalized inverse of `ProjectBox.member_ids` (the box this pill is in). */ box_id?: string; /** Per-project git-hosting profile URL that overrides the global one. Mirrored