From 73d332db9fc1e7c6e5636e863b088f081b1f3556 Mon Sep 17 00:00:00 2001 From: JetSquirrel Date: Mon, 2 Nov 2026 16:15:57 +0900 Subject: [PATCH 1/2] Make an analysis folder portable, and say what a move broke A .dash file already carries its data paths relative to itself, but a whole investigation had nothing saying what it consisted of, or whether it would work on the machine it had just arrived on. An optional ducklocal-project.json lists the dashboards, any other data and the versions and extensions the work used. `project check` runs the existing check over each dashboard and adds what a move breaks: files that did not come along, paths that were never going to, extensions this machine lacks. Each problem says what to do next. It never reaches the network or installs anything: a dashboard reading remote data is left unrun, and says so. `limitations` says what passing does not cover, such as a source with the same name and different data. `project relink` points a source at where its file is now, keeping the old dashboard as a backup. Opening the manifest in the window opens its dashboards, and only those inside its folder, since one elsewhere would run SQL from somewhere nobody looked. A lone .dash file works as it always has. Closes #70. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + examples/portable_project/data/orders.csv | 6 + .../portable_project/ducklocal-project.json | 7 + examples/portable_project/reports/orders.dash | 26 + skills/ducklocal/SKILL.md | 2 + skills/ducklocal/references/cli.md | 15 + src/cli.rs | 4 + src/main.rs | 1 + src/project.rs | 938 ++++++++++++++++++ src/spec/proposal.rs | 2 +- src/spec/tabs.rs | 8 + tests/cli.rs | 271 +++++ 12 files changed, 1280 insertions(+), 1 deletion(-) create mode 100644 examples/portable_project/data/orders.csv create mode 100644 examples/portable_project/ducklocal-project.json create mode 100644 examples/portable_project/reports/orders.dash create mode 100644 src/project.rs diff --git a/README.md b/README.md index 2a2f677..4a7776a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ ducklocal check dashboard.dash # validate a dashboard ducklocal propose dashboard.dash --from new.dash # suggest a change for a person to review ducklocal open --run --sql "SELECT ..." # hand a result to the running window ducklocal open --state # read back what the window shows +ducklocal project check # will this analysis folder work here, and if not, why ``` Database files open read-only unless you pass `--read-write`. See the [CLI guide](https://ducklocal.app/docs/cli) for every command and option, and the [handoff contract](docs/agent-handoff.md) for how an agent stays in step with the window. diff --git a/examples/portable_project/data/orders.csv b/examples/portable_project/data/orders.csv new file mode 100644 index 0000000..18d74f1 --- /dev/null +++ b/examples/portable_project/data/orders.csv @@ -0,0 +1,6 @@ +date,channel,region,orders,revenue +2026-09-01,web,EU,12,340.50 +2026-09-01,store,EU,7,210.00 +2026-09-02,web,US,20,610.25 +2026-09-02,store,US,4,95.00 +2026-09-03,web,EU,15,402.75 diff --git a/examples/portable_project/ducklocal-project.json b/examples/portable_project/ducklocal-project.json new file mode 100644 index 0000000..7cbd764 --- /dev/null +++ b/examples/portable_project/ducklocal-project.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "name": "orders", + "dashboards": [ + "reports/orders.dash" + ] +} diff --git a/examples/portable_project/reports/orders.dash b/examples/portable_project/reports/orders.dash new file mode 100644 index 0000000..3fea049 --- /dev/null +++ b/examples/portable_project/reports/orders.dash @@ -0,0 +1,26 @@ +// A dashboard inside a portable project. Its data path is relative to this +// file, so the whole folder can move: +// +// ducklocal project check examples/portable_project +// ducklocal examples/portable_project/ducklocal-project.json + +source "orders" { + path = "../data/orders.csv" +} + +query "by_channel" { + sql = < Option { | "check" | "comments" | "propose" + | "project" | "presence" | "lsp" | "--help" @@ -968,6 +970,8 @@ pub fn dispatch(args: &[OsString]) -> Option { Ok(crate::spec::proposal::PROPOSE_HELP.to_string()) } ("propose", _) => crate::spec::proposal::run(&args[1..]), + ("project", 2) if args[1] == "--help" => Ok(crate::project::HELP.to_string()), + ("project", _) => crate::project::run(&args[1..]), ("presence", 2) if args[1] == "--help" => Ok(crate::spec::presence::HELP.to_string()), ("presence", _) => crate::spec::presence::run(&args[1..]), ("lsp", 2) if args[1] == "--help" => Ok(LSP_HELP.to_string()), diff --git a/src/main.rs b/src/main.rs index 515e7b2..4407242 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ mod overview; #[cfg(test)] mod perf_probe; mod profile; +mod project; mod query; mod query_tabs; mod recents; diff --git a/src/project.rs b/src/project.rs new file mode 100644 index 0000000..3b3f9e8 --- /dev/null +++ b/src/project.rs @@ -0,0 +1,938 @@ +//! A portable analysis: a folder with a small manifest saying which dashboards +//! it is and what they need, so it can be checked and reopened somewhere else. +//! +//! The manifest is optional and adds to what is already there. A `.dash` file +//! still says where its own data is, relative to itself, and `ducklocal check` +//! still judges it; `project check` runs that over each dashboard and adds what +//! a move breaks: files that did not come along, paths that were never going +//! to, extensions this machine lacks. Data is referenced, never copied, and +//! nothing here installs an extension or reaches a remote source. + +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::cli::{parse_args, Arg, CliError, FlagSpec}; +use crate::spec::{model, proposal, source}; + +/// The manifest's name, in the project's folder. +pub const MANIFEST: &str = "ducklocal-project.json"; +const VERSION: u32 = 1; +/// How deep `init` looks for dashboards. +const MAX_DEPTH: usize = 6; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Manifest { + pub version: u32, + #[serde(default)] + pub name: String, + /// The dashboards, relative to the manifest, with forward slashes. + #[serde(default)] + pub dashboards: Vec, + /// Data the dashboards' own `source` blocks do not name: read by path in + /// their SQL, or kept here for reference. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub data: Vec, + #[serde(default, skip_serializing_if = "Requires::is_empty")] + pub requires: Requires, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataRef { + pub name: String, + pub path: String, + /// A small stand-in that travels with the project, for when `path` is not + /// to hand; said in the report, and never swapped in without asking. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Requires { + /// The DuckDB version the work was done on, such as `v2.0.0`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duckdb: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub extensions: Vec, +} + +impl Requires { + fn is_empty(&self) -> bool { + self.duckdb.is_none() && self.extensions.is_empty() + } +} + +pub fn is_manifest(path: &Path) -> bool { + path.file_name().is_some_and(|name| name == MANIFEST) && path.is_file() +} + +pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let manifest: Manifest = + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; + if manifest.version != VERSION { + return Err(format!( + "{}: version {} is not one this DuckLocal reads", + path.display(), + manifest.version + )); + } + Ok(manifest) +} + +/// Whether a relative path stays inside the folder it is relative to. +fn stays_inside(path: &str) -> bool { + let path = Path::new(path); + let mut depth = 0i32; + for part in path.components() { + match part { + Component::Normal(_) => depth += 1, + Component::CurDir => {} + Component::ParentDir => { + depth -= 1; + if depth < 0 { + return false; + } + } + Component::RootDir | Component::Prefix(_) => return false, + } + } + // A Windows path is plain text to Unix, and the reverse. + !path.to_string_lossy().contains('\\') && !looks_absolute(&path.to_string_lossy()) +} + +fn looks_absolute(path: &str) -> bool { + let bytes = path.as_bytes(); + path.starts_with('/') + || path.starts_with('\\') + || path.starts_with('~') + || (bytes.len() > 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':') +} + +fn is_remote(path: &str) -> bool { + path.contains("://") +} + +fn is_glob(path: &str) -> bool { + path.contains(['*', '?', '[']) +} + +/// The dashboards a manifest may open: those inside its own folder. One that +/// points elsewhere would run SQL from somewhere nobody looked, on the strength +/// of a file passed along with the project. +pub fn dashboards_of(manifest_path: &Path) -> Result, String> { + let manifest = load(manifest_path)?; + let root = source::base_of(manifest_path); + Ok(manifest + .dashboards + .iter() + .filter(|dashboard| stays_inside(dashboard)) + .map(|dashboard| root.join(dashboard)) + .collect()) +} + +/// One thing wrong, and what to do about it. Names paths and settings, never a +/// value that could be a secret. +fn problem(kind: &str, what: String, hint: String) -> Value { + json!({"kind": kind, "what": what, "hint": hint}) +} + +/// Absolute or remote paths written straight into a query's SQL: a move cannot +/// carry these, and they do not show as sources. +fn paths_in_sql(sql: &str) -> Vec { + let mut found = Vec::new(); + let mut rest = sql; + while let Some(start) = rest.find('\'') { + let after = &rest[start + 1..]; + let Some(end) = after.find('\'') else { break }; + let literal = &after[..end]; + if looks_absolute(literal) || is_remote(literal) { + found.push(literal.to_string()); + } + rest = &after[end + 1..]; + } + found +} + +/// Whether anything exists at `path`, resolving a glob. +fn exists(path: &Path) -> bool { + let text = path.to_string_lossy(); + if !is_glob(&text) { + return path.exists(); + } + crate::cli::open(None, false) + .ok() + .and_then(|conn| { + conn.query_row( + "SELECT count(*) FROM glob(?)", + [text.replace('\\', "/")], + |row| row.get::<_, i64>(0), + ) + .ok() + }) + .is_some_and(|count| count > 0) +} + +/// How a data path stands, and any problem with it. `base` is what it is +/// relative to, `root` the project's folder. +fn data_status(path: &str, base: &Path, root: &Path, problems: &mut Vec) -> &'static str { + if is_remote(path) { + return "remote"; + } + let resolved = source::resolve(path, base); + if !exists(&resolved) { + return "missing"; + } + if looks_absolute(path) { + problems.push(problem( + "not_portable", + format!("{path} is an absolute path, so it will not move with the project"), + "Move it into the project and use a relative path, or run `ducklocal project \ + relink` on each machine" + .to_string(), + )); + return "absolute"; + } + let inside = match ( + std::fs::canonicalize(&resolved), + std::fs::canonicalize(root), + ) { + (Ok(file), Ok(root)) => file.starts_with(root), + // A glob has no one file to ask about. + _ => stays_inside(path), + }; + if !inside { + problems.push(problem( + "not_portable", + format!("{path} is outside the project folder, so it will not move with it"), + "Move it into the project folder if it should travel".to_string(), + )); + return "outside"; + } + "ok" +} + +fn check_dashboard( + relative: &str, + root: &Path, + problems: &mut Vec, + limitations: &mut Vec, +) -> Value { + let path = root.join(relative); + if !stays_inside(relative) { + problems.push(problem( + "not_portable", + format!("Dashboard {relative} is outside the project folder, so it was not checked"), + "List dashboards relative to the manifest, inside its folder, with forward slashes" + .to_string(), + )); + return json!({"path": relative, "status": "outside", "checked": false}); + } + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(error) => { + problems.push(problem( + "missing_dashboard", + format!("Dashboard {relative}: {error}"), + "Restore the file, or take it out of the manifest's dashboards".to_string(), + )); + return json!({"path": relative, "status": "missing", "checked": false}); + } + }; + let base = source::base_of(&path); + let mut sources = Vec::new(); + let mut remote = false; + if let Ok(parsed) = model::parse(&text) { + let (spec, _) = model::validate_partial(&parsed); + for named in &spec.sources { + let status = data_status(&named.path, &base, root, problems); + remote |= status == "remote"; + if status == "missing" { + problems.push(problem( + "missing_source", + format!( + "Source {:?} in {relative}: nothing at {}", + named.name, named.path + ), + format!( + "Bring the file back, or point at where it is now: ducklocal project \ + relink --source {} --to PATH", + named.name + ), + )); + } + sources.push(json!({"name": named.name, "path": named.path, "status": status})); + } + for query in &spec.queries { + for literal in paths_in_sql(&query.sql) { + if is_remote(&literal) { + remote = true; + } else { + problems.push(problem( + "not_portable", + format!( + "Query {:?} in {relative} names {literal} directly", + query.name + ), + "Name it in a source block with a relative path, so it moves with the \ + project" + .to_string(), + )); + } + } + } + } + // Checking runs the queries, and this must not reach out on its own. + if remote { + limitations.push(format!( + "{relative} reads remote data, so its queries were not run; it depends on that \ + address and any credentials still being good" + )); + return json!({ + "path": relative, + "status": "remote", + "checked": false, + "sources": sources, + }); + } + let verdict = proposal::check(&path, &text); + let status = if verdict.passed() { "ok" } else { "failed" }; + if let Some(failure) = &verdict.failure { + // A missing file is already reported, with its way out. + if !sources.iter().any(|source| source["status"] == "missing") { + problems.push(problem( + "failed_check", + format!("{relative}: {failure}"), + format!("See `ducklocal check {relative}`"), + )); + } + } + if !verdict.unresolved.is_empty() { + limitations.push(format!( + "{relative}: {} read tables not available here, so went unchecked", + verdict.unresolved.join(", ") + )); + } + if !verdict.truncated.is_empty() { + limitations.push(format!( + "{relative}: {} hit the row budget, so what they show is not all of it", + verdict.truncated.join(", ") + )); + } + json!({ + "path": relative, + "status": status, + "checked": true, + "failure": verdict.failure, + "unresolved": verdict.unresolved, + "empty": verdict.empty, + "truncated": verdict.truncated, + "sources": sources, + }) +} + +/// DuckDB's version, and how each extension stands on this machine. +fn engine() -> (Option, Vec<(String, bool, bool)>) { + let Ok(conn) = crate::cli::open(None, false) else { + return (None, Vec::new()); + }; + let version = conn + .query_row("SELECT library_version FROM pragma_version()", [], |row| { + row.get(0) + }) + .ok(); + let extensions = conn + .prepare("SELECT extension_name, installed, loaded FROM duckdb_extensions()") + .and_then(|mut statement| { + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))? + .collect() + }) + .unwrap_or_default(); + (version, extensions) +} + +fn check_project(manifest_path: &Path) -> Result { + let manifest = load(manifest_path).map_err(|e| CliError::failure("project", e))?; + let root = source::base_of(manifest_path); + let root = if root.as_os_str().is_empty() { + PathBuf::from(".") + } else { + root + }; + let mut problems = Vec::new(); + let mut limitations = vec![ + "Sources are checked for being there, not for holding the same data: a changed file \ + gives different results with no warning here" + .to_string(), + ]; + if manifest.dashboards.is_empty() { + problems.push(problem( + "no_dashboards", + "The manifest lists no dashboards".to_string(), + format!("Add their paths to dashboards in {MANIFEST}"), + )); + } + let dashboards: Vec = manifest + .dashboards + .iter() + .map(|dashboard| check_dashboard(dashboard, &root, &mut problems, &mut limitations)) + .collect(); + + let data: Vec = manifest + .data + .iter() + .map(|entry| { + let status = data_status(&entry.path, &root, &root, &mut problems); + let sample = entry.sample.as_ref().map(|sample| { + json!({ + "path": sample, + "there": source::resolve(sample, &root).exists(), + }) + }); + if status == "missing" { + let hint = match &entry.sample { + Some(sample) => format!( + "Bring it back, or relink it: ducklocal project relink --data {} --to \ + PATH. A sample is at {sample}", + entry.name + ), + None => format!( + "Bring it back, or relink it: ducklocal project relink --data {} --to PATH", + entry.name + ), + }; + problems.push(problem( + "missing_data", + format!("Data {:?}: nothing at {}", entry.name, entry.path), + hint, + )); + } + if status == "remote" { + limitations.push(format!( + "Data {:?} is remote and was not reached; it may need credentials this \ + machine does not have", + entry.name + )); + } + json!({"name": entry.name, "path": entry.path, "status": status, "sample": sample}) + }) + .collect(); + + let (version, installed) = engine(); + if let Some(wanted) = &manifest.requires.duckdb { + if version.as_deref() != Some(wanted.as_str()) { + limitations.push(format!( + "This work was done on DuckDB {wanted}, and this is {}: results and SQL \ + behaviour can differ", + version.as_deref().unwrap_or("an unknown version") + )); + } + } + let extensions: Vec = manifest + .requires + .extensions + .iter() + .map(|name| { + let known = installed.iter().find(|(known, _, _)| known == name); + let status = match known { + Some((_, _, true)) => "loaded", + Some((_, true, _)) => "installed", + _ => "missing", + }; + if status == "missing" { + problems.push(problem( + "missing_extension", + format!("Extension {name} is not installed"), + "Nothing installs it on its own. If you want it, use the Extensions tab, \ + which downloads it from DuckDB's server, then check again" + .to_string(), + )); + } + json!({"name": name, "status": status}) + }) + .collect(); + let ok = problems.iter().all(|p| p["kind"] == "not_portable") + && dashboards + .iter() + .all(|dashboard| dashboard["status"] != "failed"); + Ok(json!({ + "project": manifest.name, + "manifest": manifest_path.to_string_lossy(), + "ok": ok, + "portable": problems.iter().all(|p| p["kind"] != "not_portable"), + "ducklocal": env!("CARGO_PKG_VERSION"), + "duckdb": version, + "dashboards": dashboards, + "data": data, + "extensions": extensions, + "problems": problems, + "limitations": limitations, + })) +} + +fn find_dashboards(dir: &Path, root: &Path, depth: usize, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut entries: Vec = entries.flatten().map(|entry| entry.path()).collect(); + entries.sort(); + for path in entries { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + if name.starts_with('.') || name == "target" || name == "node_modules" { + continue; + } + if path.is_dir() { + if depth < MAX_DEPTH { + find_dashboards(&path, root, depth + 1, found); + } + } else if crate::spec::tabs::is_spec(&path) { + if let Ok(relative) = path.strip_prefix(root) { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } +} + +fn init(dir: &Path, name: Option) -> Result { + if !dir.is_dir() { + return Err(CliError::failure( + "io", + format!("{} is not a folder", dir.display()), + )); + } + let path = dir.join(MANIFEST); + if path.exists() { + return Err( + CliError::failure("exists", format!("{} is already there", path.display())) + .with_hint("Edit it, or see how it stands with `ducklocal project check`"), + ); + } + let mut dashboards = Vec::new(); + find_dashboards(dir, dir, 0, &mut dashboards); + let folder = std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf()); + let manifest = Manifest { + version: VERSION, + name: name.unwrap_or_else(|| { + folder + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() + }), + dashboards, + data: Vec::new(), + requires: Requires { + duckdb: engine().0, + extensions: Vec::new(), + }, + }; + write_manifest(&path, &manifest)?; + Ok(json!({"manifest": path.to_string_lossy(), "project": manifest})) +} + +fn write_manifest(path: &Path, manifest: &Manifest) -> Result<(), CliError> { + let text = serde_json::to_string_pretty(manifest).map_err(|e| CliError::failure("io", e))?; + proposal::write_whole(path, (text + "\n").as_bytes()).map_err(|e| CliError::failure("io", e)) +} + +/// `text` with the `path` of `source "name"` set to `to`, by position: nothing +/// else in the file moves. `None` when it has no such source. +fn with_source_path(text: &str, name: &str, to: &str) -> Option { + let parsed = model::parse(text).ok()?; + let (spec, _) = model::validate_partial(&parsed); + let named = spec.sources.iter().find(|source| source.name == name)?; + let after_name = named.name_span.1.min(text.len()); + let rest = &text[after_name..]; + let key = rest.find("path")?; + let open = key + rest[key..].find('"')?; + let close = open + 1 + rest[open + 1..].find('"')?; + let mut out = String::with_capacity(text.len() + to.len()); + out.push_str(&text[..after_name + open + 1]); + out.push_str(to); + out.push_str(&text[after_name + close..]); + // A rewrite that does not read back as intended is not written at all. + let reread = model::parse(&out).ok()?; + let (spec, _) = model::validate_partial(&reread); + spec.sources + .iter() + .any(|source| source.name == name && source.path == to) + .then_some(out) +} + +/// `to` as a dashboard or manifest in `base` should write it. Inside the project +/// it is relative to `base`, `..` and all, with forward slashes, so it moves +/// with the folder; outside, it can only be absolute, and `check` says so. +fn written_path(to: &Path, base: &Path, root: &Path) -> String { + let absolute = |path: &Path| -> PathBuf { + let path = std::fs::canonicalize(path) + .or_else(|_| std::path::absolute(path)) + .unwrap_or_else(|_| path.to_path_buf()); + PathBuf::from(plain(&path)) + }; + let (to, base, root) = (absolute(to), absolute(base), absolute(root)); + let text = if to.starts_with(&root) { + relative_to(&to, &base).to_string_lossy().into_owned() + } else { + to.to_string_lossy().into_owned() + }; + text.replace('\\', "/") +} + +/// Windows' canonical form carries a prefix nothing else writes. +fn plain(path: &Path) -> String { + let text = path.to_string_lossy(); + match text.strip_prefix(r"\\?\") { + Some(rest) if !rest.starts_with("UNC") => rest.to_string(), + _ => text.into_owned(), + } +} + +fn relative_to(to: &Path, base: &Path) -> PathBuf { + let to: Vec<_> = to.components().collect(); + let base: Vec<_> = base.components().collect(); + let shared = to.iter().zip(&base).take_while(|(a, b)| a == b).count(); + let mut out = PathBuf::new(); + for _ in shared..base.len() { + out.push(".."); + } + for part in &to[shared..] { + out.push(part); + } + out +} + +fn relink( + manifest_path: &Path, + source_name: Option<&str>, + data_name: Option<&str>, + to: &str, +) -> Result { + let mut manifest = load(manifest_path).map_err(|e| CliError::failure("project", e))?; + let root = source::base_of(manifest_path); + let target = PathBuf::from(crate::db::expand_tilde(to)); + if is_remote(to) || to.contains('"') { + return Err(CliError::argument( + "--to names a local path, without quotation marks", + )); + } + if !exists(&target) { + return Err(CliError::argument(format!("--to: nothing at {to}"))); + } + let mut changed = Vec::new(); + if let Some(name) = data_name { + let written = written_path(&target, &root, &root); + let entry = manifest + .data + .iter_mut() + .find(|entry| entry.name == name) + .ok_or_else(|| { + CliError::failure("not_found", format!("The manifest has no data {name:?}")) + })?; + entry.path = written; + write_manifest(manifest_path, &manifest)?; + changed.push(manifest_path.to_string_lossy().into_owned()); + } + if let Some(name) = source_name { + if let Some(problem) = source::unreadable(to) { + return Err(CliError::argument(problem)); + } + // Worked out for every dashboard first, so a failure changes none. + let mut edits = Vec::new(); + for dashboard in manifest.dashboards.iter().filter(|d| stays_inside(d)) { + let path = root.join(dashboard); + let Ok(text) = std::fs::read_to_string(&path) else { + continue; + }; + let written = written_path(&target, &source::base_of(&path), &root); + if let Some(updated) = with_source_path(&text, name, &written) { + edits.push((path, text, updated)); + } + } + if edits.is_empty() { + return Err(CliError::failure( + "not_found", + format!("No dashboard in the project has a source {name:?}"), + )); + } + for (path, text, updated) in edits { + let backup = proposal::backup_path_for(&path); + proposal::write_whole(&backup, text.as_bytes()) + .and_then(|()| proposal::write_whole(&path, updated.as_bytes())) + .map_err(|e| CliError::failure("io", e))?; + changed.push(path.to_string_lossy().into_owned()); + } + } + Ok(json!({"changed": changed, "backups": "beside each dashboard, as NAME.dash.bak"})) +} + +pub(crate) const HELP: &str = "\ +Usage: ducklocal project init [DIR] [--name NAME] + ducklocal project check [PATH] + ducklocal project relink [PATH] (--source NAME | --data NAME) --to PATH + +Keep an analysis movable: a folder with a ducklocal-project.json saying which +dashboards it holds and what they need. Optional: a lone .dash file works as it +always has. Data is referenced, never copied or uploaded. Opening the manifest +in the window opens its dashboards. PATH is the folder or the manifest, by +default the current folder. + + init Write a manifest listing the dashboards found under DIR, and this + DuckDB's version. Refuses to replace one (kind exists). + check Run `ducklocal check` over each dashboard and report what a move would + break. Never reaches the network: a dashboard reading remote data is + left unrun, and says so. Installs nothing. + relink Point a source at where its file is now, in every dashboard that + names it (each kept as NAME.dash.bak), or a manifest data entry. + Ask the person first: this changes their dashboards. + +The manifest, relative paths with forward slashes: + {\"version\": 1, \"name\": \"orders\", + \"dashboards\": [\"reports/orders.dash\"], + \"data\": [{\"name\": \"archive\", \"path\": \"data/old.csv\", \"sample\": \"data/s.csv\"}], + \"requires\": {\"duckdb\": \"v2.0.0\", \"extensions\": [\"httpfs\"]}} + +check prints {project, manifest, ok, portable, ducklocal, duckdb, dashboards, +data, extensions, problems, limitations} and exits 0 whatever it found, so read +ok: false means something is missing or failed, portable: false that a path will +not survive a move. Each problem has {kind, what, hint}. limitations says what +passing does not cover. Exit 2 for arguments, 1 with kind project for a manifest +that does not read, or not_found. +"; + +const SPEC: &[FlagSpec] = &[ + FlagSpec { + name: "--name", + takes_value: true, + }, + FlagSpec { + name: "--source", + takes_value: true, + }, + FlagSpec { + name: "--data", + takes_value: true, + }, + FlagSpec { + name: "--to", + takes_value: true, + }, +]; + +fn manifest_at(path: Option) -> Result { + let path = path.unwrap_or_else(|| PathBuf::from(".")); + let manifest = if path.is_dir() { + path.join(MANIFEST) + } else { + path + }; + if !manifest.is_file() { + return Err( + CliError::failure("not_found", format!("No {}", manifest.display())) + .with_hint("Make one with `ducklocal project init`"), + ); + } + Ok(manifest) +} + +/// `ducklocal project`. +pub(crate) fn run(args: &[OsString]) -> Result { + let Some(action) = args.first().and_then(|arg| arg.to_str()) else { + return Err(CliError::argument("Say what to do: init, check or relink")); + }; + let mut path = None; + let (mut name, mut source_name, mut data_name, mut to) = (None, None, None, None); + let utf8 = |value: OsString, what: &str| { + value + .into_string() + .map_err(|_| CliError::argument(format!("{what} must be UTF-8"))) + }; + for arg in parse_args("project", &args[1..], SPEC, &[])? { + match arg { + Arg::Flag("--name", Some(value)) => name = Some(utf8(value, "--name")?), + Arg::Flag("--source", Some(value)) => source_name = Some(utf8(value, "--source")?), + Arg::Flag("--data", Some(value)) => data_name = Some(utf8(value, "--data")?), + Arg::Flag("--to", Some(value)) => to = Some(utf8(value, "--to")?), + Arg::Positional(value) if path.is_none() => path = Some(PathBuf::from(value)), + Arg::Positional(_) => return Err(CliError::argument("Name one project")), + _ => { + return Err(CliError::failure( + "internal", + "project accepted a flag it does not handle", + )) + } + } + } + let relinking = source_name.is_some() || data_name.is_some() || to.is_some(); + let output = match action { + "init" if !relinking => init(&path.unwrap_or_else(|| PathBuf::from(".")), name)?, + "check" if !relinking && name.is_none() => check_project(&manifest_at(path)?)?, + "relink" if name.is_none() => { + let to = to.ok_or_else(|| CliError::argument("Say where with --to PATH"))?; + if source_name.is_some() == data_name.is_some() { + return Err(CliError::argument("Name one of --source and --data")); + } + relink( + &manifest_at(path)?, + source_name.as_deref(), + data_name.as_deref(), + &to, + )? + } + "init" | "check" | "relink" => { + return Err(CliError::argument(format!( + "Those options do not go with {action}; see ducklocal project --help" + ))) + } + other => { + return Err(CliError::argument(format!( + "Unknown project action: {other}; use init, check or relink" + ))) + } + }; + Ok(output.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_relative_path_inside_stays_inside() { + for inside in ["a.dash", "reports/a.dash", "./a.dash", "b/../a.dash"] { + assert!(stays_inside(inside), "{inside}"); + } + for outside in [ + "../a.dash", + "a/../../b.dash", + "/etc/a.dash", + "C:/a.dash", + r"C:\a.dash", + r"reports\a.dash", + "~/a.dash", + ] { + assert!(!stays_inside(outside), "{outside}"); + } + } + + #[test] + fn a_query_naming_a_fixed_or_remote_path_is_found() { + assert_eq!( + paths_in_sql("SELECT * FROM '/data/a.csv' JOIN 'b.csv' ON x = 'y'"), + vec!["/data/a.csv"] + ); + assert_eq!( + paths_in_sql("SELECT * FROM read_csv('s3://b/a.csv'), 'C:/x.csv'"), + vec!["s3://b/a.csv", "C:/x.csv"] + ); + assert!(paths_in_sql("SELECT 'it''s' AS a").is_empty()); + } + + #[test] + fn relinking_changes_that_one_path_and_nothing_else() { + let text = "// orders\nsource \"a\" {\n path = \"old.csv\"\n}\n\nsource \"b\" {\n \ + path = \"b.csv\"\n}\n\nquery \"q\" {\n sql = < String { /// Written whole and renamed into place, so nothing ever reads half a file and /// a failed write leaves the old one as it was. -fn write_whole(path: &Path, bytes: &[u8]) -> Result<(), String> { +pub(crate) fn write_whole(path: &Path, bytes: &[u8]) -> Result<(), String> { let mut name = path.file_name().map(OsString::from).unwrap_or_default(); name.push(".writing"); let temporary = path.with_file_name(name); diff --git a/src/spec/tabs.rs b/src/spec/tabs.rs index 1d4ccae..e4b4e80 100644 --- a/src/spec/tabs.rs +++ b/src/spec/tabs.rs @@ -48,6 +48,14 @@ pub fn split(paths: Vec) -> (Vec, Vec) { let mut rest = Vec::new(); for path in paths { let expanded = PathBuf::from(crate::db::expand_tilde(&path)); + // A project's manifest opens as its dashboards. One that does not read + // goes on as data, and opening it as data says what is wrong. + if crate::project::is_manifest(&expanded) { + if let Ok(dashboards) = crate::project::dashboards_of(&expanded) { + specs.extend(dashboards.into_iter().filter(|spec| is_spec(spec))); + continue; + } + } if is_spec(&expanded) { specs.push(expanded); } else { diff --git a/tests/cli.rs b/tests/cli.rs index ddad805..aa05f07 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1909,6 +1909,277 @@ fn propose_stages_a_change_and_only_the_window_applies_it() { .contains("--based-on")); } +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let path = entry.unwrap().path(); + let target = to.join(path.file_name().unwrap()); + if path.is_dir() { + copy_tree(&path, &target); + } else { + std::fs::copy(&path, &target).unwrap(); + } + } +} + +fn project_report(s: &Sandbox, dir: &str) -> Value { + let output = s.run(&["project", "check", dir], None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +/// The shipped example, moved: what the issue means by portable. +#[test] +fn a_project_moves_as_a_folder_and_says_what_a_move_broke() { + let s = Sandbox::new(); + let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/portable_project"); + copy_tree(&example, &s.0.join("moved/somewhere/else")); + let dir = "moved/somewhere/else"; + + let report = project_report(&s, dir); + assert_eq!(report["ok"], true, "{report}"); + assert_eq!(report["portable"], true, "{report}"); + assert_eq!(report["problems"], json!([])); + assert_eq!(report["dashboards"][0]["status"], "ok"); + assert_eq!(report["dashboards"][0]["checked"], true); + assert_eq!(report["dashboards"][0]["sources"][0]["status"], "ok"); + assert!(report["duckdb"].is_string()); + assert!(report["limitations"][0] + .as_str() + .unwrap() + .contains("not for holding the same data")); + + // The data did not come along: named, and the way out given. + let data = s.0.join(dir).join("data/orders.csv"); + let moved = s.0.join(dir).join("data/2026.csv"); + std::fs::rename(&data, &moved).unwrap(); + let report = project_report(&s, dir); + assert_eq!(report["ok"], false); + assert_eq!(report["problems"][0]["kind"], "missing_source"); + assert!(report["problems"][0]["hint"] + .as_str() + .unwrap() + .contains("relink --source orders")); + assert_eq!(report["problems"].as_array().unwrap().len(), 1, "{report}"); + + // Relinked: the dashboard says where it is now, relative, and still works. + let spec = s.0.join(dir).join("reports/orders.dash"); + let before = std::fs::read_to_string(&spec).unwrap(); + let to = format!("{dir}/data/2026.csv"); + let relinked = s.run( + &["project", "relink", dir, "--source", "orders", "--to", &to], + None, + ); + assert!( + relinked.status.success(), + "{}", + String::from_utf8_lossy(&relinked.stderr) + ); + let after = std::fs::read_to_string(&spec).unwrap(); + assert_eq!( + after, + before.replace("../data/orders.csv", "../data/2026.csv") + ); + assert_eq!( + std::fs::read_to_string(s.0.join(dir).join("reports/orders.dash.bak")).unwrap(), + before + ); + assert_eq!(project_report(&s, dir)["ok"], true); + + // Refused, with nothing changed. + s.error( + &[ + "project", + "relink", + dir, + "--source", + "orders", + "--to", + "absent.csv", + ], + 2, + "argument", + ); + s.error( + &["project", "relink", dir, "--source", "nope", "--to", &to], + 1, + "not_found", + ); + s.error(&["project", "relink", dir, "--to", &to], 2, "argument"); + assert_eq!(std::fs::read_to_string(&spec).unwrap(), after); +} + +#[test] +fn a_project_check_reports_what_will_not_travel_or_is_not_here() { + let s = Sandbox::new(); + let spec = |source: &str| { + format!( + "source \"d\" {{\n path = \"{source}\"\n}}\n\nquery \"q\" {{\n sql = \"SELECT * FROM d\"\n}}\n\n\ + plot \"p\" {{\n type = \"table\"\n query = query.q\n}}\n" + ) + }; + std::fs::create_dir_all(s.0.join("p")).unwrap(); + std::fs::write(s.0.join("p/a.csv"), "n\n1\n").unwrap(); + std::fs::write(s.0.join("outside.csv"), "n\n1\n").unwrap(); + let absolute = s.0.join("outside.csv").to_string_lossy().replace('\\', "/"); + std::fs::write(s.0.join("p/absolute.dash"), spec(&absolute)).unwrap(); + std::fs::write(s.0.join("p/above.dash"), spec("../outside.csv")).unwrap(); + std::fs::write( + s.0.join("p/remote.dash"), + "query \"q\" {\n sql = \"SELECT * FROM 'https://example.com/x.csv'\"\n}\n", + ) + .unwrap(); + std::fs::write(s.0.join("p/broken.dash"), "query \"q\" {").unwrap(); + std::fs::write( + s.0.join("p/ducklocal-project.json"), + json!({ + "version": 1, + "name": "t", + "dashboards": [ + "absolute.dash", "above.dash", "remote.dash", "broken.dash", + "absent.dash", "../escape.dash", + ], + "data": [ + {"name": "here", "path": "a.csv"}, + {"name": "gone", "path": "gone.csv", "sample": "a.csv"}, + {"name": "lake", "path": "s3://bucket/key.parquet"}, + ], + "requires": {"duckdb": "v0.0.1", "extensions": ["no_such_extension", "json"]}, + }) + .to_string(), + ) + .unwrap(); + + let report = project_report(&s, "p"); + assert_eq!(report["ok"], false); + assert_eq!(report["portable"], false); + let status = |name: &str| { + report["dashboards"] + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == name) + .unwrap() + .clone() + }; + // Reads, but will not travel. + assert_eq!(status("absolute.dash")["sources"][0]["status"], "absolute"); + assert_eq!(status("above.dash")["sources"][0]["status"], "outside"); + // Left unrun rather than reaching out, which would hang here anyway. + assert_eq!(status("remote.dash")["status"], "remote"); + assert_eq!(status("remote.dash")["checked"], false); + assert_eq!(status("broken.dash")["status"], "failed"); + assert_eq!(status("absent.dash")["status"], "missing"); + assert_eq!(status("../escape.dash")["status"], "outside"); + assert_eq!(status("../escape.dash")["checked"], false); + + let kinds: Vec<&str> = report["problems"] + .as_array() + .unwrap() + .iter() + .map(|p| p["kind"].as_str().unwrap()) + .collect(); + for kind in [ + "not_portable", + "failed_check", + "missing_dashboard", + "missing_data", + "missing_extension", + ] { + assert!(kinds.contains(&kind), "{kind} in {kinds:?}"); + } + assert!(report["problems"] + .as_array() + .unwrap() + .iter() + .all(|p| p["hint"].is_string())); + assert_eq!(report["data"][0]["status"], "ok"); + assert_eq!(report["data"][1]["status"], "missing"); + assert_eq!(report["data"][1]["sample"]["there"], true); + assert_eq!(report["data"][2]["status"], "remote"); + assert_eq!(report["extensions"][0]["status"], "missing"); + assert_ne!(report["extensions"][1]["status"], "missing"); + let limits = report["limitations"].to_string(); + assert!(limits.contains("v0.0.1"), "{limits}"); + assert!(limits.contains("remote.dash"), "{limits}"); + assert!(limits.contains("lake"), "{limits}"); + // A key in the address would not show, because the address is not repeated. + assert!(!limits.contains("key.parquet"), "{limits}"); + + // A data entry relinks in the manifest, and a dashboard keeps its own path. + let moved = s.run( + &[ + "project", "relink", "p", "--data", "gone", "--to", "p/a.csv", + ], + None, + ); + assert!( + moved.status.success(), + "{}", + String::from_utf8_lossy(&moved.stderr) + ); + assert_eq!(project_report(&s, "p")["data"][1]["path"], "a.csv"); + s.error( + &[ + "project", "relink", "p", "--data", "nope", "--to", "p/a.csv", + ], + 1, + "not_found", + ); +} + +#[test] +fn a_project_starts_from_what_is_in_the_folder() { + let s = Sandbox::new(); + std::fs::create_dir_all(s.0.join("p/reports")).unwrap(); + std::fs::create_dir_all(s.0.join("p/.hidden")).unwrap(); + let spec = "query \"q\" {\n sql = \"SELECT 1 AS n\"\n}\n"; + std::fs::write(s.0.join("p/top.dash"), spec).unwrap(); + std::fs::write(s.0.join("p/reports/sub.dash"), spec).unwrap(); + std::fs::write(s.0.join("p/.hidden/no.dash"), spec).unwrap(); + + let made = s.run(&["project", "init", "p", "--name", "mine"], None); + assert!( + made.status.success(), + "{}", + String::from_utf8_lossy(&made.stderr) + ); + let manifest: Value = serde_json::from_str( + &std::fs::read_to_string(s.0.join("p/ducklocal-project.json")).unwrap(), + ) + .unwrap(); + assert_eq!(manifest["name"], "mine"); + assert_eq!( + manifest["dashboards"], + json!(["reports/sub.dash", "top.dash"]) + ); + assert!(manifest["requires"]["duckdb"].is_string()); + assert_eq!(project_report(&s, "p")["ok"], true); + + // It will not replace one, and says what to do instead. + let error = s.error(&["project", "init", "p"], 1, "exists"); + assert!(error["error"]["hint"].is_string()); + // Mistakes. + s.error(&["project"], 2, "argument"); + s.error(&["project", "dance"], 2, "argument"); + s.error(&["project", "check", "no_such_folder"], 1, "not_found"); + s.error(&["project", "init", "no_such_folder"], 1, "io"); + s.error(&["project", "check", "p", "--to", "x"], 2, "argument"); + std::fs::create_dir_all(s.0.join("bad")).unwrap(); + std::fs::write(s.0.join("bad/ducklocal-project.json"), "{ nope").unwrap(); + s.error(&["project", "check", "bad"], 1, "project"); + let help = s.run(&["project", "--help"], None); + assert!(String::from_utf8(help.stdout).unwrap().contains("relink")); + // A lone dashboard needs none of this. + std::fs::write(s.0.join("lone.dash"), spec).unwrap(); + let lone = s.run(&["check", "lone.dash"], None); + assert!(lone.status.success()); +} + #[test] fn schema_stats_profiles_one_relation() { let s = Sandbox::new(); From ac380258e2a97c13317d185150d79f316871b6f1 Mon Sep 17 00:00:00 2001 From: JetSquirrel Date: Mon, 2 Nov 2026 16:05:25 +0900 Subject: [PATCH 2/2] Give the timeout test a query that cannot run out of memory first Its cross join built up memory, so on a fast enough machine it reached the 2GB limit inside the 1s timeout and reported resource_limit, failing macOS CI. The limits were right; the query was testing the wrong one. Co-Authored-By: Claude Opus 5.5 --- tests/cli.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/cli.rs b/tests/cli.rs index aa05f07..ce70200 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1668,7 +1668,9 @@ fn restricted_leaves_a_database_file_as_it_was() { #[test] fn restricted_stops_at_its_limits_and_says_which() { let s = Sandbox::new(); - let endless = "SELECT count(*) FROM range(100000000000) a, range(100000000000) b"; + // Streams, so it holds next to nothing in memory: a join would reach the + // memory limit first on a fast machine, and say resource_limit instead. + let endless = "SELECT sum(i) FROM range(100000000000000) t(i)"; let error = s.error( &["query", "--restricted", "--timeout", "1", "--sql", endless], 1,