diff --git a/README.md b/README.md index ae21541..c5a4dba 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ ducklocal propose dashboard.dash --from new.dash # suggest a change for ducklocal session show # how an analysis got here, and what no longer holds ducklocal open --run --sql "SELECT ..." # hand a result to the running window ducklocal open --state # read back what the window shows +ducklocal project check # will this analysis folder work here, and if not, why ``` Database files open read-only unless you pass `--read-write`. See the [CLI guide](https://ducklocal.app/docs/cli) for every command and option, and the [handoff contract](docs/agent-handoff.md) for how an agent stays in step with the window. diff --git a/examples/portable_project/data/orders.csv b/examples/portable_project/data/orders.csv new file mode 100644 index 0000000..18d74f1 --- /dev/null +++ b/examples/portable_project/data/orders.csv @@ -0,0 +1,6 @@ +date,channel,region,orders,revenue +2026-09-01,web,EU,12,340.50 +2026-09-01,store,EU,7,210.00 +2026-09-02,web,US,20,610.25 +2026-09-02,store,US,4,95.00 +2026-09-03,web,EU,15,402.75 diff --git a/examples/portable_project/ducklocal-project.json b/examples/portable_project/ducklocal-project.json new file mode 100644 index 0000000..7cbd764 --- /dev/null +++ b/examples/portable_project/ducklocal-project.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "name": "orders", + "dashboards": [ + "reports/orders.dash" + ] +} diff --git a/examples/portable_project/reports/orders.dash b/examples/portable_project/reports/orders.dash new file mode 100644 index 0000000..3fea049 --- /dev/null +++ b/examples/portable_project/reports/orders.dash @@ -0,0 +1,26 @@ +// A dashboard inside a portable project. Its data path is relative to this +// file, so the whole folder can move: +// +// ducklocal project check examples/portable_project +// ducklocal examples/portable_project/ducklocal-project.json + +source "orders" { + path = "../data/orders.csv" +} + +query "by_channel" { + sql = < Option { | "check" | "comments" | "propose" + | "project" | "session" | "presence" | "lsp" @@ -982,6 +984,8 @@ pub fn dispatch(args: &[OsString]) -> Option { Ok(crate::spec::proposal::PROPOSE_HELP.to_string()) } ("propose", _) => crate::spec::proposal::run(&args[1..]), + ("project", 2) if args[1] == "--help" => Ok(crate::project::HELP.to_string()), + ("project", _) => crate::project::run(&args[1..]), ("session", 2) if args[1] == "--help" => Ok(crate::session::HELP.to_string()), ("session", _) => crate::session::run(&args[1..]), ("presence", 2) if args[1] == "--help" => Ok(crate::spec::presence::HELP.to_string()), diff --git a/src/main.rs b/src/main.rs index a34cf69..2bc916e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ mod overview; #[cfg(test)] mod perf_probe; mod profile; +mod project; mod query; mod query_tabs; mod recents; diff --git a/src/project.rs b/src/project.rs new file mode 100644 index 0000000..3b3f9e8 --- /dev/null +++ b/src/project.rs @@ -0,0 +1,938 @@ +//! A portable analysis: a folder with a small manifest saying which dashboards +//! it is and what they need, so it can be checked and reopened somewhere else. +//! +//! The manifest is optional and adds to what is already there. A `.dash` file +//! still says where its own data is, relative to itself, and `ducklocal check` +//! still judges it; `project check` runs that over each dashboard and adds what +//! a move breaks: files that did not come along, paths that were never going +//! to, extensions this machine lacks. Data is referenced, never copied, and +//! nothing here installs an extension or reaches a remote source. + +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::cli::{parse_args, Arg, CliError, FlagSpec}; +use crate::spec::{model, proposal, source}; + +/// The manifest's name, in the project's folder. +pub const MANIFEST: &str = "ducklocal-project.json"; +const VERSION: u32 = 1; +/// How deep `init` looks for dashboards. +const MAX_DEPTH: usize = 6; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Manifest { + pub version: u32, + #[serde(default)] + pub name: String, + /// The dashboards, relative to the manifest, with forward slashes. + #[serde(default)] + pub dashboards: Vec, + /// Data the dashboards' own `source` blocks do not name: read by path in + /// their SQL, or kept here for reference. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub data: Vec, + #[serde(default, skip_serializing_if = "Requires::is_empty")] + pub requires: Requires, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DataRef { + pub name: String, + pub path: String, + /// A small stand-in that travels with the project, for when `path` is not + /// to hand; said in the report, and never swapped in without asking. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Requires { + /// The DuckDB version the work was done on, such as `v2.0.0`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duckdb: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub extensions: Vec, +} + +impl Requires { + fn is_empty(&self) -> bool { + self.duckdb.is_none() && self.extensions.is_empty() + } +} + +pub fn is_manifest(path: &Path) -> bool { + path.file_name().is_some_and(|name| name == MANIFEST) && path.is_file() +} + +pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let manifest: Manifest = + serde_json::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; + if manifest.version != VERSION { + return Err(format!( + "{}: version {} is not one this DuckLocal reads", + path.display(), + manifest.version + )); + } + Ok(manifest) +} + +/// Whether a relative path stays inside the folder it is relative to. +fn stays_inside(path: &str) -> bool { + let path = Path::new(path); + let mut depth = 0i32; + for part in path.components() { + match part { + Component::Normal(_) => depth += 1, + Component::CurDir => {} + Component::ParentDir => { + depth -= 1; + if depth < 0 { + return false; + } + } + Component::RootDir | Component::Prefix(_) => return false, + } + } + // A Windows path is plain text to Unix, and the reverse. + !path.to_string_lossy().contains('\\') && !looks_absolute(&path.to_string_lossy()) +} + +fn looks_absolute(path: &str) -> bool { + let bytes = path.as_bytes(); + path.starts_with('/') + || path.starts_with('\\') + || path.starts_with('~') + || (bytes.len() > 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':') +} + +fn is_remote(path: &str) -> bool { + path.contains("://") +} + +fn is_glob(path: &str) -> bool { + path.contains(['*', '?', '[']) +} + +/// The dashboards a manifest may open: those inside its own folder. One that +/// points elsewhere would run SQL from somewhere nobody looked, on the strength +/// of a file passed along with the project. +pub fn dashboards_of(manifest_path: &Path) -> Result, String> { + let manifest = load(manifest_path)?; + let root = source::base_of(manifest_path); + Ok(manifest + .dashboards + .iter() + .filter(|dashboard| stays_inside(dashboard)) + .map(|dashboard| root.join(dashboard)) + .collect()) +} + +/// One thing wrong, and what to do about it. Names paths and settings, never a +/// value that could be a secret. +fn problem(kind: &str, what: String, hint: String) -> Value { + json!({"kind": kind, "what": what, "hint": hint}) +} + +/// Absolute or remote paths written straight into a query's SQL: a move cannot +/// carry these, and they do not show as sources. +fn paths_in_sql(sql: &str) -> Vec { + let mut found = Vec::new(); + let mut rest = sql; + while let Some(start) = rest.find('\'') { + let after = &rest[start + 1..]; + let Some(end) = after.find('\'') else { break }; + let literal = &after[..end]; + if looks_absolute(literal) || is_remote(literal) { + found.push(literal.to_string()); + } + rest = &after[end + 1..]; + } + found +} + +/// Whether anything exists at `path`, resolving a glob. +fn exists(path: &Path) -> bool { + let text = path.to_string_lossy(); + if !is_glob(&text) { + return path.exists(); + } + crate::cli::open(None, false) + .ok() + .and_then(|conn| { + conn.query_row( + "SELECT count(*) FROM glob(?)", + [text.replace('\\', "/")], + |row| row.get::<_, i64>(0), + ) + .ok() + }) + .is_some_and(|count| count > 0) +} + +/// How a data path stands, and any problem with it. `base` is what it is +/// relative to, `root` the project's folder. +fn data_status(path: &str, base: &Path, root: &Path, problems: &mut Vec) -> &'static str { + if is_remote(path) { + return "remote"; + } + let resolved = source::resolve(path, base); + if !exists(&resolved) { + return "missing"; + } + if looks_absolute(path) { + problems.push(problem( + "not_portable", + format!("{path} is an absolute path, so it will not move with the project"), + "Move it into the project and use a relative path, or run `ducklocal project \ + relink` on each machine" + .to_string(), + )); + return "absolute"; + } + let inside = match ( + std::fs::canonicalize(&resolved), + std::fs::canonicalize(root), + ) { + (Ok(file), Ok(root)) => file.starts_with(root), + // A glob has no one file to ask about. + _ => stays_inside(path), + }; + if !inside { + problems.push(problem( + "not_portable", + format!("{path} is outside the project folder, so it will not move with it"), + "Move it into the project folder if it should travel".to_string(), + )); + return "outside"; + } + "ok" +} + +fn check_dashboard( + relative: &str, + root: &Path, + problems: &mut Vec, + limitations: &mut Vec, +) -> Value { + let path = root.join(relative); + if !stays_inside(relative) { + problems.push(problem( + "not_portable", + format!("Dashboard {relative} is outside the project folder, so it was not checked"), + "List dashboards relative to the manifest, inside its folder, with forward slashes" + .to_string(), + )); + return json!({"path": relative, "status": "outside", "checked": false}); + } + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(error) => { + problems.push(problem( + "missing_dashboard", + format!("Dashboard {relative}: {error}"), + "Restore the file, or take it out of the manifest's dashboards".to_string(), + )); + return json!({"path": relative, "status": "missing", "checked": false}); + } + }; + let base = source::base_of(&path); + let mut sources = Vec::new(); + let mut remote = false; + if let Ok(parsed) = model::parse(&text) { + let (spec, _) = model::validate_partial(&parsed); + for named in &spec.sources { + let status = data_status(&named.path, &base, root, problems); + remote |= status == "remote"; + if status == "missing" { + problems.push(problem( + "missing_source", + format!( + "Source {:?} in {relative}: nothing at {}", + named.name, named.path + ), + format!( + "Bring the file back, or point at where it is now: ducklocal project \ + relink --source {} --to PATH", + named.name + ), + )); + } + sources.push(json!({"name": named.name, "path": named.path, "status": status})); + } + for query in &spec.queries { + for literal in paths_in_sql(&query.sql) { + if is_remote(&literal) { + remote = true; + } else { + problems.push(problem( + "not_portable", + format!( + "Query {:?} in {relative} names {literal} directly", + query.name + ), + "Name it in a source block with a relative path, so it moves with the \ + project" + .to_string(), + )); + } + } + } + } + // Checking runs the queries, and this must not reach out on its own. + if remote { + limitations.push(format!( + "{relative} reads remote data, so its queries were not run; it depends on that \ + address and any credentials still being good" + )); + return json!({ + "path": relative, + "status": "remote", + "checked": false, + "sources": sources, + }); + } + let verdict = proposal::check(&path, &text); + let status = if verdict.passed() { "ok" } else { "failed" }; + if let Some(failure) = &verdict.failure { + // A missing file is already reported, with its way out. + if !sources.iter().any(|source| source["status"] == "missing") { + problems.push(problem( + "failed_check", + format!("{relative}: {failure}"), + format!("See `ducklocal check {relative}`"), + )); + } + } + if !verdict.unresolved.is_empty() { + limitations.push(format!( + "{relative}: {} read tables not available here, so went unchecked", + verdict.unresolved.join(", ") + )); + } + if !verdict.truncated.is_empty() { + limitations.push(format!( + "{relative}: {} hit the row budget, so what they show is not all of it", + verdict.truncated.join(", ") + )); + } + json!({ + "path": relative, + "status": status, + "checked": true, + "failure": verdict.failure, + "unresolved": verdict.unresolved, + "empty": verdict.empty, + "truncated": verdict.truncated, + "sources": sources, + }) +} + +/// DuckDB's version, and how each extension stands on this machine. +fn engine() -> (Option, Vec<(String, bool, bool)>) { + let Ok(conn) = crate::cli::open(None, false) else { + return (None, Vec::new()); + }; + let version = conn + .query_row("SELECT library_version FROM pragma_version()", [], |row| { + row.get(0) + }) + .ok(); + let extensions = conn + .prepare("SELECT extension_name, installed, loaded FROM duckdb_extensions()") + .and_then(|mut statement| { + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))? + .collect() + }) + .unwrap_or_default(); + (version, extensions) +} + +fn check_project(manifest_path: &Path) -> Result { + let manifest = load(manifest_path).map_err(|e| CliError::failure("project", e))?; + let root = source::base_of(manifest_path); + let root = if root.as_os_str().is_empty() { + PathBuf::from(".") + } else { + root + }; + let mut problems = Vec::new(); + let mut limitations = vec![ + "Sources are checked for being there, not for holding the same data: a changed file \ + gives different results with no warning here" + .to_string(), + ]; + if manifest.dashboards.is_empty() { + problems.push(problem( + "no_dashboards", + "The manifest lists no dashboards".to_string(), + format!("Add their paths to dashboards in {MANIFEST}"), + )); + } + let dashboards: Vec = manifest + .dashboards + .iter() + .map(|dashboard| check_dashboard(dashboard, &root, &mut problems, &mut limitations)) + .collect(); + + let data: Vec = manifest + .data + .iter() + .map(|entry| { + let status = data_status(&entry.path, &root, &root, &mut problems); + let sample = entry.sample.as_ref().map(|sample| { + json!({ + "path": sample, + "there": source::resolve(sample, &root).exists(), + }) + }); + if status == "missing" { + let hint = match &entry.sample { + Some(sample) => format!( + "Bring it back, or relink it: ducklocal project relink --data {} --to \ + PATH. A sample is at {sample}", + entry.name + ), + None => format!( + "Bring it back, or relink it: ducklocal project relink --data {} --to PATH", + entry.name + ), + }; + problems.push(problem( + "missing_data", + format!("Data {:?}: nothing at {}", entry.name, entry.path), + hint, + )); + } + if status == "remote" { + limitations.push(format!( + "Data {:?} is remote and was not reached; it may need credentials this \ + machine does not have", + entry.name + )); + } + json!({"name": entry.name, "path": entry.path, "status": status, "sample": sample}) + }) + .collect(); + + let (version, installed) = engine(); + if let Some(wanted) = &manifest.requires.duckdb { + if version.as_deref() != Some(wanted.as_str()) { + limitations.push(format!( + "This work was done on DuckDB {wanted}, and this is {}: results and SQL \ + behaviour can differ", + version.as_deref().unwrap_or("an unknown version") + )); + } + } + let extensions: Vec = manifest + .requires + .extensions + .iter() + .map(|name| { + let known = installed.iter().find(|(known, _, _)| known == name); + let status = match known { + Some((_, _, true)) => "loaded", + Some((_, true, _)) => "installed", + _ => "missing", + }; + if status == "missing" { + problems.push(problem( + "missing_extension", + format!("Extension {name} is not installed"), + "Nothing installs it on its own. If you want it, use the Extensions tab, \ + which downloads it from DuckDB's server, then check again" + .to_string(), + )); + } + json!({"name": name, "status": status}) + }) + .collect(); + let ok = problems.iter().all(|p| p["kind"] == "not_portable") + && dashboards + .iter() + .all(|dashboard| dashboard["status"] != "failed"); + Ok(json!({ + "project": manifest.name, + "manifest": manifest_path.to_string_lossy(), + "ok": ok, + "portable": problems.iter().all(|p| p["kind"] != "not_portable"), + "ducklocal": env!("CARGO_PKG_VERSION"), + "duckdb": version, + "dashboards": dashboards, + "data": data, + "extensions": extensions, + "problems": problems, + "limitations": limitations, + })) +} + +fn find_dashboards(dir: &Path, root: &Path, depth: usize, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut entries: Vec = entries.flatten().map(|entry| entry.path()).collect(); + entries.sort(); + for path in entries { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + if name.starts_with('.') || name == "target" || name == "node_modules" { + continue; + } + if path.is_dir() { + if depth < MAX_DEPTH { + find_dashboards(&path, root, depth + 1, found); + } + } else if crate::spec::tabs::is_spec(&path) { + if let Ok(relative) = path.strip_prefix(root) { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } +} + +fn init(dir: &Path, name: Option) -> Result { + if !dir.is_dir() { + return Err(CliError::failure( + "io", + format!("{} is not a folder", dir.display()), + )); + } + let path = dir.join(MANIFEST); + if path.exists() { + return Err( + CliError::failure("exists", format!("{} is already there", path.display())) + .with_hint("Edit it, or see how it stands with `ducklocal project check`"), + ); + } + let mut dashboards = Vec::new(); + find_dashboards(dir, dir, 0, &mut dashboards); + let folder = std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf()); + let manifest = Manifest { + version: VERSION, + name: name.unwrap_or_else(|| { + folder + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() + }), + dashboards, + data: Vec::new(), + requires: Requires { + duckdb: engine().0, + extensions: Vec::new(), + }, + }; + write_manifest(&path, &manifest)?; + Ok(json!({"manifest": path.to_string_lossy(), "project": manifest})) +} + +fn write_manifest(path: &Path, manifest: &Manifest) -> Result<(), CliError> { + let text = serde_json::to_string_pretty(manifest).map_err(|e| CliError::failure("io", e))?; + proposal::write_whole(path, (text + "\n").as_bytes()).map_err(|e| CliError::failure("io", e)) +} + +/// `text` with the `path` of `source "name"` set to `to`, by position: nothing +/// else in the file moves. `None` when it has no such source. +fn with_source_path(text: &str, name: &str, to: &str) -> Option { + let parsed = model::parse(text).ok()?; + let (spec, _) = model::validate_partial(&parsed); + let named = spec.sources.iter().find(|source| source.name == name)?; + let after_name = named.name_span.1.min(text.len()); + let rest = &text[after_name..]; + let key = rest.find("path")?; + let open = key + rest[key..].find('"')?; + let close = open + 1 + rest[open + 1..].find('"')?; + let mut out = String::with_capacity(text.len() + to.len()); + out.push_str(&text[..after_name + open + 1]); + out.push_str(to); + out.push_str(&text[after_name + close..]); + // A rewrite that does not read back as intended is not written at all. + let reread = model::parse(&out).ok()?; + let (spec, _) = model::validate_partial(&reread); + spec.sources + .iter() + .any(|source| source.name == name && source.path == to) + .then_some(out) +} + +/// `to` as a dashboard or manifest in `base` should write it. Inside the project +/// it is relative to `base`, `..` and all, with forward slashes, so it moves +/// with the folder; outside, it can only be absolute, and `check` says so. +fn written_path(to: &Path, base: &Path, root: &Path) -> String { + let absolute = |path: &Path| -> PathBuf { + let path = std::fs::canonicalize(path) + .or_else(|_| std::path::absolute(path)) + .unwrap_or_else(|_| path.to_path_buf()); + PathBuf::from(plain(&path)) + }; + let (to, base, root) = (absolute(to), absolute(base), absolute(root)); + let text = if to.starts_with(&root) { + relative_to(&to, &base).to_string_lossy().into_owned() + } else { + to.to_string_lossy().into_owned() + }; + text.replace('\\', "/") +} + +/// Windows' canonical form carries a prefix nothing else writes. +fn plain(path: &Path) -> String { + let text = path.to_string_lossy(); + match text.strip_prefix(r"\\?\") { + Some(rest) if !rest.starts_with("UNC") => rest.to_string(), + _ => text.into_owned(), + } +} + +fn relative_to(to: &Path, base: &Path) -> PathBuf { + let to: Vec<_> = to.components().collect(); + let base: Vec<_> = base.components().collect(); + let shared = to.iter().zip(&base).take_while(|(a, b)| a == b).count(); + let mut out = PathBuf::new(); + for _ in shared..base.len() { + out.push(".."); + } + for part in &to[shared..] { + out.push(part); + } + out +} + +fn relink( + manifest_path: &Path, + source_name: Option<&str>, + data_name: Option<&str>, + to: &str, +) -> Result { + let mut manifest = load(manifest_path).map_err(|e| CliError::failure("project", e))?; + let root = source::base_of(manifest_path); + let target = PathBuf::from(crate::db::expand_tilde(to)); + if is_remote(to) || to.contains('"') { + return Err(CliError::argument( + "--to names a local path, without quotation marks", + )); + } + if !exists(&target) { + return Err(CliError::argument(format!("--to: nothing at {to}"))); + } + let mut changed = Vec::new(); + if let Some(name) = data_name { + let written = written_path(&target, &root, &root); + let entry = manifest + .data + .iter_mut() + .find(|entry| entry.name == name) + .ok_or_else(|| { + CliError::failure("not_found", format!("The manifest has no data {name:?}")) + })?; + entry.path = written; + write_manifest(manifest_path, &manifest)?; + changed.push(manifest_path.to_string_lossy().into_owned()); + } + if let Some(name) = source_name { + if let Some(problem) = source::unreadable(to) { + return Err(CliError::argument(problem)); + } + // Worked out for every dashboard first, so a failure changes none. + let mut edits = Vec::new(); + for dashboard in manifest.dashboards.iter().filter(|d| stays_inside(d)) { + let path = root.join(dashboard); + let Ok(text) = std::fs::read_to_string(&path) else { + continue; + }; + let written = written_path(&target, &source::base_of(&path), &root); + if let Some(updated) = with_source_path(&text, name, &written) { + edits.push((path, text, updated)); + } + } + if edits.is_empty() { + return Err(CliError::failure( + "not_found", + format!("No dashboard in the project has a source {name:?}"), + )); + } + for (path, text, updated) in edits { + let backup = proposal::backup_path_for(&path); + proposal::write_whole(&backup, text.as_bytes()) + .and_then(|()| proposal::write_whole(&path, updated.as_bytes())) + .map_err(|e| CliError::failure("io", e))?; + changed.push(path.to_string_lossy().into_owned()); + } + } + Ok(json!({"changed": changed, "backups": "beside each dashboard, as NAME.dash.bak"})) +} + +pub(crate) const HELP: &str = "\ +Usage: ducklocal project init [DIR] [--name NAME] + ducklocal project check [PATH] + ducklocal project relink [PATH] (--source NAME | --data NAME) --to PATH + +Keep an analysis movable: a folder with a ducklocal-project.json saying which +dashboards it holds and what they need. Optional: a lone .dash file works as it +always has. Data is referenced, never copied or uploaded. Opening the manifest +in the window opens its dashboards. PATH is the folder or the manifest, by +default the current folder. + + init Write a manifest listing the dashboards found under DIR, and this + DuckDB's version. Refuses to replace one (kind exists). + check Run `ducklocal check` over each dashboard and report what a move would + break. Never reaches the network: a dashboard reading remote data is + left unrun, and says so. Installs nothing. + relink Point a source at where its file is now, in every dashboard that + names it (each kept as NAME.dash.bak), or a manifest data entry. + Ask the person first: this changes their dashboards. + +The manifest, relative paths with forward slashes: + {\"version\": 1, \"name\": \"orders\", + \"dashboards\": [\"reports/orders.dash\"], + \"data\": [{\"name\": \"archive\", \"path\": \"data/old.csv\", \"sample\": \"data/s.csv\"}], + \"requires\": {\"duckdb\": \"v2.0.0\", \"extensions\": [\"httpfs\"]}} + +check prints {project, manifest, ok, portable, ducklocal, duckdb, dashboards, +data, extensions, problems, limitations} and exits 0 whatever it found, so read +ok: false means something is missing or failed, portable: false that a path will +not survive a move. Each problem has {kind, what, hint}. limitations says what +passing does not cover. Exit 2 for arguments, 1 with kind project for a manifest +that does not read, or not_found. +"; + +const SPEC: &[FlagSpec] = &[ + FlagSpec { + name: "--name", + takes_value: true, + }, + FlagSpec { + name: "--source", + takes_value: true, + }, + FlagSpec { + name: "--data", + takes_value: true, + }, + FlagSpec { + name: "--to", + takes_value: true, + }, +]; + +fn manifest_at(path: Option) -> Result { + let path = path.unwrap_or_else(|| PathBuf::from(".")); + let manifest = if path.is_dir() { + path.join(MANIFEST) + } else { + path + }; + if !manifest.is_file() { + return Err( + CliError::failure("not_found", format!("No {}", manifest.display())) + .with_hint("Make one with `ducklocal project init`"), + ); + } + Ok(manifest) +} + +/// `ducklocal project`. +pub(crate) fn run(args: &[OsString]) -> Result { + let Some(action) = args.first().and_then(|arg| arg.to_str()) else { + return Err(CliError::argument("Say what to do: init, check or relink")); + }; + let mut path = None; + let (mut name, mut source_name, mut data_name, mut to) = (None, None, None, None); + let utf8 = |value: OsString, what: &str| { + value + .into_string() + .map_err(|_| CliError::argument(format!("{what} must be UTF-8"))) + }; + for arg in parse_args("project", &args[1..], SPEC, &[])? { + match arg { + Arg::Flag("--name", Some(value)) => name = Some(utf8(value, "--name")?), + Arg::Flag("--source", Some(value)) => source_name = Some(utf8(value, "--source")?), + Arg::Flag("--data", Some(value)) => data_name = Some(utf8(value, "--data")?), + Arg::Flag("--to", Some(value)) => to = Some(utf8(value, "--to")?), + Arg::Positional(value) if path.is_none() => path = Some(PathBuf::from(value)), + Arg::Positional(_) => return Err(CliError::argument("Name one project")), + _ => { + return Err(CliError::failure( + "internal", + "project accepted a flag it does not handle", + )) + } + } + } + let relinking = source_name.is_some() || data_name.is_some() || to.is_some(); + let output = match action { + "init" if !relinking => init(&path.unwrap_or_else(|| PathBuf::from(".")), name)?, + "check" if !relinking && name.is_none() => check_project(&manifest_at(path)?)?, + "relink" if name.is_none() => { + let to = to.ok_or_else(|| CliError::argument("Say where with --to PATH"))?; + if source_name.is_some() == data_name.is_some() { + return Err(CliError::argument("Name one of --source and --data")); + } + relink( + &manifest_at(path)?, + source_name.as_deref(), + data_name.as_deref(), + &to, + )? + } + "init" | "check" | "relink" => { + return Err(CliError::argument(format!( + "Those options do not go with {action}; see ducklocal project --help" + ))) + } + other => { + return Err(CliError::argument(format!( + "Unknown project action: {other}; use init, check or relink" + ))) + } + }; + Ok(output.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_relative_path_inside_stays_inside() { + for inside in ["a.dash", "reports/a.dash", "./a.dash", "b/../a.dash"] { + assert!(stays_inside(inside), "{inside}"); + } + for outside in [ + "../a.dash", + "a/../../b.dash", + "/etc/a.dash", + "C:/a.dash", + r"C:\a.dash", + r"reports\a.dash", + "~/a.dash", + ] { + assert!(!stays_inside(outside), "{outside}"); + } + } + + #[test] + fn a_query_naming_a_fixed_or_remote_path_is_found() { + assert_eq!( + paths_in_sql("SELECT * FROM '/data/a.csv' JOIN 'b.csv' ON x = 'y'"), + vec!["/data/a.csv"] + ); + assert_eq!( + paths_in_sql("SELECT * FROM read_csv('s3://b/a.csv'), 'C:/x.csv'"), + vec!["s3://b/a.csv", "C:/x.csv"] + ); + assert!(paths_in_sql("SELECT 'it''s' AS a").is_empty()); + } + + #[test] + fn relinking_changes_that_one_path_and_nothing_else() { + let text = "// orders\nsource \"a\" {\n path = \"old.csv\"\n}\n\nsource \"b\" {\n \ + path = \"b.csv\"\n}\n\nquery \"q\" {\n sql = < String { /// Written whole and renamed into place, so nothing ever reads half a file and /// a failed write leaves the old one as it was. -fn write_whole(path: &Path, bytes: &[u8]) -> Result<(), String> { +pub(crate) fn write_whole(path: &Path, bytes: &[u8]) -> Result<(), String> { let mut name = path.file_name().map(OsString::from).unwrap_or_default(); name.push(".writing"); let temporary = path.with_file_name(name); diff --git a/src/spec/tabs.rs b/src/spec/tabs.rs index 1d4ccae..e4b4e80 100644 --- a/src/spec/tabs.rs +++ b/src/spec/tabs.rs @@ -48,6 +48,14 @@ pub fn split(paths: Vec) -> (Vec, Vec) { let mut rest = Vec::new(); for path in paths { let expanded = PathBuf::from(crate::db::expand_tilde(&path)); + // A project's manifest opens as its dashboards. One that does not read + // goes on as data, and opening it as data says what is wrong. + if crate::project::is_manifest(&expanded) { + if let Ok(dashboards) = crate::project::dashboards_of(&expanded) { + specs.extend(dashboards.into_iter().filter(|spec| is_spec(spec))); + continue; + } + } if is_spec(&expanded) { specs.push(expanded); } else { diff --git a/tests/cli.rs b/tests/cli.rs index 122b883..d358f89 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1911,6 +1911,277 @@ fn propose_stages_a_change_and_only_the_window_applies_it() { .contains("--based-on")); } +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let path = entry.unwrap().path(); + let target = to.join(path.file_name().unwrap()); + if path.is_dir() { + copy_tree(&path, &target); + } else { + std::fs::copy(&path, &target).unwrap(); + } + } +} + +fn project_report(s: &Sandbox, dir: &str) -> Value { + let output = s.run(&["project", "check", dir], None); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +/// The shipped example, moved: what the issue means by portable. +#[test] +fn a_project_moves_as_a_folder_and_says_what_a_move_broke() { + let s = Sandbox::new(); + let example = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/portable_project"); + copy_tree(&example, &s.0.join("moved/somewhere/else")); + let dir = "moved/somewhere/else"; + + let report = project_report(&s, dir); + assert_eq!(report["ok"], true, "{report}"); + assert_eq!(report["portable"], true, "{report}"); + assert_eq!(report["problems"], json!([])); + assert_eq!(report["dashboards"][0]["status"], "ok"); + assert_eq!(report["dashboards"][0]["checked"], true); + assert_eq!(report["dashboards"][0]["sources"][0]["status"], "ok"); + assert!(report["duckdb"].is_string()); + assert!(report["limitations"][0] + .as_str() + .unwrap() + .contains("not for holding the same data")); + + // The data did not come along: named, and the way out given. + let data = s.0.join(dir).join("data/orders.csv"); + let moved = s.0.join(dir).join("data/2026.csv"); + std::fs::rename(&data, &moved).unwrap(); + let report = project_report(&s, dir); + assert_eq!(report["ok"], false); + assert_eq!(report["problems"][0]["kind"], "missing_source"); + assert!(report["problems"][0]["hint"] + .as_str() + .unwrap() + .contains("relink --source orders")); + assert_eq!(report["problems"].as_array().unwrap().len(), 1, "{report}"); + + // Relinked: the dashboard says where it is now, relative, and still works. + let spec = s.0.join(dir).join("reports/orders.dash"); + let before = std::fs::read_to_string(&spec).unwrap(); + let to = format!("{dir}/data/2026.csv"); + let relinked = s.run( + &["project", "relink", dir, "--source", "orders", "--to", &to], + None, + ); + assert!( + relinked.status.success(), + "{}", + String::from_utf8_lossy(&relinked.stderr) + ); + let after = std::fs::read_to_string(&spec).unwrap(); + assert_eq!( + after, + before.replace("../data/orders.csv", "../data/2026.csv") + ); + assert_eq!( + std::fs::read_to_string(s.0.join(dir).join("reports/orders.dash.bak")).unwrap(), + before + ); + assert_eq!(project_report(&s, dir)["ok"], true); + + // Refused, with nothing changed. + s.error( + &[ + "project", + "relink", + dir, + "--source", + "orders", + "--to", + "absent.csv", + ], + 2, + "argument", + ); + s.error( + &["project", "relink", dir, "--source", "nope", "--to", &to], + 1, + "not_found", + ); + s.error(&["project", "relink", dir, "--to", &to], 2, "argument"); + assert_eq!(std::fs::read_to_string(&spec).unwrap(), after); +} + +#[test] +fn a_project_check_reports_what_will_not_travel_or_is_not_here() { + let s = Sandbox::new(); + let spec = |source: &str| { + format!( + "source \"d\" {{\n path = \"{source}\"\n}}\n\nquery \"q\" {{\n sql = \"SELECT * FROM d\"\n}}\n\n\ + plot \"p\" {{\n type = \"table\"\n query = query.q\n}}\n" + ) + }; + std::fs::create_dir_all(s.0.join("p")).unwrap(); + std::fs::write(s.0.join("p/a.csv"), "n\n1\n").unwrap(); + std::fs::write(s.0.join("outside.csv"), "n\n1\n").unwrap(); + let absolute = s.0.join("outside.csv").to_string_lossy().replace('\\', "/"); + std::fs::write(s.0.join("p/absolute.dash"), spec(&absolute)).unwrap(); + std::fs::write(s.0.join("p/above.dash"), spec("../outside.csv")).unwrap(); + std::fs::write( + s.0.join("p/remote.dash"), + "query \"q\" {\n sql = \"SELECT * FROM 'https://example.com/x.csv'\"\n}\n", + ) + .unwrap(); + std::fs::write(s.0.join("p/broken.dash"), "query \"q\" {").unwrap(); + std::fs::write( + s.0.join("p/ducklocal-project.json"), + json!({ + "version": 1, + "name": "t", + "dashboards": [ + "absolute.dash", "above.dash", "remote.dash", "broken.dash", + "absent.dash", "../escape.dash", + ], + "data": [ + {"name": "here", "path": "a.csv"}, + {"name": "gone", "path": "gone.csv", "sample": "a.csv"}, + {"name": "lake", "path": "s3://bucket/key.parquet"}, + ], + "requires": {"duckdb": "v0.0.1", "extensions": ["no_such_extension", "json"]}, + }) + .to_string(), + ) + .unwrap(); + + let report = project_report(&s, "p"); + assert_eq!(report["ok"], false); + assert_eq!(report["portable"], false); + let status = |name: &str| { + report["dashboards"] + .as_array() + .unwrap() + .iter() + .find(|d| d["path"] == name) + .unwrap() + .clone() + }; + // Reads, but will not travel. + assert_eq!(status("absolute.dash")["sources"][0]["status"], "absolute"); + assert_eq!(status("above.dash")["sources"][0]["status"], "outside"); + // Left unrun rather than reaching out, which would hang here anyway. + assert_eq!(status("remote.dash")["status"], "remote"); + assert_eq!(status("remote.dash")["checked"], false); + assert_eq!(status("broken.dash")["status"], "failed"); + assert_eq!(status("absent.dash")["status"], "missing"); + assert_eq!(status("../escape.dash")["status"], "outside"); + assert_eq!(status("../escape.dash")["checked"], false); + + let kinds: Vec<&str> = report["problems"] + .as_array() + .unwrap() + .iter() + .map(|p| p["kind"].as_str().unwrap()) + .collect(); + for kind in [ + "not_portable", + "failed_check", + "missing_dashboard", + "missing_data", + "missing_extension", + ] { + assert!(kinds.contains(&kind), "{kind} in {kinds:?}"); + } + assert!(report["problems"] + .as_array() + .unwrap() + .iter() + .all(|p| p["hint"].is_string())); + assert_eq!(report["data"][0]["status"], "ok"); + assert_eq!(report["data"][1]["status"], "missing"); + assert_eq!(report["data"][1]["sample"]["there"], true); + assert_eq!(report["data"][2]["status"], "remote"); + assert_eq!(report["extensions"][0]["status"], "missing"); + assert_ne!(report["extensions"][1]["status"], "missing"); + let limits = report["limitations"].to_string(); + assert!(limits.contains("v0.0.1"), "{limits}"); + assert!(limits.contains("remote.dash"), "{limits}"); + assert!(limits.contains("lake"), "{limits}"); + // A key in the address would not show, because the address is not repeated. + assert!(!limits.contains("key.parquet"), "{limits}"); + + // A data entry relinks in the manifest, and a dashboard keeps its own path. + let moved = s.run( + &[ + "project", "relink", "p", "--data", "gone", "--to", "p/a.csv", + ], + None, + ); + assert!( + moved.status.success(), + "{}", + String::from_utf8_lossy(&moved.stderr) + ); + assert_eq!(project_report(&s, "p")["data"][1]["path"], "a.csv"); + s.error( + &[ + "project", "relink", "p", "--data", "nope", "--to", "p/a.csv", + ], + 1, + "not_found", + ); +} + +#[test] +fn a_project_starts_from_what_is_in_the_folder() { + let s = Sandbox::new(); + std::fs::create_dir_all(s.0.join("p/reports")).unwrap(); + std::fs::create_dir_all(s.0.join("p/.hidden")).unwrap(); + let spec = "query \"q\" {\n sql = \"SELECT 1 AS n\"\n}\n"; + std::fs::write(s.0.join("p/top.dash"), spec).unwrap(); + std::fs::write(s.0.join("p/reports/sub.dash"), spec).unwrap(); + std::fs::write(s.0.join("p/.hidden/no.dash"), spec).unwrap(); + + let made = s.run(&["project", "init", "p", "--name", "mine"], None); + assert!( + made.status.success(), + "{}", + String::from_utf8_lossy(&made.stderr) + ); + let manifest: Value = serde_json::from_str( + &std::fs::read_to_string(s.0.join("p/ducklocal-project.json")).unwrap(), + ) + .unwrap(); + assert_eq!(manifest["name"], "mine"); + assert_eq!( + manifest["dashboards"], + json!(["reports/sub.dash", "top.dash"]) + ); + assert!(manifest["requires"]["duckdb"].is_string()); + assert_eq!(project_report(&s, "p")["ok"], true); + + // It will not replace one, and says what to do instead. + let error = s.error(&["project", "init", "p"], 1, "exists"); + assert!(error["error"]["hint"].is_string()); + // Mistakes. + s.error(&["project"], 2, "argument"); + s.error(&["project", "dance"], 2, "argument"); + s.error(&["project", "check", "no_such_folder"], 1, "not_found"); + s.error(&["project", "init", "no_such_folder"], 1, "io"); + s.error(&["project", "check", "p", "--to", "x"], 2, "argument"); + std::fs::create_dir_all(s.0.join("bad")).unwrap(); + std::fs::write(s.0.join("bad/ducklocal-project.json"), "{ nope").unwrap(); + s.error(&["project", "check", "bad"], 1, "project"); + let help = s.run(&["project", "--help"], None); + assert!(String::from_utf8(help.stdout).unwrap().contains("relink")); + // A lone dashboard needs none of this. + std::fs::write(s.0.join("lone.dash"), spec).unwrap(); + let lone = s.run(&["check", "lone.dash"], None); + assert!(lone.status.success()); +} + /// A session, end to end through the real commands: what it records, what it /// keeps out, and what it says once the world has moved on. #[test]