Data: From Web ▶️ import a table from a web page, with one-click refresh

Cargo.toml

@@ -23,6 +23,7 @@ gridcraft-chart = { path = "crates/chart" }
 gridcraft-engine = { path = "crates/engine" }
 gridcraft-ui-egui = { path = "crates/ui-egui" }
 gridcraft-mcp = { path = "crates/mcp" }
+gridcraft-webquery = { path = "crates/webquery" }
 gridcraft-pdf = { path = "crates/pdf" }
 # Every craft-libs crate (craft-update and craft-window here, craft-ui in ui-egui, craft-print in
 # the app) comes from one storytold/craft-libs revision.

apps/gridcraft-cli/Cargo.toml

@@ -15,6 +15,7 @@ path = "src/main.rs"
 
 [dependencies]
 gridcraft-engine = { workspace = true }
+gridcraft-webquery = { workspace = true }
 gridcraft-mcp = { workspace = true }
 serde_json = { workspace = true }
 log = { workspace = true }

apps/gridcraft-cli/src/main.rs

@@ -165,14 +165,23 @@ fn exec(s: &mut Session, id: &str, params: Value) -> Result<Value, String> {
     r
 }
 
-fn open(path: &str) -> Result<Session, String> {
+/// A session wired for web imports.
+fn session() -> Session {
     let mut s = Session::new();
+    s.web_source = Some(gridcraft_webquery::provider());
+    s
+}
+
+/// A session with the file at `path` open, wired for web imports.
+fn open(path: &str) -> Result<Session, String> {
+    let mut s = session();
     exec(&mut s, "file.open", json!({"path": path}))?;
     Ok(s)
 }
 
+/// A session with a fresh workbook, wired for web imports.
 fn blank() -> Session {
-    let mut s = Session::new();
+    let mut s = session();
     s.new_workbook();
     s
 }
@@ -418,7 +427,7 @@ fn run(args: &[String]) -> Result<(), String> {
         (Some(_), Some(_)) => return Err("give --in or --sample, not both".into()),
         (Some(p), None) => open(p)?,
         (None, Some(name)) => {
-            let mut s = Session::new();
+            let mut s = session();
             exec(&mut s, "file.new", json!({"sample": name}))?;
             s
         }
@@ -540,7 +549,7 @@ fn mcp(args: &[String]) -> Result<(), String> {
             let session = match (a.opt("in"), a.opt("sample")) {
                 (Some(p), _) => open(p)?,
                 (None, Some(name)) => {
-                    let mut s = Session::new();
+                    let mut s = session();
                     exec(&mut s, "file.new", json!({"sample": name}))?;
                     s
                 }

apps/gridcraft/Cargo.toml

@@ -12,6 +12,7 @@ workspace = true
 
 [dependencies]
 gridcraft-engine = { workspace = true }
+gridcraft-webquery = { workspace = true }
 gridcraft-ui-egui = { workspace = true }
 eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "clipboard"] }
 egui = { workspace = true }

apps/gridcraft/src/main.rs

@@ -233,6 +233,7 @@ fn icon() -> Option<egui::IconData> {
     Some(egui::IconData { width: img.width(), height: img.height(), rgba: img.into_raw() })
 }
 
+/// The desktop app: logging, the window, the session with its web source, and the event loop.
 fn main() -> eframe::Result<()> {
     // First, so the panic hook and every start-up warning are recorded (`logging`).
     let logger = logging::install();
@@ -306,6 +307,7 @@ fn main() -> eframe::Result<()> {
         }
     }
     let mut session = Session::new();
+    session.web_source = Some(gridcraft_webquery::provider());
     if let Some(s) = &sample
         && let Err(e) = session.execute("file.new", json!({"sample": s}))
     {

crates/engine/src/cmd.rs

@@ -21,6 +21,7 @@ pub mod solver;
 pub mod spelling;
 pub mod stash;
 pub mod view;
+pub mod web;
 pub mod whatif;
 
 use gridcraft_calc::Key;
@@ -79,12 +80,21 @@ pub fn is_secret_param(key: &str) -> bool {
     key == "password" || key.ends_with("Password")
 }
 
+/// Params too large for the journal: a background worker's downloaded page tables. Replaying
+/// the entry re-fetches the page instead (the commands fall back to the fetch source without
+/// the param).
+fn is_bulk_param(key: &str) -> bool {
+    key == "prefetched" || key == "prefetchedByUrl"
+}
+
 /// The params as the journal records them: secret params are left out, so a password never
 /// reaches the journal dialog, `engine.journal` or a recorded script (saved in the workbook).
 /// Replaying such an entry runs the command without that param.
 pub fn journal_params(params: &Json) -> Json {
     match params {
-        Json::Object(o) => Json::Object(o.iter().filter(|(k, _)| !is_secret_param(k)).map(|(k, v)| (k.clone(), v.clone())).collect()),
+        Json::Object(o) => {
+            Json::Object(o.iter().filter(|(k, _)| !is_secret_param(k) && !is_bulk_param(k)).map(|(k, v)| (k.clone(), v.clone())).collect())
+        }
         other => other.clone(),
     }
 }
@@ -128,6 +138,7 @@ macro_rules! cmd {
 }
 pub(crate) use cmd;
 
+/// Every command the engine holds, in menu order, built once.
 pub fn command_specs() -> &'static [CommandSpec] {
     static SPECS: std::sync::OnceLock<Vec<CommandSpec>> = std::sync::OnceLock::new();
     SPECS.get_or_init(|| {
@@ -138,6 +149,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
         v.extend(sheet::specs());
         v.extend(insert::specs());
         v.extend(data::specs());
+        v.extend(web::specs());
         v.extend(formulas::specs());
         v.extend(review::specs());
         v.extend(view::specs());
@@ -386,6 +398,14 @@ pub(crate) fn commit<R>(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result<R>)
     if let Some(t0) = t0 {
         eprintln!("commit: + rows {:?}", t0.elapsed());
     }
+    // A saved From Web connection whose table is gone goes with it, whatever deleted the table
+    // (deleted rows or columns collapsing its range, Convert to Range, a deleted sheet): a
+    // dangling id could later match a new table that reuses the id. A set, not a sparse vector:
+    // ids come from opened files, so the allocation must depend on the table count, not the id.
+    if !wb.web_queries.is_empty() {
+        let live: std::collections::HashSet<u32> = wb.sheets.iter().flat_map(|sh| sh.tables.iter().map(|t| t.id)).collect();
+        wb.web_queries.retain(|q| q.table_id.is_none_or(|id| live.contains(&id)));
+    }
     d.wb = std::sync::Arc::new(wb);
     d.selection = sel;
     Ok(r)

crates/engine/src/cmd/extra/data.rs

@@ -107,6 +107,7 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result<Json> {
     Ok(json!({"sheet": r["name"], "rows": rows.len(), "cols": w}))
 }
 
+/// `data.fromTableRange`: the selected block as CSV text into a copy of its sheet.
 pub(super) fn from_table_range(s: &mut Session, p: &Json) -> Result<Json> {
     let r = target_range(s, p)?;
     let d = s.doc()?;
@@ -116,16 +117,36 @@ pub(super) fn from_table_range(s: &mut Session, p: &Json) -> Result<Json> {
     from_csv(s, &json!({"text": text, "delimiter": "\t", "sheet": format!("{} (2)", sh.name).chars().take(31).collect::<String>()}))
 }
 
+/// `data.queriesConnections`: the saved From Web connections (source, last sync) followed by
+/// the workbook's imported tables — the Queries & Connections pane's list.
 pub(super) fn queries(s: &mut Session, _: &Json) -> Result<Json> {
     let d = s.doc()?;
-    Ok(Json::Array(
+    let tables: Vec<Json> =
         d.wb.sheets
             .iter()
             .flat_map(|sh| {
                 sh.tables
                     .iter()
                     .map(move |t| json!({"name": t.name, "sheet": sh.name, "range": t.range.a1(), "rows": t.range.height().saturating_sub(1)}))
             })
-            .collect(),
-    ))
+            .collect();
+    // The saved From Web connections, with the source and the last sync the pane shows. The
+    // sheet and name are resolved from the live table (by its stable id) so the pane and "Go to
+    // table" follow renames even in workbooks saved before connections tracked them.
+    let web: Vec<Json> =
+        d.wb.web_queries
+            .iter()
+            .map(|q| {
+                let live = q.table_id.and_then(|id| {
+                    d.wb.sheets.iter().find_map(|sh| sh.tables.iter().find(|t| t.id == id).map(|t| (t.name.clone(), sh.name.clone())))
+                });
+                json!({
+                    "name": live.as_ref().map(|(n, _)| n).unwrap_or(&q.name),
+                    "sheet": live.as_ref().map(|(_, s)| s).unwrap_or(&q.sheet),
+                    "url": q.url, "table": q.table,
+                    "last_synced": q.last_synced.unwrap_or(0), "web": true,
+                })
+            })
+            .collect();
+    Ok(Json::Array([web, tables].concat()))
 }

crates/engine/src/cmd/insert.rs

@@ -21,6 +21,7 @@ use chart::{chart_delete, chart_set, chart_switch, insert_chart};
 use cond_format::{add_cf, clear_cf, insert_icon, manage_cf};
 use links::{comment, delete_comment, insert_link, open_link, reply_comment, resolve_comment, show_note};
 use objects::{delete_object, insert_picture, insert_shape, insert_sparkline, move_object, set_anchor_mode, shape_set_text};
+pub(crate) use table::write_totals;
 use table::{format_as_table, insert_table, table_convert, table_flag, table_rename, table_style, table_total_fn};
 
 pub fn specs() -> Vec<CommandSpec> {

crates/engine/src/cmd/insert/table.rs

@@ -198,7 +198,7 @@ pub(super) fn table_flag(s: &mut Session, p: &Json, which: &str) -> Result<Json>
 }
 
 /// Writes the totals row cells (labels and SUBTOTAL formulas).
-fn write_totals(sh: &mut Sheet, ti: usize) {
+pub(crate) fn write_totals(sh: &mut Sheet, ti: usize) {
     let Some(t) = sh.tables.get(ti).cloned() else { return };
     if !t.totals_row {
         return;
@@ -282,6 +282,9 @@ pub(super) fn table_rename(s: &mut Session, p: &Json) -> Result<Json> {
         if let Some(t) = cx.sheet_mut(si)?.tables.get_mut(ti) {
             t.name = name.clone();
         }
+        // A saved From Web connection to this table follows the rename.
+        let sheet = cx.wb.sheet(si).map(|sh| sh.name.clone()).unwrap_or_default();
+        crate::cmd::web::rename_connection(&mut cx.wb, &sheet, &old, &name);
         // Structured references follow the rename.
         for i in 0..cx.wb.sheets.len() {
             let keys: Vec<(CellRef, std::sync::Arc<Formula>)> =

crates/engine/src/cmd/sheet/tabs.rs

@@ -140,6 +140,8 @@ pub(super) fn rename_sheet(s: &mut Session, p: &Json) -> Result<Json> {
                 n.formula = gridcraft_formula::print(&gridcraft_formula::adjust::rename_sheet(e, &old, &name));
             }
         }
+        // Saved From Web connections on the sheet follow its rename.
+        crate::cmd::web::rename_connection_sheet(&mut cx.wb, &old, &name);
         cx.structural = true;
         Ok(Json::Null)
     })