From b039f44305bcbd2f71d7fdf290daea3e25e20990 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 29 May 2026 12:07:24 +0200 Subject: [PATCH 01/12] fix(dashboard): clear stale auth cookies on token validation failure When /_api/auth/me returns non-200, dashboard_auth/dashboard_admin_auth redirected to login but left the dead sdb_token/sdb_server cookies in place, so an expired/stale token could trap the user in a redirect loop. Expire the cookies on validation failure so the next login isn't shadowed by the dead one. Defensive hardening; the primary cause of the recent login loop was a proxy cookie-coalescing bug fixed separately in soli-proxy. Co-Authored-By: Claude Opus 4.8 (1M context) --- www/config/middleware_config.lua | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/www/config/middleware_config.lua b/www/config/middleware_config.lua index 69463eec..172a46a3 100644 --- a/www/config/middleware_config.lua +++ b/www/config/middleware_config.lua @@ -84,6 +84,16 @@ local function read_sdb_server_cookie() return raw, raw end +-- Expire the auth cookies so an invalid/stale token self-heals on the next +-- login instead of trapping the user in a redirect loop: the browser sends an +-- old token, validation 401s, we bounce to /login, the old cookie is still +-- there, repeat. Clearing here matches the host-only Path=/ cookies set by the +-- login controller (dashboard_controller#do_login). +local function clear_auth_cookies() + SetCookie("sdb_token", "", { MaxAge = 0, Path = "/" }) + SetCookie("sdb_server", "", { MaxAge = 0, Path = "/" }) +end + -- Dashboard authentication middleware Middleware.register("dashboard_auth", function(ctx, next) local token = GetCookie("sdb_token") @@ -130,6 +140,9 @@ Middleware.register("dashboard_auth", function(ctx, next) if status ~= 200 then Log(kLogWarn, "[dashboard_auth] Token validation failed, status=" .. tostring(status) .. ", body=" .. tostring(body)) + -- Token is present but rejected (expired/stale): drop it so the next login + -- isn't shadowed by the dead cookie. + clear_auth_cookies() local current_path = GetPath() or "/" return ctx:redirect("/dashboard/login?redirect=" .. current_path) end @@ -161,11 +174,15 @@ Middleware.register("dashboard_admin_auth", function(ctx, next) Log(kLogInfo, "[dashboard_admin_auth] Validation status=" .. tostring(status)) if status ~= 200 then + -- Token present but rejected (expired/stale): clear it so the user isn't + -- stuck re-submitting login behind a dead cookie. + clear_auth_cookies() return ctx:redirect("/dashboard/login?redirect=" .. current_path) end local ok, user_data = pcall(DecodeJson, body) if not ok or not user_data then + clear_auth_cookies() return ctx:redirect("/dashboard/login?redirect=" .. current_path) end From 4fedf4056718ba5405f98757289fb8f58fc90834 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Sat, 30 May 2026 15:01:56 +0200 Subject: [PATCH 02/12] Bound timeouts on the SoliDB connection path to fix hung requests Production requests intermittently stalled (~5s, sometimes indefinitely) with the system idle, surfacing only under DB use. Root cause: hops in the request path waited for a reply with no bounded timeout, so a stale/half-open connection or a stalled shard op parked a thread forever. - main.rs: bound the per-connection protocol-sniff read (10s); drive HTTP via hyper_util's auto Builder with an HTTP/1 header_read_timeout (30s) instead of axum::serve, preserving graceful shutdown (adds hyper-util dependency). - sdbql/executor/execution/clauses.rs: sharded INSERT/UPDATE/DELETE use recv_sharded with a 10s recv_timeout instead of an unbounded blocking recv(); adds unit tests. - error.rs: new DbError::Timeout mapped to HTTP 504. Also folds in in-progress dashboard/docs view edits already present in the tree. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 3 + src/error.rs | 4 ++ src/main.rs | 79 ++++++++++++++++++++--- src/sdbql/executor/execution/clauses.rs | 83 ++++++++++++++++++++++--- www/app/views/dashboard/env.etlua | 2 +- www/app/views/docs/scripting-core.etlua | 4 +- 7 files changed, 157 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7bbeb2af..6c9dc4f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4442,6 +4442,7 @@ dependencies = [ "hex", "hmac", "hostname", + "hyper-util", "image", "indicatif", "jsonschema", diff --git a/Cargo.toml b/Cargo.toml index ac943ebb..e93ea5c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,9 @@ tokio = { version = "1.35", features = ["full"] } # HTTP server axum = { version = "0.8.7", features = ["multipart", "ws", "macros"] } +# Drive HTTP connections manually (instead of `axum::serve`) so we can set an +# HTTP/1 header-read timeout — see the multiplexed HTTP server in main.rs. +hyper-util = { version = "0.1", features = ["server-auto", "server-graceful", "service", "tokio", "http1", "http2"] } tower = { version = "0.5.2", features = ["util"] } tower-http = { version = "0.6.8", features = ["trace", "cors", "compression-gzip", "compression-zstd"] } async-stream = "0.3" diff --git a/src/error.rs b/src/error.rs index 0e64df7c..15bf5cd6 100644 --- a/src/error.rs +++ b/src/error.rs @@ -42,6 +42,9 @@ pub enum DbError { #[error("Internal error: {0}")] InternalError(String), + #[error("Operation timed out: {0}")] + Timeout(String), + #[error("Network error: {0}")] NetworkError(String), @@ -114,6 +117,7 @@ impl IntoResponse for DbError { DbError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.clone()), DbError::OperationNotSupported(msg) => (StatusCode::NOT_IMPLEMENTED, msg.clone()), DbError::TransactionTimeout(_) => (StatusCode::REQUEST_TIMEOUT, self.to_string()), + DbError::Timeout(_) => (StatusCode::GATEWAY_TIMEOUT, self.to_string()), // Default to 500 _ => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()), }; diff --git a/src/main.rs b/src/main.rs index bc048ab6..26c5b193 100644 --- a/src/main.rs +++ b/src/main.rs @@ -540,14 +540,56 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let (sync_tx, sync_rx) = mpsc::channel(100); // 1. Spawn HTTP Server + // + // Driven via hyper_util's auto Builder rather than `axum::serve` so we + // can set an HTTP/1 header-read timeout. Without it, a client that + // opens a keep-alive connection and then sends partial (or no) request + // headers parks a server task indefinitely — the keep-alive analogue of + // the unbounded protocol-sniff read bounded above. Graceful shutdown is + // preserved via `GracefulShutdown`. let channel_listener = ChannelListener::new(http_rx, local_addr); + let http_shutdown = shutdown_signal(shutdown_storage); tokio::spawn(async move { - if let Err(e) = axum::serve(channel_listener, app) - .with_graceful_shutdown(shutdown_signal(shutdown_storage)) - .await - { - tracing::error!("HTTP server error: {}", e); + use axum::serve::Listener; + use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; + use hyper_util::server::conn::auto::Builder as HttpConnBuilder; + use hyper_util::server::graceful::GracefulShutdown; + use hyper_util::service::TowerToHyperService; + + let mut listener = channel_listener; + let graceful = GracefulShutdown::new(); + tokio::pin!(http_shutdown); + + loop { + let (io, _addr) = tokio::select! { + conn = listener.accept() => conn, + _ = &mut http_shutdown => { + tracing::info!("HTTP server received shutdown signal, draining connections"); + break; + } + }; + + let mut builder = HttpConnBuilder::new(TokioExecutor::new()); + // Bound the time a connection may take to send a complete set + // of request headers (defends against slow/half-open clients). + // `header_read_timeout` requires a registered timer. + builder + .http1() + .timer(TokioTimer::new()) + .header_read_timeout(Duration::from_secs(30)); + let service = TowerToHyperService::new(app.clone()); + let conn = builder + .serve_connection_with_upgrades(TokioIo::new(io), service) + .into_owned(); + let watched = graceful.watch(conn); + tokio::spawn(async move { + if let Err(e) = watched.await { + tracing::debug!("HTTP connection error: {}", e); + } + }); } + + graceful.shutdown().await; }); // 2. Spawn Sync Worker (background mode) @@ -611,9 +653,32 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let connection_mgr = cluster_manager.clone(); tokio::spawn(async move { - // Read initial bytes to determine protocol + // Read initial bytes to determine protocol. + // + // Bound this read: a connection that is accepted but + // never sends bytes (half-open keep-alive reuse, a + // dead pooled connection, a port scanner) would + // otherwise park this task forever holding the socket. + // That unbounded wait is a primary source of the + // intermittent "idle pending" stalls seen in prod. let mut buf = vec![0u8; 14]; - let n = stream.read(&mut buf).await.unwrap_or(0); + let n = match tokio::time::timeout( + std::time::Duration::from_secs(10), + stream.read(&mut buf), + ) + .await + { + Ok(Ok(n)) => n, + Ok(Err(_)) => 0, + Err(_) => { + tracing::warn!( + layer = "solidb_detect", + peer = %addr, + "protocol detection read timed out; dropping connection" + ); + return; + } + }; let peeked_data = buf[..n].to_vec(); diff --git a/src/sdbql/executor/execution/clauses.rs b/src/sdbql/executor/execution/clauses.rs index 2853b99d..f0142d87 100644 --- a/src/sdbql/executor/execution/clauses.rs +++ b/src/sdbql/executor/execution/clauses.rs @@ -18,6 +18,42 @@ use crate::error::{DbError, DbResult}; use crate::sdbql::ast::*; use crate::sync::protocol::Operation; +/// Block on the result of a sharded mutation that was spawned onto the tokio +/// runtime, bounding the wait. The executor thread is synchronous and uses a +/// `sync_channel` to hand the async coordinator call its result; an unbounded +/// `recv()` there parks this thread forever if the spawned future stalls +/// (unreachable shard, lock, fsync) — surfacing to callers as an idle, never +/// answered request. `recv_timeout` turns that into a 504 instead. +fn recv_sharded(rx: std::sync::mpsc::Receiver>, op: &str) -> DbResult { + recv_sharded_with(rx, op, std::time::Duration::from_secs(10)) +} + +fn recv_sharded_with( + rx: std::sync::mpsc::Receiver>, + op: &str, + wait: std::time::Duration, +) -> DbResult { + use std::sync::mpsc::RecvTimeoutError; + match rx.recv_timeout(wait) { + Ok(inner) => inner, + Err(RecvTimeoutError::Timeout) => { + tracing::warn!( + layer = "solidb", + op = op, + timeout_secs = wait.as_secs(), + "sharded operation timed out" + ); + Err(DbError::Timeout(format!( + "sharded {op} exceeded {}s", + wait.as_secs() + ))) + } + Err(RecvTimeoutError::Disconnected) => { + Err(DbError::InternalError(format!("sharded {op} task failed"))) + } + } +} + impl<'a> QueryExecutor<'a> { /// Execute body clauses and return row contexts with mutation stats pub(super) fn execute_body_clauses( @@ -174,9 +210,7 @@ impl<'a> QueryExecutor<'a> { }); // Wait for batch result - let result = rx.recv().map_err(|_| { - DbError::InternalError("Sharded batch insert failed".to_string()) - })??; + let result = recv_sharded(rx, "insert")?; tracing::debug!( "INSERT: Sharded batch completed - {} success, {} failed", result.0, @@ -335,9 +369,7 @@ impl<'a> QueryExecutor<'a> { let res = coord.update(&db, &coll, &conf, &k, doc).await; let _ = tx.send(res); }); - let updated_doc = rx.recv().map_err(|_| { - DbError::InternalError("Sharded update task failed".to_string()) - })??; + let updated_doc = recv_sharded(rx, "update")?; stats.documents_updated += 1; // Inject NEW variable @@ -518,9 +550,7 @@ impl<'a> QueryExecutor<'a> { let res = coord.delete(&db, &coll, &conf, &k).await; let _ = tx.send(res); }); - rx.recv().map_err(|_| { - DbError::InternalError("Sharded remove task failed".to_string()) - })??; + recv_sharded(rx, "remove")?; stats.documents_removed += 1; } i += 1; // CRITICAL: Advance to next clause @@ -1234,3 +1264,38 @@ impl<'a> QueryExecutor<'a> { Ok((rows, stats)) } } + +#[cfg(test)] +mod recv_sharded_tests { + use super::*; + use std::time::Duration; + + #[test] + fn timeout_when_sender_never_sends() { + // Hold the sender open but never send → recv_timeout elapses → + // a bounded 504-mapped Timeout, not an indefinite park. + let (_tx, rx) = std::sync::mpsc::sync_channel::>(1); + let err = recv_sharded_with(rx, "insert", Duration::from_millis(50)).unwrap_err(); + assert!(matches!(err, DbError::Timeout(_)), "got {err:?}"); + } + + #[test] + fn disconnected_when_sender_dropped() { + // Sender dropped without sending (e.g. spawned task panicked) → + // InternalError, distinct from the timeout case. + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + drop(tx); + let err = recv_sharded_with(rx, "update", Duration::from_secs(10)).unwrap_err(); + assert!(matches!(err, DbError::InternalError(_)), "got {err:?}"); + } + + #[test] + fn passes_through_inner_result() { + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + tx.send(Ok(7)).unwrap(); + assert_eq!( + recv_sharded_with(rx, "remove", Duration::from_secs(10)).unwrap(), + 7 + ); + } +} diff --git a/www/app/views/dashboard/env.etlua b/www/app/views/dashboard/env.etlua index 58421766..f9e8e969 100644 --- a/www/app/views/dashboard/env.etlua +++ b/www/app/views/dashboard/env.etlua @@ -21,7 +21,7 @@
-

Environment variables are available in Lua scripts via os.getenv("VAR_NAME")

+

Environment variables are available in Lua scripts via solidb.env.VAR_NAME

Changes require a script reload to take effect

diff --git a/www/app/views/docs/scripting-core.etlua b/www/app/views/docs/scripting-core.etlua index a46a4036..7f284812 100644 --- a/www/app/views/docs/scripting-core.etlua +++ b/www/app/views/docs/scripting-core.etlua @@ -45,8 +45,8 @@
Generate a V4 UUID.
  • - solidb.env(key) -> string -
    Read permitted environment variable (must start with PUBLIC_).
    + solidb.env -> table +
    Hash of environment variables loaded from the _env collection. Access by key, e.g. solidb.env.MY_KEY.
    • From 81137bc339aacc1c7bbf3712f17abaefa1bdaa89 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Sat, 30 May 2026 15:02:22 +0200 Subject: [PATCH 03/12] chore: release v0.26.1 Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c9dc4f7..ae10e071 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4414,7 +4414,7 @@ dependencies = [ [[package]] name = "solidb" -version = "0.26.0" +version = "0.26.1" dependencies = [ "anyhow", "argon2", diff --git a/Cargo.toml b/Cargo.toml index e93ea5c3..b3283459 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["clients/rust-client", "sdbql-core", "benchmarks"] [package] name = "solidb" -version = "0.26.0" +version = "0.26.1" edition = "2021" default-run = "solidb" description = "A lightweight, high-performance structured database server written in Rust." From de57b215142f626eb9d7699a63b99ea083ca5686 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Wed, 3 Jun 2026 16:36:50 +0200 Subject: [PATCH 04/12] perf(server): hot-path optimizations with security hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Performance: - Prepared-statement and query caches: sync parking_lot RwLock (no .await on hot paths) with a per-collection invalidation index under the same lock — invalidation is O(matches), eviction prunes only its own index sets, and emptied sets are dropped (no leak) - New in-memory ApiKeyCache (Arc): validate_api_key is O(1) instead of scanning _api_keys per request; pre-warmed at startup, kept in sync by handlers AND replication apply paths - Accept loop: try_send dispatch (8192-deep channels) so a saturated protocol worker drops new connections instead of blocking accepts - SyncLog::append: entry + sequence in one WriteBatch (one fsync) - Scans: scan_values() skips the intermediate Document allocation - storage/collection: std RwLock -> parking_lot (no poisoning) - Compression: skip 4xx/5xx and video/audio, composed with tower-http's DefaultPredicate (size/SSE/gRPC exemptions preserved) Security/correctness: - Login rate limiter keys on the real socket IP via ConnectInfo; X-Forwarded-For/X-Real-IP only trusted with SOLIDB_TRUST_PROXY_HEADERS=1 (headers are client-spoofable); limiter and Basic-auth caches are bounded LRUs (50K/10K) - Replicated _api_keys deletes evict the auth cache so revoked keys stop authenticating on peer nodes immediately - Schema validator cache sets hash+validator under both write guards (no mismatched pair under concurrent schema updates) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/main.rs | 75 +++-- src/sdbql/executor/data_source.rs | 21 +- src/sdbql/prepared.rs | 227 ++++++++++---- src/server/auth.rs | 303 +++++++++++++++---- src/server/handlers/auth.rs | 54 +++- src/server/handlers/blob_upload.rs | 4 +- src/server/handlers/blobs.rs | 8 +- src/server/handlers/collections/lifecycle.rs | 4 +- src/server/handlers/collections/ops.rs | 4 +- src/server/handlers/documents.rs | 20 +- src/server/handlers/query.rs | 22 +- src/server/routes.rs | 31 +- src/server/transaction_handlers.rs | 2 +- src/storage/collection/blobs.rs | 6 +- src/storage/collection/core.rs | 8 +- src/storage/collection/crud.rs | 20 +- src/storage/collection/mod.rs | 3 +- src/storage/collection/schema.rs | 36 ++- src/storage/collection/txn.rs | 6 +- src/storage/query_cache.rs | 256 +++++++++++++--- src/sync/log.rs | 14 +- src/sync/worker.rs | 37 +++ 22 files changed, 887 insertions(+), 274 deletions(-) diff --git a/src/main.rs b/src/main.rs index 26c5b193..36b76f1e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -535,9 +535,14 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let local_addr = listener.local_addr()?; - // Channels for dispatch - let (http_tx, http_rx) = mpsc::channel(100); - let (sync_tx, sync_rx) = mpsc::channel(100); + // Channels for dispatch. Sized to absorb burst traffic between + // accept and the protocol worker without back-pressuring the + // accept loop. 8192 is enough to hold several seconds of typical + // in-flight requests even at 10K req/s. If the receiver is + // genuinely saturated, dropping new accepts is preferable to + // head-of-line blocking behind an `await` on `send()`. + let (http_tx, http_rx) = mpsc::channel(8192); + let (sync_tx, sync_rx) = mpsc::channel(8192); // 1. Spawn HTTP Server // @@ -560,8 +565,14 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let graceful = GracefulShutdown::new(); tokio::pin!(http_shutdown); + // Provide `ConnectInfo` to handlers (e.g. login rate + // limiting keys on the real peer address rather than spoofable + // proxy headers). + let mut make_service = + app.into_make_service_with_connect_info::(); + loop { - let (io, _addr) = tokio::select! { + let (io, addr) = tokio::select! { conn = listener.accept() => conn, _ = &mut http_shutdown => { tracing::info!("HTTP server received shutdown signal, draining connections"); @@ -577,7 +588,16 @@ async fn async_main(args: Args) -> anyhow::Result<()> { .http1() .timer(TokioTimer::new()) .header_read_timeout(Duration::from_secs(30)); - let service = TowerToHyperService::new(app.clone()); + // Infallible: IntoMakeServiceWithConnectInfo is always ready + // and its error type is Infallible. + let tower_service = { + use tower::Service; + match make_service.call(addr).await { + Ok(svc) => svc, + Err(never) => match never {}, + } + }; + let service = TowerToHyperService::new(tower_service); let conn = builder .serve_connection_with_upgrades(TokioIo::new(io), service) .into_owned(); @@ -682,21 +702,21 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let peeked_data = buf[..n].to_vec(); - // Detection logic - check magic headers first + // Detection logic - check magic headers first. + // Dispatch via `dispatch_or_drop` (try_send) so a + // saturated protocol worker doesn't back-pressure the + // accept loop. // Check for Sync Protocol: "solidb-sync-v1" if &peeked_data == b"solidb-sync-v1" { // For sync traffic, pass the raw stream - the magic header has been consumed // and verified, so we don't need to put it back in a PeekedStream - if sync_tx.send((Box::new(stream), addr.to_string())).await.is_err() { - tracing::error!("Sync worker channel closed"); - } + let sync_stream: solidb::sync::transport::SyncStream = Box::new(stream); + dispatch_or_drop(&sync_tx, (sync_stream, addr.to_string()), "sync", &addr); } // Check for Native Driver Protocol: "solidb-drv-v1\0" else if &peeked_data == b"solidb-drv-v1\0" { // For driver traffic, pass the raw stream to the driver handler - if driver_tx.send((stream, addr.to_string())).await.is_err() { - tracing::error!("Driver handler channel closed"); - } + dispatch_or_drop(&driver_tx, (stream, addr.to_string()), "driver", &addr); } // Check for Cluster JSON Messages else if peeked_data.first() == Some(&b'{') { @@ -717,9 +737,7 @@ async fn async_main(args: Args) -> anyhow::Result<()> { } else { // HTTP traffic - need peeked bytes for HTTP parsing let peeked_stream = PeekedStream::new(stream, peeked_data.clone()); - if http_tx.send((peeked_stream, addr)).await.is_err() { - tracing::error!("HTTP server channel closed"); - } + dispatch_or_drop(&http_tx, (peeked_stream, addr), "HTTP", &addr); } }); } @@ -742,14 +760,35 @@ async fn async_main(args: Args) -> anyhow::Result<()> { let listener = tokio::net::TcpListener::bind(&addr).await?; tracing::info!("Server listening on {}", addr); - axum::serve(listener, app) - .with_graceful_shutdown(shutdown_signal(shutdown_storage)) - .await?; + axum::serve( + listener, + // Provide `ConnectInfo` (login rate limiting keys on + // the real peer address). + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown_signal(shutdown_storage)) + .await?; } Ok(()) } +/// Hand a freshly accepted connection to a protocol worker without awaiting: +/// if the dispatch channel is full we drop the connection (the client will +/// see a closed connection and retry) rather than back-pressure the accept +/// loop behind an `await` on `send()`. +fn dispatch_or_drop(tx: &mpsc::Sender, item: T, what: &str, peer: &std::net::SocketAddr) { + match tx.try_send(item) { + Ok(()) => {} + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + tracing::warn!(peer = %peer, "{} dispatch channel full, dropping connection", what); + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + tracing::error!("{} dispatch channel closed", what); + } + } +} + async fn shutdown_signal(storage: Arc) { let ctrl_c = async { tokio::signal::ctrl_c() diff --git a/src/sdbql/executor/data_source.rs b/src/sdbql/executor/data_source.rs index 398e019c..9d7f10a3 100644 --- a/src/sdbql/executor/data_source.rs +++ b/src/sdbql/executor/data_source.rs @@ -99,12 +99,12 @@ impl<'a> QueryExecutor<'a> { } } - // Local scan - for non-sharded collections or when no coordinator - Ok(collection - .scan(limit) - .into_iter() - .map(|d| d.into_value()) - .collect()) + // Local scan - for non-sharded collections or when no coordinator. + // Use `scan_values` to skip the intermediate `Document` allocation and + // go straight from the stored bytes to a `serde_json::Value`. This + // is materially faster on large collections (no extra struct + // construction or re-merging of metadata). + Ok(collection.scan_values(limit)) } pub(super) fn scatter_gather_docs( &self, @@ -122,11 +122,7 @@ impl<'a> QueryExecutor<'a> { collection_name ); let collection = self.get_collection(collection_name)?; - return Ok(collection - .scan(limit) - .into_iter() - .map(|d| d.into_value()) - .collect()); + return Ok(collection.scan_values(limit)); }; let my_node_id = coordinator.my_node_id(); @@ -149,8 +145,7 @@ impl<'a> QueryExecutor<'a> { .get_database(db_name) .and_then(|db| db.get_collection(&physical_coll)) { - for doc in coll.scan(limit) { - let value = doc.into_value(); + for value in coll.scan_values(limit) { if let Some(key) = value.get("_key").and_then(|k| k.as_str()) { local_docs.push((key.to_string(), value)); } diff --git a/src/sdbql/prepared.rs b/src/sdbql/prepared.rs index 0b32d41d..64a92def 100644 --- a/src/sdbql/prepared.rs +++ b/src/sdbql/prepared.rs @@ -1,10 +1,10 @@ -use std::collections::HashMap; +use parking_lot::RwLock; +use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; -use crate::sdbql::ast::Query; +use crate::sdbql::ast::{BodyClause, Query}; use crate::sdbql::parser; #[derive(Clone)] @@ -15,8 +15,52 @@ pub struct PreparedStatement { pub use_count: u64, } +/// A cached statement plus the collections it references (computed once at +/// `put` time so removal can prune only the matching index sets). +struct CacheEntry { + stmt: Arc, + collections: HashSet, +} + +/// Entry map + per-collection invalidation index, guarded by a single lock. +#[derive(Default)] +struct Inner { + entries: HashMap, + /// collection_name -> set of statement hashes that reference it + by_collection: HashMap>, +} + +impl Inner { + /// Remove an entry and prune it from its collections' index sets, + /// dropping index sets that become empty. + fn remove_entry(&mut self, key: &str) { + if let Some(entry) = self.entries.remove(key) { + for coll in &entry.collections { + let emptied = match self.by_collection.get_mut(coll) { + Some(set) => { + set.remove(key); + set.is_empty() + } + None => false, + }; + if emptied { + self.by_collection.remove(coll); + } + } + } + } +} + +/// Prepared-statement cache. +/// +/// Concurrency: one `parking_lot::RwLock` guards both the entry map and the +/// per-collection invalidation index, so the read path doesn't suspend the +/// async runtime, the two structures can never diverge (a put racing an +/// invalidate is fully serialized), and `invalidate_collection` is +/// O(matches) instead of the previous O(total cache) scan that also walked +/// every `Arc`. pub struct PreparedStatementCache { - cache: RwLock>>, + inner: RwLock, max_entries: usize, ttl: Duration, hits: AtomicU64, @@ -26,7 +70,7 @@ pub struct PreparedStatementCache { impl PreparedStatementCache { pub fn new(max_entries: usize, ttl_secs: u64) -> Self { Self { - cache: RwLock::new(HashMap::new()), + inner: RwLock::new(Inner::default()), max_entries, ttl: Duration::from_secs(ttl_secs), hits: AtomicU64::new(0), @@ -34,21 +78,21 @@ impl PreparedStatementCache { } } - pub async fn get(&self, query_text: &str) -> Option> { + pub fn get(&self, query_text: &str) -> Option> { let hash = Self::hash_query(query_text); - let cache = self.cache.read().await; - - if let Some(stmt) = cache.get(&hash) { - if stmt.created_at.elapsed() < self.ttl { + let inner = self.inner.read(); + if let Some(entry) = inner.entries.get(&hash) { + if entry.stmt.created_at.elapsed() < self.ttl { self.hits.fetch_add(1, Ordering::Relaxed); - return Some(stmt.clone()); + return Some(entry.stmt.clone()); } } None } - pub async fn put(&self, query_text: &str, query: Query) -> Arc { + pub fn put(&self, query_text: &str, query: Query) -> Arc { let hash = Self::hash_query(query_text); + let collections = collect_collection_names(&query); let stmt = Arc::new(PreparedStatement { query: Arc::new(query), hash: hash.clone(), @@ -56,30 +100,50 @@ impl PreparedStatementCache { use_count: 0, }); - let mut cache = self.cache.write().await; + let mut inner = self.inner.write(); - if cache.len() >= self.max_entries { - let keys_to_remove: Vec = - cache.keys().take(self.max_entries / 2).cloned().collect(); - for key in keys_to_remove { - cache.remove(&key); + // Simple eviction: if over capacity, clear half the cache. Each + // removal prunes only the index sets the entry belongs to. + if inner.entries.len() >= self.max_entries { + let keys_to_remove: Vec = inner + .entries + .keys() + .take(self.max_entries / 2) + .cloned() + .collect(); + for key in &keys_to_remove { + inner.remove_entry(key); } } - cache.insert(hash.clone(), stmt.clone()); + for coll in &collections { + inner + .by_collection + .entry(coll.clone()) + .or_default() + .insert(hash.clone()); + } + inner.entries.insert( + hash, + CacheEntry { + stmt: stmt.clone(), + collections, + }, + ); + stmt } - pub async fn parse_if_needed( + pub fn parse_if_needed( &self, query_text: &str, ) -> crate::error::DbResult> { - if let Some(stmt) = self.get(query_text).await { + if let Some(stmt) = self.get(query_text) { Ok(stmt) } else { self.misses.fetch_add(1, Ordering::Relaxed); let query = parser::parse(query_text)?; - Ok(self.put(query_text, query).await) + Ok(self.put(query_text, query)) } } @@ -92,41 +156,38 @@ impl PreparedStatementCache { format!("{:016x}", hasher.finish()) } - pub async fn stats(&self) -> (u64, u64, usize) { + pub fn stats(&self) -> (u64, u64, usize) { let hits = self.hits.load(Ordering::Relaxed); let misses = self.misses.load(Ordering::Relaxed); - let size = self.cache.read().await.len(); + let size = self.inner.read().entries.len(); (hits, misses, size) } - pub async fn invalidate_all(&self) { - let mut cache = self.cache.write().await; - cache.clear(); + pub fn invalidate_all(&self) { + let mut inner = self.inner.write(); + inner.entries.clear(); + inner.by_collection.clear(); self.hits.store(0, Ordering::Relaxed); self.misses.store(0, Ordering::Relaxed); } - pub async fn invalidate_collection(&self, collection_name: &str) { - let mut cache = self.cache.write().await; - let all_keys: Vec = cache.keys().cloned().collect(); - let keys_to_remove: Vec = all_keys - .into_iter() - .filter(|k| { - if let Some(stmt) = cache.get(k) { - stmt.query.body_clauses.iter().any(|clause| { - matches!(clause, crate::sdbql::ast::BodyClause::For(for_clause) - if for_clause.collection == collection_name) - }) - } else { - false - } - }) - .collect(); - - for key in keys_to_remove { - cache.remove(&key); + pub fn invalidate_collection(&self, collection_name: &str) { + let mut inner = self.inner.write(); + let Some(to_remove) = inner.by_collection.remove(collection_name) else { + return; + }; + for key in &to_remove { + // Also prunes cross-references from other collections' sets. + inner.remove_entry(key); } } + + /// Number of collections currently tracked by the invalidation index + /// (test/observability helper). + #[cfg(test)] + fn index_len(&self) -> usize { + self.inner.read().by_collection.len() + } } impl Default for PreparedStatementCache { @@ -135,6 +196,19 @@ impl Default for PreparedStatementCache { } } +/// Extract the collection names referenced by a query's body clauses. +/// Used to populate the per-collection invalidation index at `put` time, +/// so `invalidate_collection` doesn't have to re-walk every `Arc`. +fn collect_collection_names(query: &Query) -> HashSet { + let mut out = HashSet::new(); + for clause in &query.body_clauses { + if let BodyClause::For(for_clause) = clause { + out.insert(for_clause.collection.clone()); + } + } + out +} + use std::sync::OnceLock; static PREPARED_STATEMENT_CACHE: OnceLock = OnceLock::new(); @@ -146,39 +220,80 @@ pub fn get_prepared_statement_cache() -> &'static PreparedStatementCache { mod tests { use super::*; - #[tokio::test] - async fn test_cache_basic() { + #[test] + fn test_cache_basic() { let cache = PreparedStatementCache::default(); let query = "FOR doc IN users RETURN doc"; - let stmt1 = cache.parse_if_needed(query).await.unwrap(); - let stmt2 = cache.parse_if_needed(query).await.unwrap(); + let stmt1 = cache.parse_if_needed(query).unwrap(); + let stmt2 = cache.parse_if_needed(query).unwrap(); assert_eq!(stmt1.hash, stmt2.hash); - let (hits, misses, _) = cache.stats().await; + let (hits, misses, _) = cache.stats(); assert_eq!(hits, 1); assert_eq!(misses, 1); } - #[tokio::test] - async fn test_cache_miss() { + #[test] + fn test_cache_miss() { let cache = PreparedStatementCache::default(); let stmt1 = cache .parse_if_needed("FOR doc IN users RETURN doc") - .await .unwrap(); let stmt2 = cache .parse_if_needed("FOR doc IN orders RETURN doc") - .await .unwrap(); assert_ne!(stmt1.hash, stmt2.hash); - let (hits, misses, _) = cache.stats().await; + let (hits, misses, _) = cache.stats(); assert_eq!(hits, 0); assert_eq!(misses, 2); } + + #[test] + fn test_invalidate_collection_uses_index() { + let cache = PreparedStatementCache::default(); + let _ = cache + .parse_if_needed("FOR doc IN users RETURN doc") + .unwrap(); + let _ = cache + .parse_if_needed("FOR doc IN orders RETURN doc") + .unwrap(); + let (_, _, size_before) = cache.stats(); + assert_eq!(size_before, 2); + + cache.invalidate_collection("users"); + let (_, _, size_after) = cache.stats(); + assert_eq!(size_after, 1); + + // The users index entry is gone entirely (no empty-set leak). + assert_eq!(cache.index_len(), 1); + + // The remaining statement (orders) should still be there. + let stmt = cache + .parse_if_needed("FOR doc IN orders RETURN doc") + .unwrap(); + // It's a cache hit on the still-present statement. + let (hits, _, _) = cache.stats(); + assert!(hits >= 1); + assert_eq!(stmt.query.body_clauses.len(), 1); + } + + #[test] + fn test_eviction_prunes_index() { + let cache = PreparedStatementCache::new(2, 300); + let _ = cache.parse_if_needed("FOR doc IN a RETURN doc").unwrap(); + let _ = cache.parse_if_needed("FOR doc IN b RETURN doc").unwrap(); + // Triggers eviction of half the entries. + let _ = cache.parse_if_needed("FOR doc IN c RETURN doc").unwrap(); + + let (_, _, size) = cache.stats(); + assert!(size <= 2); + // Index never tracks more collections than live entries reference. + assert!(cache.index_len() <= size); + } } diff --git a/src/server/auth.rs b/src/server/auth.rs index 0871667a..fedb473f 100644 --- a/src/server/auth.rs +++ b/src/server/auth.rs @@ -17,11 +17,15 @@ use axum::{ }; use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use dashmap::DashMap; +use lru::LruCache; use once_cell::sync::Lazy; +use parking_lot::Mutex; use rand_core::{OsRng, RngCore}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::sync::RwLock; +use std::num::NonZeroUsize; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use subtle::ConstantTimeEq; @@ -32,33 +36,63 @@ const RATE_LIMIT_WINDOW_SECS: u64 = 60; /// Basic auth cache TTL in seconds (avoid repeated Argon2 verification) const BASIC_AUTH_CACHE_TTL_SECS: u64 = 60; +/// Max number of distinct IPs we keep in the rate-limiter LRU. +const RATE_LIMITER_CAPACITY: usize = 50_000; + +/// Max number of Basic-auth entries we keep in the cache LRU. +const BASIC_AUTH_CACHE_CAPACITY: usize = 10_000; + /// Cache entry for Basic auth results struct AuthCacheEntry { claims: Claims, expires_at: Instant, } -/// In-memory rate limiter for login attempts -/// Tracks attempts per IP address with automatic cleanup -static LOGIN_RATE_LIMITER: Lazy>>> = - Lazy::new(|| RwLock::new(HashMap::new())); +/// In-memory rate limiter for login attempts. +/// Bounded LRU keyed by client IP, values are recent attempt timestamps. +/// Parking-lot `Mutex` is non-async and faster than `std::sync::RwLock` here +/// because every operation is a brief mutation (no concurrent readers). +static LOGIN_RATE_LIMITER: Lazy>>> = Lazy::new(|| { + Mutex::new(LruCache::new( + NonZeroUsize::new(RATE_LIMITER_CAPACITY).unwrap(), + )) +}); -/// Cache for Basic auth results to avoid repeated Argon2 verification +/// Cache for Basic auth results to avoid repeated Argon2 verification. +/// Bounded LRU so a hostile or buggy client cannot grow the cache without +/// limit (the previous `RwLock` only evicted opportunistically on +/// the write path). /// Key: base64(username:password_hash), Value: Claims + expiry -static BASIC_AUTH_CACHE: Lazy>> = - Lazy::new(|| RwLock::new(HashMap::new())); +static BASIC_AUTH_CACHE: Lazy>> = Lazy::new(|| { + Mutex::new(LruCache::new( + NonZeroUsize::new(BASIC_AUTH_CACHE_CAPACITY).unwrap(), + )) +}); + +/// Whether `X-Forwarded-For` / `X-Real-IP` may be trusted for client +/// identity (rate limiting). Off by default: those headers are +/// client-controlled, so trusting them lets a single machine rotate fake +/// IPs to dodge the login rate limit. Set `SOLIDB_TRUST_PROXY_HEADERS=1` +/// only when the server sits behind a proxy that overwrites them. +static TRUST_PROXY_HEADERS: Lazy = Lazy::new(|| { + std::env::var("SOLIDB_TRUST_PROXY_HEADERS") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) +}); + +pub fn trust_proxy_headers() -> bool { + *TRUST_PROXY_HEADERS +} /// Check if an IP is rate limited, return error if too many attempts pub fn check_rate_limit(ip: &str) -> Result<(), crate::error::DbError> { let now = Instant::now(); let window = std::time::Duration::from_secs(RATE_LIMIT_WINDOW_SECS); - let mut limiter = LOGIN_RATE_LIMITER - .write() - .unwrap_or_else(|e| e.into_inner()); + let mut limiter = LOGIN_RATE_LIMITER.lock(); - // Get or create entry for this IP - let attempts = limiter.entry(ip.to_string()).or_default(); + // Get or create entry for this IP (LRU bump on access). + let attempts = limiter.get_or_insert_mut(ip.to_string(), Vec::new); // Remove old attempts outside the window attempts.retain(|t| now.duration_since(*t) < window); @@ -79,28 +113,30 @@ pub fn check_rate_limit(ip: &str) -> Result<(), crate::error::DbError> { /// Get cached Basic auth claims if still valid fn get_cached_basic_auth(cache_key: &str) -> Option { - let cache = BASIC_AUTH_CACHE.read().ok()?; + let mut cache = BASIC_AUTH_CACHE.lock(); if let Some(entry) = cache.get(cache_key) { if Instant::now() < entry.expires_at { return Some(entry.claims.clone()); } + // Expired - drop it. + cache.pop(cache_key); } None } /// Cache a successful Basic auth result fn cache_basic_auth(cache_key: String, claims: Claims) { - if let Ok(mut cache) = BASIC_AUTH_CACHE.write() { - cache.retain(|_, v| Instant::now() < v.expires_at); - cache.insert( - cache_key, - AuthCacheEntry { - claims, - expires_at: Instant::now() - + std::time::Duration::from_secs(BASIC_AUTH_CACHE_TTL_SECS), - }, - ); - } + let mut cache = BASIC_AUTH_CACHE.lock(); + // Expired entries are dropped lazily when a lookup hits them (see + // `get_cached_basic_auth`); the LRU's bounded capacity caps the worst + // case at BASIC_AUTH_CACHE_CAPACITY entries. + cache.push( + cache_key, + AuthCacheEntry { + claims, + expires_at: Instant::now() + std::time::Duration::from_secs(BASIC_AUTH_CACHE_TTL_SECS), + }, + ); } const ADMIN_DB: &str = "_system"; @@ -368,6 +404,22 @@ impl AuthService { // Initialize RBAC system collections Self::init_rbac(storage, replication_log, should_skip_defaults)?; + // Pre-warm the in-memory API-key cache so `validate_api_key` is + // O(1) on the request hot path. Best-effort: any failure to scan + // just means the cache stays empty and validate_api_key falls back + // to a (slower) full scan on the first miss. + if let Err(e) = Self::load_api_key_cache(storage) { + tracing::warn!("Failed to pre-warm API key cache: {}", e); + } else { + let (hits, misses, len) = api_key_cache().stats(); + tracing::info!( + "API key cache pre-warmed: {} keys loaded (hits={}, misses={})", + len, + hits, + misses + ); + } + Ok(()) } @@ -706,46 +758,69 @@ impl AuthService { /// Validate an API key against stored keys pub fn validate_api_key(storage: &StorageEngine, raw_key: &str) -> Result { - let db = storage.get_database(ADMIN_DB)?; - let collection = db.get_collection(API_KEYS_COLL)?; - - // Hash the incoming key once let incoming_hash = Self::hash_api_key(raw_key); - // Iterate through all keys and compare hashes (O(n) but fast with SHA-256) + // Fast path: in-memory hash -> ApiKey lookup. O(1) instead of O(N) scan + // over the _api_keys collection on every authenticated request. + if let Some(api_key) = api_key_cache().lookup(&incoming_hash) { + return api_key_to_claims(&api_key); + } + + // Slow path: cache miss (cache not yet populated, or a key was just + // inserted on a peer node and hasn't replicated into the local cache + // yet). Fall back to a full scan, then backfill the cache for next time. + if !api_key_cache().is_loaded() { + // Lazy-load: best-effort populate; if storage doesn't have the + // collection yet (first boot), we just return the "invalid key" + // error from the scan. + let _ = Self::load_api_key_cache(storage); + } + + let db = match storage.get_database(ADMIN_DB) { + Ok(db) => db, + Err(_) => return Err(DbError::BadRequest("Invalid API key".to_string())), + }; + let collection = match db.get_collection(API_KEYS_COLL) { + Ok(c) => c, + Err(DbError::CollectionNotFound(_)) => { + return Err(DbError::BadRequest("Invalid API key".to_string())); + } + Err(_) => return Err(DbError::BadRequest("Invalid API key".to_string())), + }; + for doc in collection.scan(None) { let api_key: ApiKey = serde_json::from_value(doc.to_value()) .map_err(|_| DbError::InternalError("Corrupted API key data".to_string()))?; - // Constant-time comparison to prevent timing attacks - if constant_time_eq(incoming_hash.as_bytes(), api_key.key_hash.as_bytes()) { - // Check if API key has expired - if let Some(ref expires_at) = api_key.expires_at { - if let Ok(expiry) = chrono::DateTime::parse_from_rfc3339(expires_at) { - if expiry < chrono::Utc::now() { - return Err(DbError::BadRequest("API key has expired".to_string())); - } - } - } + // Backfill the cache so subsequent requests are O(1). + api_key_cache().insert(api_key.clone()); - // Return claims with roles and scoped_databases from the API key - return Ok(Claims { - sub: format!("api-key:{}", api_key.name), - exp: usize::MAX, // Claims never expire (API key expiry checked above) - livequery: None, - roles: if api_key.roles.is_empty() { - None - } else { - Some(api_key.roles) - }, - scoped_databases: api_key.scoped_databases, - }); + if constant_time_eq(incoming_hash.as_bytes(), api_key.key_hash.as_bytes()) { + return api_key_to_claims(&api_key); } } Err(DbError::BadRequest("Invalid API key".to_string())) } + /// Load the API key cache from storage. Called at startup and as a + /// backfill on first cache miss. + pub fn load_api_key_cache(storage: &StorageEngine) -> Result { + let mut loaded = 0; + if let Ok(db) = storage.get_database(ADMIN_DB) { + if let Ok(collection) = db.get_collection(API_KEYS_COLL) { + for doc in collection.scan(None) { + if let Ok(api_key) = serde_json::from_value::(doc.to_value()) { + api_key_cache().insert(api_key); + loaded += 1; + } + } + } + } + api_key_cache().mark_loaded(); + Ok(loaded) + } + /// Get roles for a user from _user_roles collection pub fn get_user_roles(storage: &StorageEngine, username: &str) -> Option> { let db = match storage.get_database(ADMIN_DB) { @@ -783,6 +858,130 @@ pub(crate) fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { a.ct_eq(b).unwrap_u8() == 1 } +/// Convert a stored `ApiKey` to a `Claims` value, applying expiration checks. +fn api_key_to_claims(api_key: &ApiKey) -> Result { + if let Some(ref expires_at) = api_key.expires_at { + if let Ok(expiry) = chrono::DateTime::parse_from_rfc3339(expires_at) { + if expiry < chrono::Utc::now() { + return Err(DbError::BadRequest("API key has expired".to_string())); + } + } + } + Ok(Claims { + sub: format!("api-key:{}", api_key.name), + exp: usize::MAX, + livequery: None, + roles: if api_key.roles.is_empty() { + None + } else { + Some(api_key.roles.clone()) + }, + scoped_databases: api_key.scoped_databases.clone(), + }) +} + +/// In-memory cache of API keys, keyed by SHA-256 hash. Populated at startup +/// (or lazily on first miss) so `validate_api_key` is O(1) instead of +/// scanning the whole `_api_keys` collection on every authenticated request. +pub struct ApiKeyCache { + /// key_hash -> ApiKey (Arc so hot-path lookups don't deep-clone the key) + by_hash: DashMap>, + /// id -> key_hash (for removal by id) + by_id: DashMap, + /// True once the cache has been loaded from storage at least once. + loaded: std::sync::atomic::AtomicBool, + /// Number of lookups, for observability. + hits: AtomicU64, + misses: AtomicU64, +} + +impl ApiKeyCache { + pub fn new() -> Self { + Self { + by_hash: DashMap::new(), + by_id: DashMap::new(), + loaded: std::sync::atomic::AtomicBool::new(false), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + } + } + + pub fn lookup(&self, key_hash: &str) -> Option> { + if let Some(v) = self.by_hash.get(key_hash) { + self.hits.fetch_add(1, Ordering::Relaxed); + Some(v.value().clone()) + } else { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + + pub fn insert(&self, api_key: ApiKey) { + let hash = api_key.key_hash.clone(); + let id = api_key.id.clone(); + self.by_hash + .insert(hash.clone(), std::sync::Arc::new(api_key)); + self.by_id.insert(id, hash); + } + + pub fn remove_by_id(&self, id: &str) { + if let Some((_, hash)) = self.by_id.remove(id) { + self.by_hash.remove(&hash); + } + } + + pub fn is_loaded(&self) -> bool { + self.loaded.load(Ordering::Acquire) + } + + pub fn mark_loaded(&self) { + self.loaded.store(true, Ordering::Release); + } + + pub fn clear(&self) { + self.by_hash.clear(); + self.by_id.clear(); + self.loaded.store(false, Ordering::Release); + } + + pub fn stats(&self) -> (u64, u64, usize) { + ( + self.hits.load(Ordering::Relaxed), + self.misses.load(Ordering::Relaxed), + self.by_hash.len(), + ) + } +} + +static API_KEY_CACHE: Lazy = Lazy::new(ApiKeyCache::new); + +impl Default for ApiKeyCache { + fn default() -> Self { + Self::new() + } +} + +pub fn api_key_cache() -> &'static ApiKeyCache { + &API_KEY_CACHE +} + +/// Keep the in-memory API key cache in sync when a replicated write to +/// `_system._api_keys` is applied locally (the sync worker bypasses the +/// HTTP handlers that normally maintain the cache). +pub fn note_replicated_api_key_upsert(doc: &serde_json::Value) { + match serde_json::from_value::(doc.clone()) { + Ok(api_key) => api_key_cache().insert(api_key), + Err(e) => tracing::warn!("Replicated _api_keys doc did not parse as ApiKey: {}", e), + } +} + +/// Evict a key from the in-memory cache when a replicated delete on +/// `_system._api_keys` is applied locally, so a key revoked on a peer node +/// stops authenticating here immediately. +pub fn note_replicated_api_key_delete(id: &str) { + api_key_cache().remove_by_id(id); +} + /// Axum Middleware for Authentication /// Supports both JWT (Authorization: Bearer ) and API keys (X-API-Key: ) pub async fn auth_middleware( diff --git a/src/server/handlers/auth.rs b/src/server/handlers/auth.rs index 4abb4e15..76731b1a 100644 --- a/src/server/handlers/auth.rs +++ b/src/server/handlers/auth.rs @@ -186,6 +186,11 @@ pub async fn create_api_key_handler( collection.insert(doc_value.clone())?; + // Update the in-memory API key cache so this new key can authenticate + // requests immediately (O(1) lookup) without waiting for a lazy + // full-collection scan on the first request. + crate::server::auth::api_key_cache().insert(api_key.clone()); + // Record write for replication if let Some(ref log) = state.replication_log { let entry = LogEntry { @@ -261,6 +266,10 @@ pub async fn delete_api_key_handler( collection.delete(&key_id)?; + // Drop the key from the in-memory cache so a deleted key can no longer + // authenticate (even if the raw key is still being presented). + crate::server::auth::api_key_cache().remove_by_id(&key_id); + // Record write for replication if let Some(ref log) = state.replication_log { let entry = LogEntry { @@ -284,22 +293,41 @@ pub async fn delete_api_key_handler( pub async fn login_handler( State(state): State, + // `Result` (not `Option`): tests build the router without connect info, + // and axum 0.8's `Option` extractor requires OptionalFromRequestParts, + // which ConnectInfo doesn't implement. + peer: Result< + axum::extract::ConnectInfo, + axum::extract::rejection::ExtensionRejection, + >, headers: HeaderMap, Json(req): Json, ) -> Result, DbError> { - // Extract client IP for rate limiting (check X-Forwarded-For first for proxied requests) - let client_ip = headers - .get("X-Forwarded-For") - .and_then(|h| h.to_str().ok()) - .and_then(|s| s.split(',').next()) - .map(|s| s.trim().to_string()) - .or_else(|| { - headers - .get("X-Real-IP") - .and_then(|h| h.to_str().ok()) - .map(|s| s.to_string()) - }) - .unwrap_or_else(|| "unknown".to_string()); + // Extract client IP for rate limiting. The socket peer address is + // authoritative; X-Forwarded-For / X-Real-IP are client-controlled and + // only consulted when SOLIDB_TRUST_PROXY_HEADERS is set (i.e. the + // server is behind a proxy that overwrites them). Trusting them by + // default would let one machine rotate fake IPs past the rate limit. + let socket_ip = peer + .ok() + .map(|axum::extract::ConnectInfo(addr)| addr.ip().to_string()); + let client_ip = if crate::server::auth::trust_proxy_headers() { + headers + .get("X-Forwarded-For") + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.split(',').next()) + .map(|s| s.trim().to_string()) + .or_else(|| { + headers + .get("X-Real-IP") + .and_then(|h| h.to_str().ok()) + .map(|s| s.to_string()) + }) + .or(socket_ip) + .unwrap_or_else(|| "unknown".to_string()) + } else { + socket_ip.unwrap_or_else(|| "unknown".to_string()) + }; // Check rate limit before processing crate::server::auth::check_rate_limit(&client_ip)?; diff --git a/src/server/handlers/blob_upload.rs b/src/server/handlers/blob_upload.rs index 44a6529e..62ec98f9 100644 --- a/src/server/handlers/blob_upload.rs +++ b/src/server/handlers/blob_upload.rs @@ -315,9 +315,7 @@ pub async fn complete_upload( state.upload_session_store.remove(&upload_id); // Invalidate cached listings so the new file is immediately visible. - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); Ok(Json(doc_value)) } diff --git a/src/server/handlers/blobs.rs b/src/server/handlers/blobs.rs index eeb294a0..9f59f371 100644 --- a/src/server/handlers/blobs.rs +++ b/src/server/handlers/blobs.rs @@ -136,9 +136,7 @@ pub async fn upload_blob( chunks_buffer, ) .await?; - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); return Ok(Json(doc)); } else { return Err(DbError::InternalError( @@ -223,9 +221,7 @@ pub async fn upload_blob( } // Invalidate cached listings so the new file is immediately visible. - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); Ok(Json(doc_value)) } diff --git a/src/server/handlers/collections/lifecycle.rs b/src/server/handlers/collections/lifecycle.rs index a0d413ed..3b170862 100644 --- a/src/server/handlers/collections/lifecycle.rs +++ b/src/server/handlers/collections/lifecycle.rs @@ -249,9 +249,7 @@ pub async fn delete_collection( database.delete_collection(&coll_name)?; // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // Record to replication log if let Some(ref log) = state.replication_log { diff --git a/src/server/handlers/collections/ops.rs b/src/server/handlers/collections/ops.rs index fa476bc3..abcb90e1 100644 --- a/src/server/handlers/collections/ops.rs +++ b/src/server/handlers/collections/ops.rs @@ -126,9 +126,7 @@ pub async fn truncate_collection( } // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // Record to replication log (only for non-direct requests to avoid duplicate logging) if !is_shard_direct { diff --git a/src/server/handlers/documents.rs b/src/server/handlers/documents.rs index c6bf5cbf..7edeca72 100644 --- a/src/server/handlers/documents.rs +++ b/src/server/handlers/documents.rs @@ -153,9 +153,7 @@ pub async fn insert_document( } // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // Fire triggers for the insert if !coll_name.starts_with('_') { @@ -223,9 +221,7 @@ pub async fn insert_documents_batch( }; // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // NOTE: Do NOT log to replication log for sharded data! // This endpoint is for internal shard operations (X-Shard-Direct). @@ -352,9 +348,7 @@ pub async fn insert_documents_replica( let insert_count = collection.upsert_batch(keyed_docs)?; // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); tracing::debug!( "REPLICA: Stored {} docs for {}/{}", @@ -691,9 +685,7 @@ pub async fn update_document( } // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // Fire triggers for the update (or insert if upsert) if !coll_name.starts_with('_') { @@ -767,9 +759,7 @@ pub async fn delete_document( collection.delete(&key)?; // Invalidate query cache for this collection - query_cache::get_query_cache() - .invalidate_collection(&coll_name) - .await; + query_cache::get_query_cache().invalidate_collection(&coll_name); // If this is a blob collection, trigger compaction to reclaim space from deleted chunks immediately if collection.get_type() == "blob" { diff --git a/src/server/handlers/query.rs b/src/server/handlers/query.rs index 8a18a5fc..3b477a50 100644 --- a/src/server/handlers/query.rs +++ b/src/server/handlers/query.rs @@ -198,9 +198,7 @@ pub async fn execute_query( // Execute transactional SDBQL query use crate::sdbql::ast::BodyClause; - let prepared = crate::sdbql::get_prepared_statement_cache() - .parse_if_needed(&req.query) - .await?; + let prepared = crate::sdbql::get_prepared_statement_cache().parse_if_needed(&req.query)?; let query = prepared.query.as_ref(); // Get transaction manager @@ -458,9 +456,7 @@ pub async fn execute_query( // Non-transactional execution (existing logic) // Use prepared statement cache to avoid re-parsing frequently executed queries - let prepared = crate::sdbql::get_prepared_statement_cache() - .parse_if_needed(&req.query) - .await?; + let prepared = crate::sdbql::get_prepared_statement_cache().parse_if_needed(&req.query)?; let query = prepared.query.as_ref(); // Check if query is cacheable (read-only with no mutations) @@ -488,7 +484,7 @@ pub async fn execute_query( // Check cache if query is read-only and caching is enabled if let Some(ref key) = cache_key { - let cached_result = query_cache::get_query_cache().get(key).await; + let cached_result = query_cache::get_query_cache().get(key); if let Some(result) = cached_result { tracing::debug!( "Query cache hit for: {}", @@ -633,19 +629,17 @@ pub async fn execute_query( // Cache read-only query results if let Some(key) = cache_key { let result_clone: Vec = query_result.results.clone(); - query_cache::get_query_cache().put(key, result_clone).await; + query_cache::get_query_cache().put(key, result_clone); } // Invalidate query cache when mutations occurred if mutations.has_mutations() { let collections = mutated_collections(query); if collections.is_empty() { - query_cache::get_query_cache().invalidate_all().await; + query_cache::get_query_cache().invalidate_all(); } else { for collection in collections { - query_cache::get_query_cache() - .invalidate_collection(collection) - .await; + query_cache::get_query_cache().invalidate_collection(collection); } } } @@ -688,9 +682,7 @@ pub async fn explain_query( headers: HeaderMap, Json(req): Json, ) -> Result, DbError> { - let prepared = crate::sdbql::get_prepared_statement_cache() - .parse_if_needed(&req.query) - .await?; + let prepared = crate::sdbql::get_prepared_statement_cache().parse_if_needed(&req.query)?; let query = (*prepared.query).clone(); let bind_vars = req.bind_vars.clone(); let storage = state.storage.clone(); diff --git a/src/server/routes.rs b/src/server/routes.rs index aefdddab..2ea6532b 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -1100,7 +1100,36 @@ pub fn create_router( .gzip(true) .zstd(true) .no_br() - .no_deflate(), + .no_deflate() + // NOTE: `compress_when` REPLACES the default predicate, so we + // explicitly AND our extra rules onto `DefaultPredicate` + // (which skips responses smaller than 32 bytes, gRPC, images, + // and SSE). The closure adds: don't bother gzip-ing 4xx/5xx + // replies (avoids the encode + Content-Length negotiation on + // every error), and skip already-compressed video/audio. + .compress_when({ + use tower_http::compression::predicate::{DefaultPredicate, Predicate}; + DefaultPredicate::new().and( + |status: axum::http::StatusCode, + _version: axum::http::Version, + headers: &axum::http::HeaderMap, + _extensions: &axum::http::Extensions| { + if status.is_client_error() || status.is_server_error() { + return false; + } + if let Some(ct) = headers.get(axum::http::header::CONTENT_TYPE) { + if let Ok(s) = ct.to_str() { + // Already-compressed formats not covered + // by DefaultPredicate. + if s.starts_with("video/") || s.starts_with("audio/") { + return false; + } + } + } + true + }, + ) + }), ) .layer({ use axum::http::header; diff --git a/src/server/transaction_handlers.rs b/src/server/transaction_handlers.rs index e30c0e39..e5efa06c 100644 --- a/src/server/transaction_handlers.rs +++ b/src/server/transaction_handlers.rs @@ -84,7 +84,7 @@ pub async fn commit_transaction( state.storage.commit_transaction(tx_id)?; // Invalidate query cache since committed data is now visible - query_cache::get_query_cache().invalidate_all().await; + query_cache::get_query_cache().invalidate_all(); Ok(Json(CommitTransactionResponse { id: tx_id.to_string(), diff --git a/src/storage/collection/blobs.rs b/src/storage/collection/blobs.rs index a970713f..efc54113 100644 --- a/src/storage/collection/blobs.rs +++ b/src/storage/collection/blobs.rs @@ -8,7 +8,7 @@ impl Collection { /// Store a blob chunk pub fn put_blob_chunk(&self, key: &str, chunk_index: u32, data: &[u8]) -> DbResult<()> { - if *self.collection_type.read().unwrap() != "blob" { + if self.collection_type.read().as_str() != "blob" { return Err(DbError::OperationNotSupported( "Blob operations only supported on blob collections".to_string(), )); @@ -93,7 +93,7 @@ impl Collection { chunk_index: u32, data: &[u8], ) -> DbResult<()> { - if *self.collection_type.read().unwrap() != "blob" { + if self.collection_type.read().as_str() != "blob" { return Err(DbError::OperationNotSupported( "Blob operations only supported on blob collections".to_string(), )); @@ -186,7 +186,7 @@ impl Collection { /// Get blob statistics for this collection pub fn blob_stats(&self) -> DbResult<(usize, u64)> { - if *self.collection_type.read().unwrap() != "blob" { + if self.collection_type.read().as_str() != "blob" { return Ok((0, 0)); } diff --git a/src/storage/collection/core.rs b/src/storage/collection/core.rs index 900303ae..9bcca6d2 100644 --- a/src/storage/collection/core.rs +++ b/src/storage/collection/core.rs @@ -2,9 +2,10 @@ use super::*; use crate::error::{DbError, DbResult}; use dashmap::DashMap; use hex; +use parking_lot::RwLock; use rust_rocksdb::DB; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; impl Collection { /// Create a new collection handle @@ -68,7 +69,7 @@ impl Collection { /// Get collection type pub fn get_type(&self) -> String { - self.collection_type.read().unwrap().clone() + self.collection_type.read().clone() } /// Set collection type (persists to disk) @@ -83,8 +84,7 @@ impl Collection { .map_err(|e| DbError::InternalError(format!("Failed to set collection type: {}", e)))?; // Update in-memory state - let mut mg = self.collection_type.write().unwrap(); - *mg = type_.to_string(); + *self.collection_type.write() = type_.to_string(); Ok(()) } diff --git a/src/storage/collection/crud.rs b/src/storage/collection/crud.rs index 4ae4d6f4..33798a4f 100644 --- a/src/storage/collection/crud.rs +++ b/src/storage/collection/crud.rs @@ -48,7 +48,7 @@ impl Collection { update_indexes: bool, ) -> DbResult { // Validate edge documents - if *self.collection_type.read().unwrap() == "edge" { + if self.collection_type.read().as_str() == "edge" { self.validate_edge_document(&data)?; } @@ -155,7 +155,7 @@ impl Collection { /// Update a document with atomic document + index writes pub fn update(&self, key: &str, data: Value) -> DbResult { - if *self.collection_type.read().unwrap() == "timeseries" { + if self.collection_type.read().as_str() == "timeseries" { return Err(DbError::OperationNotSupported( "Update operations are not allowed on timeseries collections".to_string(), )); @@ -170,7 +170,7 @@ impl Collection { let new_value = doc.to_value(); // Validate edge documents after update - if *self.collection_type.read().unwrap() == "edge" { + if self.collection_type.read().as_str() == "edge" { self.validate_edge_document(&new_value)?; } @@ -261,7 +261,7 @@ impl Collection { expected_rev: &str, data: Value, ) -> DbResult { - if *self.collection_type.read().unwrap() == "timeseries" { + if self.collection_type.read().as_str() == "timeseries" { return Err(DbError::OperationNotSupported( "Update operations are not allowed on timeseries collections".to_string(), )); @@ -419,7 +419,7 @@ impl Collection { } // If blob collection, delete chunks (separate from WriteBatch) - if *self.collection_type.read().unwrap() == "blob" { + if self.collection_type.read().as_str() == "blob" { self.delete_blob_data(key)?; } @@ -448,7 +448,7 @@ impl Collection { return Ok(0); } - if *self.collection_type.read().unwrap() == "timeseries" { + if self.collection_type.read().as_str() == "timeseries" { return Err(DbError::OperationNotSupported( "Upsert (update) operations are not allowed on timeseries collections. Use insert_batch instead.".to_string(), )); @@ -576,7 +576,7 @@ impl Collection { } // Handle blobs (separate from WriteBatch) - if *self.collection_type.read().unwrap() == "blob" { + if self.collection_type.read().as_str() == "blob" { let _ = self.delete_blob_data(key); } @@ -626,7 +626,7 @@ impl Collection { } // Check timeseries restriction - if *self.collection_type.read().unwrap() == "timeseries" { + if self.collection_type.read().as_str() == "timeseries" { return Err(DbError::OperationNotSupported( "Update operations are not allowed on timeseries collections".to_string(), )); @@ -654,7 +654,7 @@ impl Collection { let new_value = doc.to_value(); // Validate edge documents after update - if *self.collection_type.read().unwrap() == "edge" { + if self.collection_type.read().as_str() == "edge" { if let Err(e) = self.validate_edge_document(&new_value) { tracing::warn!("Failed to validate edge for {}: {}", key, e); continue; @@ -769,7 +769,7 @@ impl Collection { return Ok(Vec::new()); } - let is_edge = *self.collection_type.read().unwrap() == "edge"; + let is_edge = self.collection_type.read().as_str() == "edge"; let schema_validator = self.get_cached_schema_validator()?; let db = &self.db; diff --git a/src/storage/collection/mod.rs b/src/storage/collection/mod.rs index c81770c2..2f06aaf0 100644 --- a/src/storage/collection/mod.rs +++ b/src/storage/collection/mod.rs @@ -1,9 +1,10 @@ use dashmap::DashMap; +use parking_lot::RwLock; use rust_rocksdb::DB; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::sync::atomic::{AtomicBool, AtomicUsize}; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; pub use super::document::Document; pub use super::geo::{GeoIndex, GeoIndexStats}; diff --git a/src/storage/collection/schema.rs b/src/storage/collection/schema.rs index abcbf98e..1efbb4b8 100644 --- a/src/storage/collection/schema.rs +++ b/src/storage/collection/schema.rs @@ -20,13 +20,16 @@ impl Collection { if let Some(schema) = self.get_json_schema() { let schema_hash = Self::compute_schema_hash(&schema); - let mut cached_hash = self.schema_hash.write().unwrap(); - let mut cached_validator = self.schema_validator.write().unwrap(); - - if let Some(ref current_hash) = *cached_hash { - if *current_hash == schema_hash { - if let Some(ref validator) = *cached_validator { - return Ok(Some(validator.clone())); + // Fast path: read guards only (parking_lot's RwLock never + // poisons, so no .unwrap() is needed). + { + let cached_hash = self.schema_hash.read(); + let cached_validator = self.schema_validator.read(); + if let Some(ref current_hash) = *cached_hash { + if *current_hash == schema_hash { + if let Some(ref validator) = *cached_validator { + return Ok(Some(validator.clone())); + } } } } @@ -35,8 +38,16 @@ impl Collection { DbError::InvalidDocument(format!("Schema compilation error: {}", e)) })?; - *cached_hash = Some(schema_hash); - *cached_validator = Some(validator.clone()); + // Hold BOTH write guards while setting the pair so the hash and + // validator can never be observed (or written) mismatched when + // two threads race with different schema versions. Lock order is + // always hash -> validator. + { + let mut cached_hash = self.schema_hash.write(); + let mut cached_validator = self.schema_validator.write(); + *cached_hash = Some(schema_hash); + *cached_validator = Some(validator.clone()); + } Ok(Some(validator)) } else { @@ -44,10 +55,11 @@ impl Collection { } } - /// Invalidate schema cache (called when schema changes) + /// Invalidate schema cache (called when schema changes). + /// Same lock order as the setter (hash -> validator), both held together. fn invalidate_schema_cache(&self) { - let mut cached_hash = self.schema_hash.write().unwrap(); - let mut cached_validator = self.schema_validator.write().unwrap(); + let mut cached_hash = self.schema_hash.write(); + let mut cached_validator = self.schema_validator.write(); *cached_hash = None; *cached_validator = None; } diff --git a/src/storage/collection/txn.rs b/src/storage/collection/txn.rs index 24187fc6..e3d2d641 100644 --- a/src/storage/collection/txn.rs +++ b/src/storage/collection/txn.rs @@ -40,7 +40,7 @@ impl Collection { lock_manager: &Arc, mut data: Value, ) -> DbResult { - if *self.collection_type.read().unwrap() == "edge" { + if self.collection_type.read().as_str() == "edge" { self.validate_edge_document(&data)?; } @@ -84,7 +84,7 @@ impl Collection { key: &str, data: Value, ) -> DbResult { - if *self.collection_type.read().unwrap() == "timeseries" { + if self.collection_type.read().as_str() == "timeseries" { return Err(DbError::OperationNotSupported( "Update operations are not allowed on timeseries collections".to_string(), )); @@ -98,7 +98,7 @@ impl Collection { doc.update(data); - if *self.collection_type.read().unwrap() == "edge" { + if self.collection_type.read().as_str() == "edge" { self.validate_edge_document(&doc.to_value())?; } diff --git a/src/storage/query_cache.rs b/src/storage/query_cache.rs index 052a432f..44850491 100644 --- a/src/storage/query_cache.rs +++ b/src/storage/query_cache.rs @@ -1,11 +1,20 @@ //! Query result cache for caching frequently executed query results. //! //! This module provides caching for query results to improve read performance. +//! +//! Concurrency model: +//! - One `parking_lot::RwLock` guards both the entry map and the +//! per-collection invalidation index, so they can never diverge (a put +//! racing an invalidate is fully serialized) and reads stay cheap and +//! non-async on the request hot path (no `.await` suspension). +//! - Each entry remembers the collections it references, so eviction and +//! invalidation only touch the index sets they belong to — O(matching +//! entries), not O(total cache). -use std::collections::HashMap; +use parking_lot::RwLock; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; /// Configuration for query caching #[derive(Debug, Clone)] @@ -29,11 +38,43 @@ impl Default for QueryCacheConfig { pub struct CachedQueryResult { pub result: Arc>, pub cached_at: Instant, + /// Collections this entry references (parsed from the cache key at + /// `put` time) so removal can prune only the matching index sets. + collections: Vec, +} + +/// Entry map + per-collection invalidation index, guarded by a single lock. +#[derive(Default)] +struct Inner { + entries: HashMap, + /// collection_name -> set of cache keys referencing that collection + by_collection: HashMap>, +} + +impl Inner { + /// Remove an entry and prune it from its collections' index sets, + /// dropping index sets that become empty. + fn remove_entry(&mut self, key: &str) { + if let Some(entry) = self.entries.remove(key) { + for coll in &entry.collections { + let emptied = match self.by_collection.get_mut(coll) { + Some(set) => { + set.remove(key); + set.is_empty() + } + None => false, + }; + if emptied { + self.by_collection.remove(coll); + } + } + } + } } -/// Query cache with TTL support +/// Query cache with TTL support and per-collection invalidation index. pub struct QueryCache { - cache: RwLock>, + inner: RwLock, max_entries: usize, ttl: Duration, } @@ -41,7 +82,7 @@ pub struct QueryCache { impl QueryCache { pub fn new(max_entries: usize, ttl_secs: u64) -> Self { Self { - cache: RwLock::new(HashMap::new()), + inner: RwLock::new(Inner::default()), max_entries, ttl: Duration::from_secs(ttl_secs), } @@ -51,11 +92,11 @@ impl QueryCache { Self::new(config.max_entries, config.ttl_secs) } - /// Get a cached query result - pub async fn get(&self, query_hash: &str) -> Option>> { - let cache = self.cache.read().await; - if let Some(cached) = cache.get(query_hash) { - // Check if expired + /// Get a cached query result. Synchronous and non-blocking; safe to call + /// from request hot paths. + pub fn get(&self, query_hash: &str) -> Option>> { + let inner = self.inner.read(); + if let Some(cached) = inner.entries.get(query_hash) { if cached.cached_at.elapsed() < self.ttl { return Some(cached.result.clone()); } @@ -63,56 +104,79 @@ impl QueryCache { None } - /// Store a query result in cache - pub async fn put(&self, query_hash: String, result: Vec) { - let mut cache = self.cache.write().await; + /// Store a query result in cache. Parses the cache key to maintain the + /// per-collection invalidation index. + pub fn put(&self, query_hash: String, result: Vec) { + let collections = extract_collections_from_key(&query_hash); - // Simple eviction: if over capacity, clear half the cache - if cache.len() >= self.max_entries { - let keys_to_remove: Vec = - cache.keys().take(self.max_entries / 2).cloned().collect(); - for key in keys_to_remove { - cache.remove(&key); + let mut inner = self.inner.write(); + + // Simple eviction: if over capacity, clear half the cache. Each + // removal prunes only the index sets the entry belongs to. + if inner.entries.len() >= self.max_entries { + let keys_to_remove: Vec = inner + .entries + .keys() + .take(self.max_entries / 2) + .cloned() + .collect(); + for key in &keys_to_remove { + inner.remove_entry(key); } } - cache.insert( + for coll in &collections { + inner + .by_collection + .entry(coll.clone()) + .or_default() + .insert(query_hash.clone()); + } + inner.entries.insert( query_hash, CachedQueryResult { result: Arc::new(result), cached_at: Instant::now(), + collections, }, ); } - /// Invalidate all cached results (e.g., after a write operation) - pub async fn invalidate_all(&self) { - let mut cache = self.cache.write().await; - cache.clear(); + /// Invalidate all cached results. + pub fn invalidate_all(&self) { + let mut inner = self.inner.write(); + inner.entries.clear(); + inner.by_collection.clear(); } - /// Invalidate queries related to a specific collection - pub async fn invalidate_collection(&self, collection_name: &str) { - let mut cache = self.cache.write().await; - let keys_to_remove: Vec = cache - .keys() - .filter(|k| k.contains(collection_name)) - .cloned() - .collect(); - for key in keys_to_remove { - cache.remove(&key); + /// Invalidate queries related to a specific collection. O(matches) + /// instead of the previous O(total cache) scan. + pub fn invalidate_collection(&self, collection_name: &str) { + let mut inner = self.inner.write(); + let Some(keys_to_remove) = inner.by_collection.remove(collection_name) else { + return; + }; + for key in &keys_to_remove { + // Also prunes cross-references from other collections' sets. + inner.remove_entry(key); } } /// Get cache statistics - pub async fn stats(&self) -> QueryCacheStats { - let cache = self.cache.read().await; + pub fn stats(&self) -> QueryCacheStats { QueryCacheStats { - entries: cache.len(), + entries: self.inner.read().entries.len(), max_entries: self.max_entries, ttl_secs: self.ttl.as_secs(), } } + + /// Number of collections currently tracked by the invalidation index + /// (test/observability helper). + #[cfg(test)] + fn index_len(&self) -> usize { + self.inner.read().by_collection.len() + } } impl Default for QueryCache { @@ -139,6 +203,23 @@ pub fn get_query_cache() -> &'static QueryCache { QUERY_CACHE.get_or_init(QueryCache::default) } +/// Extract the collection names from a cache key built by `hash_query`. +/// Key format: `"db/coll1,coll2:hash"` (collections part may be empty). +fn extract_collections_from_key(key: &str) -> Vec { + let Some(slash_pos) = key.find('/') else { + return vec![]; + }; + let after_slash = &key[slash_pos + 1..]; + let Some(colon_pos) = after_slash.rfind(':') else { + return vec![]; + }; + let colls_str = &after_slash[..colon_pos]; + if colls_str.is_empty() { + return vec![]; + } + colls_str.split(',').map(|s| s.to_string()).collect() +} + /// Generate a cache key for a query that includes the database name and /// collection names for targeted invalidation. /// @@ -192,3 +273,104 @@ pub fn hash_query( ) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_put_and_get() { + let cache = QueryCache::new(10, 60); + cache.put("db/coll:abc".to_string(), vec![json!({"a": 1})]); + let got = cache.get("db/coll:abc"); + assert!(got.is_some()); + assert_eq!(got.unwrap().len(), 1); + } + + #[test] + fn test_get_missing() { + let cache = QueryCache::new(10, 60); + assert!(cache.get("db/coll:missing").is_none()); + } + + #[test] + fn test_invalidate_collection() { + let cache = QueryCache::new(10, 60); + cache.put("db/users:1".to_string(), vec![json!({"a": 1})]); + cache.put("db/orders:2".to_string(), vec![json!({"b": 2})]); + cache.put("db/users,orders:3".to_string(), vec![json!({"c": 3})]); + + cache.invalidate_collection("users"); + // Key referencing only users: gone + assert!(cache.get("db/users:1").is_none()); + // Key referencing only orders: still there + assert!(cache.get("db/orders:2").is_some()); + // Key referencing both: also gone (it referenced users) + assert!(cache.get("db/users,orders:3").is_none()); + } + + #[test] + fn test_invalidate_all() { + let cache = QueryCache::new(10, 60); + cache.put("db/users:1".to_string(), vec![json!({"a": 1})]); + cache.put("db/orders:2".to_string(), vec![json!({"b": 2})]); + cache.invalidate_all(); + assert!(cache.get("db/users:1").is_none()); + assert!(cache.get("db/orders:2").is_none()); + } + + #[test] + fn test_extract_collections() { + let mut got = extract_collections_from_key("db/users,orders:abc"); + got.sort(); + assert_eq!(got, vec!["orders".to_string(), "users".to_string()]); + assert_eq!( + extract_collections_from_key("db/:abc"), + Vec::::new() + ); + assert_eq!( + extract_collections_from_key("no_slash"), + Vec::::new() + ); + } + + #[test] + fn test_eviction() { + let cache = QueryCache::new(2, 60); + cache.put("db/a:1".to_string(), vec![json!({"a": 1})]); + cache.put("db/b:2".to_string(), vec![json!({"b": 2})]); + // This triggers eviction of half the entries. + cache.put("db/c:3".to_string(), vec![json!({"c": 3})]); + // We don't assert which key was evicted, only that the cache still works. + let stats = cache.stats(); + assert!(stats.entries <= 2); + // The invalidation index never holds more collections than live + // entries reference (evicted entries are pruned, empty sets dropped). + assert!(cache.index_len() <= stats.entries); + } + + #[test] + fn test_invalidate_prunes_index() { + let cache = QueryCache::new(10, 60); + cache.put("db/users:1".to_string(), vec![json!({"a": 1})]); + cache.put("db/users,orders:2".to_string(), vec![json!({"b": 2})]); + assert_eq!(cache.index_len(), 2); // users + orders + + cache.invalidate_collection("users"); + // Both entries referenced users, so the orders set emptied out and + // its index entry must be gone too (no leak of empty sets). + assert_eq!(cache.index_len(), 0); + assert_eq!(cache.stats().entries, 0); + } + + #[test] + fn test_hash_query_format() { + let key = hash_query( + "mydb", + "FOR doc IN users RETURN doc", + &std::collections::HashMap::new(), + ); + assert!(key.starts_with("mydb/users:")); + } +} diff --git a/src/sync/log.rs b/src/sync/log.rs index 38765340..c9ef9e4d 100644 --- a/src/sync/log.rs +++ b/src/sync/log.rs @@ -157,16 +157,20 @@ impl SyncLog { entry.node_id = self.node_id.clone(); } - // Write to RocksDB + // Use a single WriteBatch so the entry and the bumped sequence go + // through one `db.write` (one fsync) instead of two independent + // `put` calls (two fsyncs). This roughly halves the per-write + // disk cost on hot insert/update paths. let key = format!("sync_log:{:020}", *seq); let value = serde_json::to_vec(&entry).unwrap(); - if let Err(e) = self.db.put(key.as_bytes(), &value) { + let mut batch = WriteBatch::default(); + batch.put(key.as_bytes(), &value); + batch.put(SEQ_KEY, seq.to_be_bytes()); + + if let Err(e) = self.db.write(&batch) { tracing::error!("SyncLog: Failed to write entry {}: {}", *seq, e); } - if let Err(e) = self.db.put(SEQ_KEY, seq.to_be_bytes()) { - tracing::error!("SyncLog: Failed to write sequence {}: {}", *seq, e); - } // Update cache let mut cache = self.cache.write().unwrap(); diff --git a/src/sync/worker.rs b/src/sync/worker.rs index 4761c2bb..da048402 100644 --- a/src/sync/worker.rs +++ b/src/sync/worker.rs @@ -597,11 +597,25 @@ impl SyncWorker { } if !batch_data.is_empty() { + // Replicated writes to _api_keys must also update the + // in-memory auth cache (the HTTP handlers that normally + // maintain it are bypassed on this path). + let api_key_docs: Vec = + if database == "_system" && collection == "_api_keys" { + batch_data.iter().map(|(_, doc)| doc.clone()).collect() + } else { + Vec::new() + }; + if let Err(e) = coll.upsert_batch(batch_data) { warn!( "apply_batch: upsert failed for {}.{}: {}", database, collection, e ); + } else { + for doc in &api_key_docs { + crate::server::auth::note_replicated_api_key_upsert(doc); + } } } } @@ -624,6 +638,13 @@ impl SyncWorker { } if !keys_to_delete.is_empty() { + // Evict replicated _api_keys deletes from the in-memory + // auth cache so revoked keys stop authenticating here. + if database == "_system" && collection == "_api_keys" { + for key in &keys_to_delete { + crate::server::auth::note_replicated_api_key_delete(key); + } + } let _ = coll.delete_batch(keys_to_delete); } } @@ -689,6 +710,15 @@ impl SyncWorker { } if let Ok(coll) = db.get_collection(&entry.collection) { + // Replicated _api_keys writes must also refresh the + // in-memory auth cache (handlers are bypassed here). + let api_key_doc = + if entry.database == "_system" && entry.collection == "_api_keys" { + Some(doc.clone()) + } else { + None + }; + if let Err(e) = coll.upsert_batch(vec![(entry.document_key.clone(), doc)]) { @@ -696,6 +726,8 @@ impl SyncWorker { "apply_entry: upsert failed for {}: {}", entry.document_key, e ); + } else if let Some(ref doc) = api_key_doc { + crate::server::auth::note_replicated_api_key_upsert(doc); } } } @@ -707,6 +739,11 @@ impl SyncWorker { let _ = coll.delete(&entry.document_key); } } + // Evict replicated _api_keys deletes from the in-memory auth + // cache so revoked keys stop authenticating here. + if entry.database == "_system" && entry.collection == "_api_keys" { + crate::server::auth::note_replicated_api_key_delete(&entry.document_key); + } } Operation::CreateDatabase => { let _ = self.storage.create_database(entry.database.clone()); From 416348b0266a383e86cf76fe2d96136bcef79ef5 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Wed, 3 Jun 2026 16:36:57 +0200 Subject: [PATCH 05/12] feat(dashboard): relationships map of collections New /database/:db/relationships page rendering an ER-style graph of all collections: FK relations inferred from index fields (snake_case and camelCase naming heuristics) and graph relations sampled from edge collection _from/_to references. Data endpoint sends Cache-Control: private, max-age=30 so quick navigations don't redo the per-collection index walk. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../dashboard/relationships_controller.lua | 283 +++++++++ www/app/views/dashboard/_sidebar.etlua | 6 + www/app/views/dashboard/relationships.etlua | 589 ++++++++++++++++++ www/config/routes.lua | 4 + 4 files changed, 882 insertions(+) create mode 100644 www/app/controllers/dashboard/relationships_controller.lua create mode 100644 www/app/views/dashboard/relationships.etlua diff --git a/www/app/controllers/dashboard/relationships_controller.lua b/www/app/controllers/dashboard/relationships_controller.lua new file mode 100644 index 00000000..18a29e47 --- /dev/null +++ b/www/app/controllers/dashboard/relationships_controller.lua @@ -0,0 +1,283 @@ +-- Dashboard Relationships Controller +-- Builds an entity-relationship style map of all collections, inferring +-- relations between them from index keys (foreign-key style fields) and from +-- edge collection _from/_to references. +local DashboardBaseController = require("dashboard.base_controller") +local RelationshipsController = DashboardBaseController:extend() + +-- --------------------------------------------------------------------------- +-- Naming helpers used to match an indexed field back to a collection name. +-- --------------------------------------------------------------------------- + +local function depluralize(s) + if s:match("ies$") then + return (s:gsub("ies$", "y")) + elseif s:match("ses$") or s:match("xes$") or s:match("zes$") or s:match("ches$") or s:match("shes$") then + return (s:gsub("es$", "")) + elseif s:match("s$") and not s:match("ss$") then + return (s:gsub("s$", "")) + end + return s +end + +local function pluralize(s) + if s:match("y$") then + return (s:gsub("y$", "ies")) + elseif s:match("s$") or s:match("x$") or s:match("z$") or s:match("ch$") or s:match("sh$") then + return s .. "es" + end + return s .. "s" +end + +-- Strip a foreign-key style suffix off a field name and return the base noun, +-- or nil if the field doesn't look like a reference. Handles snake_case +-- (user_id, account_key) and camelCase (userId, accountRef). Nested paths use +-- the trailing segment (e.g. "owner.user_id" -> "user"). +local function fk_base(field) + if type(field) ~= "string" or field == "" then + return nil + end + + -- Use the last path segment for nested fields. + local seg = field:match("([^.]+)$") or field + + -- Skip document metadata fields (_id/_key/_rev/_from/_to handled elsewhere). + if seg:sub(1, 1) == "_" then + return nil + end + + local lower = seg:lower() + + -- snake_case suffixes + local snake_suffixes = { "_ids", "_id", "_keys", "_key", "_refs", "_ref", "_fk", "_uuid", "_guid" } + for _, suf in ipairs(snake_suffixes) do + if #lower > #suf and lower:sub(-#suf) == suf then + return lower:sub(1, #lower - #suf) + end + end + + -- camelCase / PascalCase suffixes (preserve original case to detect the boundary) + local camel_suffixes = { "Ids", "Id", "Keys", "Key", "Refs", "Ref", "UUID", "Uuid", "Guid" } + for _, suf in ipairs(camel_suffixes) do + if #seg > #suf and seg:sub(-#suf) == suf then + local base = seg:sub(1, #seg - #suf) + if base ~= "" then + return base:lower() + end + end + end + + return nil +end + +-- --------------------------------------------------------------------------- +-- Main page +-- --------------------------------------------------------------------------- +function RelationshipsController:index() + self.layout = "dashboard" + local db = self:get_db() + self:render("dashboard/relationships", { + title = "Relationships - " .. db, + db = db, + current_page = "relationships" + }) +end + +-- --------------------------------------------------------------------------- +-- JSON data endpoint: nodes (collections) + edges (inferred relations) +-- --------------------------------------------------------------------------- +function RelationshipsController:data() + local db = self:get_db() + local include_system = self.params.include_system == "true" + + -- 1. Load all collections ------------------------------------------------ + local collections = {} + local status, _, body = self:fetch_api("/_api/database/" .. db .. "/collection") + if status == 200 then + local ok, parsed = pcall(DecodeJson, body) + if ok and parsed then + collections = parsed.collections or parsed or {} + end + end + + -- Build a lookup table from normalized name forms -> actual collection name. + local coll_map = {} + local nodes = {} + local node_index = {} -- name -> position in nodes (for quick lookups) + local edge_collections = {} + + local function register_name(form, actual) + if form and form ~= "" and not coll_map[form] then + coll_map[form] = actual + end + end + + for _, c in ipairs(collections) do + local name = c.name + if name and (include_system or name:sub(1, 1) ~= "_") then + local lower = name:lower() + register_name(lower, name) + register_name(depluralize(lower), name) + register_name(pluralize(lower), name) + + local is_edge = c.type == "edge" or c.type == 3 + local node = { + id = name, + label = name, + type = is_edge and "edge" or "document", + count = c.count or c.document_count or 0, + indexCount = 0, + relationCount = 0 + } + nodes[#nodes + 1] = node + node_index[name] = node + if is_edge then + edge_collections[#edge_collections + 1] = name + end + end + end + + local function resolve(base) + if not base or base == "" then + return nil + end + return coll_map[base] or coll_map[depluralize(base)] or coll_map[pluralize(base)] + end + + -- 2. Inspect indexes of each collection to infer FK relations ------------ + local edges = {} + local seen_edges = {} + + local function add_edge(source, target, label, kind, extra) + -- Stable key so we don't draw duplicate relations. + local key = kind .. "|" .. source .. "|" .. target .. "|" .. (label or "") + if seen_edges[key] then + return + end + seen_edges[key] = true + local e = { + id = "e" .. (#edges + 1), + source = source, + target = target, + label = label, + kind = kind + } + if extra then + for k, v in pairs(extra) do + e[k] = v + end + end + edges[#edges + 1] = e + if node_index[source] then + node_index[source].relationCount = node_index[source].relationCount + 1 + end + end + + for _, node in ipairs(nodes) do + local name = node.id + local idx_status, _, idx_body = self:fetch_api("/_api/database/" .. db .. "/index/" .. name) + if idx_status == 200 then + local ok, parsed = pcall(DecodeJson, idx_body) + if ok and parsed then + local indexes = parsed.indexes or parsed or {} + for _, idx in ipairs(indexes) do + node.indexCount = node.indexCount + 1 + + -- Collect the fields covered by this index. + local fields = {} + if type(idx.fields) == "table" then + for _, f in ipairs(idx.fields) do + fields[#fields + 1] = f + end + end + if idx.field and idx.field ~= "" then + fields[#fields + 1] = idx.field + end + + local idx_type = idx.index_type or idx.type or "hash" + if type(idx_type) ~= "string" then + idx_type = "hash" + end + + for _, field in ipairs(fields) do + local base = fk_base(field) + local target = resolve(base) + -- Don't draw self-loops from a collection's own-name suffix + -- (e.g. "users" collection with a "user_id" pk-ish field) unless + -- the field truly points elsewhere; self references are kept. + if target and target ~= "" then + add_edge(name, target, field, "fk", { + index = idx.name, + unique = idx.unique == true, + indexType = idx_type + }) + end + end + end + end + end + end + + -- 3. Edge collections: sample _from/_to to connect vertex collections ---- + for _, edge_coll in ipairs(edge_collections) do + local query = string.format( + "FOR d IN %s FILTER d._from != null && d._to != null LIMIT 50 RETURN { f: d._from, t: d._to }", + edge_coll + ) + local q_status, _, q_body = self:fetch_api("/_api/database/" .. db .. "/cursor", { + method = "POST", + body = EncodeJson({ query = query }) + }) + if q_status == 200 then + local ok, parsed = pcall(DecodeJson, q_body) + if ok and parsed and parsed.result then + local from_seen = {} + local to_seen = {} + for _, row in ipairs(parsed.result) do + local from_coll = type(row.f) == "string" and row.f:match("^([^/]+)/") or nil + local to_coll = type(row.t) == "string" and row.t:match("^([^/]+)/") or nil + if from_coll and node_index[from_coll] and not from_seen[from_coll] then + from_seen[from_coll] = true + add_edge(from_coll, edge_coll, "_from", "graph") + end + if to_coll and node_index[to_coll] and not to_seen[to_coll] then + to_seen[to_coll] = true + add_edge(edge_coll, to_coll, "_to", "graph") + end + end + end + end + end + + -- 4. Summary stats ------------------------------------------------------- + local fk_count = 0 + local graph_count = 0 + for _, e in ipairs(edges) do + if e.kind == "graph" then + graph_count = graph_count + 1 + else + fk_count = fk_count + 1 + end + end + + -- Building this graph costs one API round-trip per collection (index + -- listing) plus one per edge collection (sample query). Redbean forks per + -- connection, so an in-process Lua memo would not survive across requests; + -- instead let the browser cache the response briefly so re-renders and + -- quick navigations don't redo the N+1 walk. + SetHeader("Cache-Control", "private, max-age=30") + + self:json({ + nodes = nodes, + edges = edges, + stats = { + collections = #nodes, + edgeCollections = #edge_collections, + relations = #edges, + fkRelations = fk_count, + graphRelations = graph_count + } + }) +end + +return RelationshipsController diff --git a/www/app/views/dashboard/_sidebar.etlua b/www/app/views/dashboard/_sidebar.etlua index d9c45f6f..521754bc 100644 --- a/www/app/views/dashboard/_sidebar.etlua +++ b/www/app/views/dashboard/_sidebar.etlua @@ -90,6 +90,12 @@ end Graph Explorer + + + Relationships + +

    Development

    diff --git a/www/app/views/dashboard/relationships.etlua b/www/app/views/dashboard/relationships.etlua new file mode 100644 index 00000000..b9f9be04 --- /dev/null +++ b/www/app/views/dashboard/relationships.etlua @@ -0,0 +1,589 @@ +<% local db = db %> + +
    +
    +

    + + Relationships +

    +

    + A map of every collection in <%= db %> and the relations + inferred from their index keys. Drag nodes to rearrange · scroll to zoom · drag the canvas to pan. +

    +
    + + +
    +
    + + – + collections +
    +
    + + – + relations +
    +
    +
    + + +
    +
    + + +
    + + + +
    + + +
    + + + +
    + + + + + +
    + + +
    + +
    + + +
    + + Mapping collections & index keys… +
    + + + + + +
    +
    + + Document collection +
    +
    + + Edge collection +
    +
    + + Index-key relation +
    +
    + + Edge _from/_to +
    +
    + + +
    +
    +
    +
    +
    Collection
    +

    +
    + +
    + +
    +
    +
    0
    +
    docs
    +
    +
    +
    0
    +
    indexes
    +
    +
    +
    0
    +
    refs
    +
    +
    + +
    +

    References out

    +
    +
    +
    +

    Referenced by

    +
    +
    + +
    + + Browse documents + + + Manage indexes + +
    +
    +
    +
    + + diff --git a/www/config/routes.lua b/www/config/routes.lua index 945dada7..31a03812 100644 --- a/www/config/routes.lua +++ b/www/config/routes.lua @@ -154,6 +154,10 @@ router.scope("/database/:db", { middleware = { "dashboard_auth" } }, function() router.get("/graph/modal/vertex", "dashboard/graph#modal_vertex") router.get("/graph/modal/edge", "dashboard/graph#modal_edge") + -- Relationships map (collections + inferred relations from index keys) + router.get("/relationships", "dashboard/relationships#index") + router.get("/relationships/data", "dashboard/relationships#data") + -- Columnar routes router.get("/columnar", "dashboard/collections#columnar") router.get("/columnar/table", "dashboard/collections#columnar_table") From 7aa803c04f89626e8e5393fe15c67f80b7e612ee Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Thu, 4 Jun 2026 17:29:08 +0200 Subject: [PATCH 06/12] perf(storage): switch to MultiThreaded RocksDB and share tuned CF options - Use DBWithThreadMode (storage::RocksDb): create_cf/drop_cf take &self with internal synchronization, removing the unsafe Arc::as_ptr-as-*mut-DB casts that raced lock-free cf_handle() reads (and each other across StorageEngine's and Database's separate cf_locks) - Apply shared tuned CF options everywhere: Database::create_collection used bare Options::default(), so most collections got no LZ4 compression or tuning - Attach an explicit block-based table factory with a process-wide shared 512MB block cache and bloom filters to every CF; previously the configured cache applied only to the default CF while each of the other CFs lazily grew its own private 32MB default cache with no bloom filter - list_collections: use the live in-memory cf_names() instead of DB::list_cf(), which re-read the MANIFEST from disk on every call (~20x faster at 1000 CFs: 678us -> 34us per call) - Add examples/cf_perf_probe.rs to track CF-scaling costs (the per-create OPTIONS rewrite remains O(total CF count), tracked in tasks/todo/slow-database-drop-per-cf-options-rewrite.md) Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/cf_perf_probe.rs | 87 +++++++++++++++ src/storage/collection/blobs.rs | 24 ++--- src/storage/collection/core.rs | 38 +++---- src/storage/collection/crud.rs | 116 ++++++++++---------- src/storage/collection/fulltext.rs | 30 +++--- src/storage/collection/geo.rs | 20 ++-- src/storage/collection/indexes.rs | 106 +++++++++---------- src/storage/collection/mod.rs | 2 +- src/storage/collection/schema.rs | 6 +- src/storage/collection/ttl.rs | 14 +-- src/storage/collection/txn.rs | 38 +++---- src/storage/collection/vector.rs | 16 +-- src/storage/columnar.rs | 163 +++++++++++++++-------------- src/storage/database.rs | 46 ++++---- src/storage/engine.rs | 123 +++++++++++----------- src/storage/mod.rs | 8 ++ src/sync/log.rs | 3 +- 17 files changed, 470 insertions(+), 370 deletions(-) create mode 100644 examples/cf_perf_probe.rs diff --git a/examples/cf_perf_probe.rs b/examples/cf_perf_probe.rs new file mode 100644 index 00000000..4819ccf0 --- /dev/null +++ b/examples/cf_perf_probe.rs @@ -0,0 +1,87 @@ +//! Temporary perf probe: list_collections (in-memory cf_names vs disk list_cf) +//! and per-CF create cost as CF count grows. +//! +//! Run: cargo run --release --example cf_perf_probe + +use solidb::storage::{RocksDb, StorageEngine}; +use std::time::Instant; + +fn main() { + let dir = std::env::temp_dir().join(format!("cf_probe_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let engine = StorageEngine::new(&dir).expect("open engine"); + engine.initialize().expect("init"); + let db = engine.get_database("_system").expect("get db"); + + // --- CF create scaling --- + println!("--- create_collection cost as CF count grows ---"); + for batch in 0..10 { + let start = Instant::now(); + for i in 0..100 { + db.create_collection(format!("coll_{}_{}", batch, i), None) + .expect("create"); + } + let total_cfs = (batch + 1) * 100; + println!( + "create #{:4}..{:4}: {:>8.1?} total, {:>7.2?}/collection", + batch * 100, + total_cfs, + start.elapsed(), + start.elapsed() / 100 + ); + } + + // --- list_collections: new (in-memory) vs old (disk MANIFEST read) --- + println!("\n--- list_collections with ~1000 CFs, 200 iterations ---"); + + let start = Instant::now(); + let mut n = 0; + for _ in 0..200 { + n = engine.list_collections().len(); + } + println!( + "new (cf_names, in-memory): {:>9.2?} total, {:>9.2?}/call ({} collections)", + start.elapsed(), + start.elapsed() / 200, + n + ); + + let start = Instant::now(); + let mut n = 0; + for _ in 0..200 { + n = RocksDb::list_cf(&rust_rocksdb::Options::default(), &dir) + .unwrap_or_default() + .len(); + } + println!( + "old (list_cf, disk read) : {:>9.2?} total, {:>9.2?}/call ({} CFs)", + start.elapsed(), + start.elapsed() / 200, + n + ); + + // --- point reads (bloom filter + shared cache apply to new SSTs) --- + println!("\n--- 10k point gets on a 10k-doc collection ---"); + let coll = db.get_or_create_collection("bench_docs").expect("coll"); + for i in 0..10_000 { + let doc = serde_json::json!({"_key": format!("k{}", i), "v": i}); + coll.insert(doc).expect("insert"); + } + engine.flush().expect("flush"); + + let start = Instant::now(); + for i in 0..10_000 { + let _ = coll.get(&format!("k{}", i)).expect("get"); + } + println!( + "10k gets: {:>9.2?} total, {:>9.2?}/get", + start.elapsed(), + start.elapsed() / 10_000 + ); + + drop(coll); + drop(db); + drop(engine); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/src/storage/collection/blobs.rs b/src/storage/collection/blobs.rs index efc54113..2d24fa67 100644 --- a/src/storage/collection/blobs.rs +++ b/src/storage/collection/blobs.rs @@ -22,9 +22,9 @@ impl Collection { let chunk_key = Self::blo_chunk_key(key, chunk_index as usize); // ... existence check ... - let exists = db.get_cf(cf, &chunk_key).ok().flatten().is_some(); + let exists = db.get_cf(&cf, &chunk_key).ok().flatten().is_some(); - db.put_cf(cf, chunk_key, data) + db.put_cf(&cf, chunk_key, data) .map_err(|e| DbError::InternalError(format!("Failed to store blob chunk: {}", e)))?; if !exists { @@ -43,7 +43,7 @@ impl Collection { .ok_or(DbError::CollectionNotFound(self.name.clone()))?; let chunk_key = Self::blo_chunk_key(key, chunk_index as usize); - match db.get_cf(cf, chunk_key) { + match db.get_cf(&cf, chunk_key) { Ok(Some(data)) => Ok(Some(data)), Ok(None) => Ok(None), Err(e) => Err(DbError::InternalError(format!( @@ -61,7 +61,7 @@ impl Collection { .expect("Column family should exist"); let prefix = format!("{}{}:", BLO_PREFIX, key); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let mut batch = WriteBatch::default(); let mut count = 0; @@ -70,7 +70,7 @@ impl Collection { if !k.starts_with(prefix.as_bytes()) { break; } - batch.delete_cf(cf, k); + batch.delete_cf(&cf, k); count += 1; } @@ -105,7 +105,7 @@ impl Collection { .expect("Column family should exist"); let key = format!("{}{}:{}", BLO_TMP_PREFIX, upload_id, chunk_index); - db.put_cf(cf, key.as_bytes(), data).map_err(|e| { + db.put_cf(&cf, key.as_bytes(), data).map_err(|e| { DbError::InternalError(format!("Failed to store temp blob chunk: {}", e)) })?; @@ -130,7 +130,7 @@ impl Collection { for i in 0..total_chunks { let tmp_key = format!("{}{}:{}", BLO_TMP_PREFIX, upload_id, i); let data = db - .get_cf(cf, tmp_key.as_bytes()) + .get_cf(&cf, tmp_key.as_bytes()) .map_err(|e| DbError::InternalError(format!("Failed to read temp chunk: {}", e)))? .ok_or_else(|| { DbError::InternalError(format!( @@ -140,8 +140,8 @@ impl Collection { })?; let perm_key = Self::blo_chunk_key(blob_key, i as usize); - batch.put_cf(cf, &perm_key, &data); - batch.delete_cf(cf, tmp_key.as_bytes()); + batch.put_cf(&cf, &perm_key, &data); + batch.delete_cf(&cf, tmp_key.as_bytes()); } db.write(&batch).map_err(|e| { @@ -163,7 +163,7 @@ impl Collection { .expect("Column family should exist"); let prefix = format!("{}{}:", BLO_TMP_PREFIX, upload_id); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let mut batch = WriteBatch::default(); let mut count = 0; @@ -172,7 +172,7 @@ impl Collection { if !k.starts_with(prefix.as_bytes()) { break; } - batch.delete_cf(cf, k); + batch.delete_cf(&cf, k); count += 1; } @@ -199,7 +199,7 @@ impl Collection { let mut total_bytes = 0u64; let mut chunk_count = 0usize; - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for item in iter.flatten() { let (key, value) = item; if !key.starts_with(prefix) { diff --git a/src/storage/collection/core.rs b/src/storage/collection/core.rs index 9bcca6d2..6f2f5fb2 100644 --- a/src/storage/collection/core.rs +++ b/src/storage/collection/core.rs @@ -1,9 +1,9 @@ use super::*; use crate::error::{DbError, DbResult}; +use crate::storage::RocksDb as DB; use dashmap::DashMap; use hex; use parking_lot::RwLock; -use rust_rocksdb::DB; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; @@ -12,14 +12,14 @@ impl Collection { pub fn new(name: String, db: Arc) -> Self { // Load cached count from disk, or calculate if not present let count = if let Some(cf) = db.cf_handle(&name) { - match db.get_cf(cf, STATS_COUNT_KEY.as_bytes()) { + match db.get_cf(&cf, STATS_COUNT_KEY.as_bytes()) { Ok(Some(bytes)) => String::from_utf8_lossy(&bytes) .parse::() .unwrap_or(0), _ => { // No cached count - calculate from documents let prefix = DOC_PREFIX.as_bytes(); - db.prefix_iterator_cf(cf, prefix) + db.prefix_iterator_cf(&cf, prefix) .take_while(|r| r.as_ref().is_ok_and(|(k, _)| k.starts_with(prefix))) .count() } @@ -31,7 +31,7 @@ impl Collection { // Determine initial chunk count (only relevant if it's a blob collection) let chunk_count = if let Some(cf) = db.cf_handle(&name) { let prefix = BLO_PREFIX.as_bytes(); - db.prefix_iterator_cf(cf, prefix) + db.prefix_iterator_cf(&cf, prefix) .take_while(|r| r.as_ref().is_ok_and(|(k, _)| k.starts_with(prefix))) .count() } else { @@ -42,7 +42,7 @@ impl Collection { // Load collection type let collection_type = if let Some(cf) = db.cf_handle(&name) { - match db.get_cf(cf, COLLECTION_TYPE_KEY.as_bytes()) { + match db.get_cf(&cf, COLLECTION_TYPE_KEY.as_bytes()) { Ok(Some(bytes)) => String::from_utf8_lossy(&bytes).to_string(), _ => "document".to_string(), } @@ -80,7 +80,7 @@ impl Collection { .expect("Column family should exist"); self.db - .put_cf(cf, COLLECTION_TYPE_KEY.as_bytes(), type_.as_bytes()) + .put_cf(&cf, COLLECTION_TYPE_KEY.as_bytes(), type_.as_bytes()) .map_err(|e| DbError::InternalError(format!("Failed to set collection type: {}", e)))?; // Update in-memory state @@ -94,9 +94,11 @@ impl Collection { if self.count_dirty.swap(false, Ordering::Relaxed) { let count = self.doc_count.load(Ordering::Relaxed); if let Some(cf) = self.db.cf_handle(&self.name) { - let _ = - self.db - .put_cf(cf, STATS_COUNT_KEY.as_bytes(), count.to_string().as_bytes()); + let _ = self.db.put_cf( + &cf, + STATS_COUNT_KEY.as_bytes(), + count.to_string().as_bytes(), + ); } // Update last flush time let now = std::time::SystemTime::now() @@ -129,7 +131,7 @@ impl Collection { /// Compact the collection to remove tombstones and reclaim space pub fn compact(&self) { if let Some(cf) = self.db.cf_handle(&self.name) { - self.db.compact_range_cf(cf, None::<&[u8]>, None::<&[u8]>); + self.db.compact_range_cf(&cf, None::<&[u8]>, None::<&[u8]>); } } @@ -162,7 +164,7 @@ impl Collection { // Get SST files size let sst_files_size = self .db - .property_int_value_cf(cf, "rocksdb.total-sst-files-size") + .property_int_value_cf(&cf, "rocksdb.total-sst-files-size") .ok() .flatten() .unwrap_or(0); @@ -170,7 +172,7 @@ impl Collection { // Get estimated live data size let live_data_size = self .db - .property_int_value_cf(cf, "rocksdb.estimate-live-data-size") + .property_int_value_cf(&cf, "rocksdb.estimate-live-data-size") .ok() .flatten() .unwrap_or(0); @@ -180,7 +182,7 @@ impl Collection { for i in 0..7 { num_sst_files += self .db - .property_int_value_cf(cf, &format!("rocksdb.num-files-at-level{}", i)) + .property_int_value_cf(&cf, &format!("rocksdb.num-files-at-level{}", i)) .ok() .flatten() .unwrap_or(0); @@ -189,7 +191,7 @@ impl Collection { // Get memtable size let memtable_size = self .db - .property_int_value_cf(cf, "rocksdb.cur-size-all-mem-tables") + .property_int_value_cf(&cf, "rocksdb.cur-size-all-mem-tables") .ok() .flatten() .unwrap_or(0); @@ -216,7 +218,7 @@ impl Collection { let config_bytes = serde_json::to_vec(config)?; self.db - .put_cf(cf, SHARD_CONFIG_KEY.as_bytes(), &config_bytes) + .put_cf(&cf, SHARD_CONFIG_KEY.as_bytes(), &config_bytes) .map_err(|e| DbError::InternalError(format!("Failed to store shard config: {}", e)))?; tracing::info!( @@ -233,7 +235,7 @@ impl Collection { let cf = self.db.cf_handle(&self.name)?; self.db - .get_cf(cf, SHARD_CONFIG_KEY.as_bytes()) + .get_cf(&cf, SHARD_CONFIG_KEY.as_bytes()) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -251,7 +253,7 @@ impl Collection { let table_bytes = serde_json::to_vec(table)?; self.db - .put_cf(cf, SHARD_TABLE_KEY.as_bytes(), &table_bytes) + .put_cf(&cf, SHARD_TABLE_KEY.as_bytes(), &table_bytes) .map_err(|e| DbError::InternalError(format!("Failed to store shard table: {}", e)))?; Ok(()) @@ -262,7 +264,7 @@ impl Collection { let cf = self.db.cf_handle(&self.name)?; self.db - .get_cf(cf, SHARD_TABLE_KEY.as_bytes()) + .get_cf(&cf, SHARD_TABLE_KEY.as_bytes()) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) diff --git a/src/storage/collection/crud.rs b/src/storage/collection/crud.rs index 33798a4f..c1af6afe 100644 --- a/src/storage/collection/crud.rs +++ b/src/storage/collection/crud.rs @@ -18,7 +18,7 @@ impl Collection { .expect("Column family should exist"); let bytes = db - .get_cf(cf, Self::doc_key(key)) + .get_cf(&cf, Self::doc_key(key)) .map_err(|e| DbError::InternalError(format!("Failed to get document: {}", e)))? .ok_or_else(|| DbError::DocumentNotFound(key.to_string()))?; @@ -94,7 +94,7 @@ impl Collection { let mut batch = WriteBatch::default(); // Add document to batch - batch.put_cf(cf, Self::doc_key(&key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(&key), &doc_bytes); // Add index entries to batch (if enabled) if update_indexes { @@ -102,22 +102,22 @@ impl Collection { let (regular_entries, geo_entries) = self.compute_index_entries_for_insert(&key, &doc_value)?; for (entry_key, entry_value) in regular_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add fulltext entries let fulltext_entries = self.compute_fulltext_entries_for_insert(&key, &doc_value); for (entry_key, entry_value) in fulltext_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add TTL expiry entries let ttl_expiry_entries = self.compute_ttl_expiry_entries_for_insert(&key, &doc_value); for (entry_key, _entry_value) in ttl_expiry_entries { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } } @@ -192,7 +192,7 @@ impl Collection { let mut batch = WriteBatch::default(); // Update document in batch - batch.put_cf(cf, Self::doc_key(key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(key), &doc_bytes); // Compute and apply index updates atomically let (entries_to_add, keys_to_remove, geo_entries_to_add, geo_keys_to_remove) = @@ -200,39 +200,39 @@ impl Collection { // Remove old index entries for key_to_remove in keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for geo_key in geo_keys_to_remove { - batch.delete_cf(cf, geo_key); + batch.delete_cf(&cf, geo_key); } // Add new index entries for (entry_key, entry_value) in entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply fulltext updates let fulltext_keys_to_remove = self.compute_fulltext_entries_for_delete(key, &old_value); for key_to_remove in fulltext_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } let fulltext_entries_to_add = self.compute_fulltext_entries_for_insert(key, &new_value); for (entry_key, entry_value) in fulltext_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply TTL expiry updates let (ttl_entries_to_add, ttl_keys_to_remove) = self.compute_ttl_expiry_entries_for_update(key, &old_value, &new_value); for key_to_remove in ttl_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for (entry_key, _entry_value) in ttl_entries_to_add { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } // Atomic write: document + all index updates together @@ -296,7 +296,7 @@ impl Collection { let mut batch = WriteBatch::default(); // Update document in batch - batch.put_cf(cf, Self::doc_key(key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(key), &doc_bytes); // Compute and apply index updates atomically let (entries_to_add, keys_to_remove, geo_entries_to_add, geo_keys_to_remove) = @@ -304,39 +304,39 @@ impl Collection { // Remove old index entries for key_to_remove in keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for geo_key in geo_keys_to_remove { - batch.delete_cf(cf, geo_key); + batch.delete_cf(&cf, geo_key); } // Add new index entries for (entry_key, entry_value) in entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply fulltext updates let fulltext_keys_to_remove = self.compute_fulltext_entries_for_delete(key, &old_value); for key_to_remove in fulltext_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } let fulltext_entries_to_add = self.compute_fulltext_entries_for_insert(key, &new_value); for (entry_key, entry_value) in fulltext_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply TTL expiry updates let (ttl_entries_to_add, ttl_keys_to_remove) = self.compute_ttl_expiry_entries_for_update(key, &old_value, &new_value); for key_to_remove in ttl_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for (entry_key, _entry_value) in ttl_entries_to_add { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } // Atomic write: document + all index updates together @@ -382,27 +382,27 @@ impl Collection { let mut batch = WriteBatch::default(); // Delete document from batch - batch.delete_cf(cf, Self::doc_key(key)); + batch.delete_cf(&cf, Self::doc_key(key)); // Compute and remove regular + geo index entries let (regular_keys, geo_keys) = self.compute_index_entries_for_delete(key, &doc_value)?; for key_to_remove in regular_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for key_to_remove in geo_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and remove fulltext entries let fulltext_keys = self.compute_fulltext_entries_for_delete(key, &doc_value); for key_to_remove in fulltext_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and remove TTL expiry entries let ttl_keys = self.compute_ttl_expiry_entries_for_delete(key, &doc_value); for key_to_remove in ttl_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Atomic write: document deletion + index removals together @@ -471,10 +471,10 @@ impl Collection { } // Check if document exists to determine insert vs update - let exists = db.get_cf(cf, Self::doc_key(&key)).ok().flatten().is_some(); + let exists = db.get_cf(&cf, Self::doc_key(&key)).ok().flatten().is_some(); let doc = if exists { - if let Ok(Some(bytes)) = db.get_cf(cf, Self::doc_key(&key)) { + if let Ok(Some(bytes)) = db.get_cf(&cf, Self::doc_key(&key)) { if let Ok(mut existing) = deserialize_doc(&bytes) { existing.update(data); existing @@ -489,7 +489,7 @@ impl Collection { }; if let Ok(doc_bytes) = serialize_doc(&doc) { - batch.put_cf(cf, Self::doc_key(&key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(&key), &doc_bytes); upserted_docs.push((key.clone(), doc.to_value())); if !exists { insert_count += 1; @@ -540,12 +540,12 @@ impl Collection { // Prepare batch with document deletions and index removals atomically for key in &keys { // Get document first (needed for index cleanup and change events) - if let Ok(Some(bytes)) = db.get_cf(cf, Self::doc_key(key)) { + if let Ok(Some(bytes)) = db.get_cf(&cf, Self::doc_key(key)) { if let Ok(doc) = deserialize_doc(&bytes) { let doc_value = doc.to_value(); // Add document deletion to batch - batch.delete_cf(cf, Self::doc_key(key)); + batch.delete_cf(&cf, Self::doc_key(key)); // Compute and add index key removals to batch let (regular_keys, geo_keys) = @@ -557,22 +557,22 @@ impl Collection { } }; for key_to_remove in regular_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for key_to_remove in geo_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and add fulltext key removals to batch let fulltext_keys = self.compute_fulltext_entries_for_delete(key, &doc_value); for key_to_remove in fulltext_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and add TTL expiry key removals to batch let ttl_keys = self.compute_ttl_expiry_entries_for_delete(key, &doc_value); for key_to_remove in ttl_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Handle blobs (separate from WriteBatch) @@ -672,7 +672,7 @@ impl Collection { // Serialize document if let Ok(doc_bytes) = serialize_doc(&doc) { // Add document to batch - batch.put_cf(cf, Self::doc_key(key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(key), &doc_bytes); // Compute and add index updates to batch atomically let (entries_to_add, keys_to_remove, geo_entries_to_add, geo_keys_to_remove) = @@ -690,41 +690,41 @@ impl Collection { // Remove old index entries for key_to_remove in keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for geo_key in geo_keys_to_remove { - batch.delete_cf(cf, geo_key); + batch.delete_cf(&cf, geo_key); } // Add new index entries for (entry_key, entry_value) in entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add fulltext updates to batch let fulltext_keys_to_remove = self.compute_fulltext_entries_for_delete(key, &old_value); for key_to_remove in fulltext_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } let fulltext_entries_to_add = self.compute_fulltext_entries_for_insert(key, &new_value); for (entry_key, entry_value) in fulltext_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply TTL expiry updates let (ttl_entries_to_add, ttl_keys_to_remove) = self.compute_ttl_expiry_entries_for_update(key, &old_value, &new_value); for key_to_remove in ttl_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for (entry_key, _entry_value) in ttl_entries_to_add { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } // Update vector indexes in-memory (separate from WriteBatch) @@ -821,7 +821,7 @@ impl Collection { // Check if document with this key already exists in the DB if db - .get_cf(cf, Self::doc_key(&key)) + .get_cf(&cf, Self::doc_key(&key)) .map_err(|e| { DbError::InternalError(format!("Failed to check existing key: {}", e)) })? @@ -842,28 +842,28 @@ impl Collection { let doc_bytes = serialize_doc(&doc)?; // Add document to batch - batch.put_cf(cf, Self::doc_key(&key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(&key), &doc_bytes); // Compute and add regular + geo index entries let (regular_entries, geo_entries) = self.compute_index_entries_for_insert(&key, &doc_value)?; for (entry_key, entry_value) in regular_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add fulltext entries let fulltext_entries = self.compute_fulltext_entries_for_insert(&key, &doc_value); for (entry_key, entry_value) in fulltext_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add TTL expiry entries let ttl_expiry_entries = self.compute_ttl_expiry_entries_for_insert(&key, &doc_value); for (entry_key, _entry_value) in ttl_expiry_entries { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } doc_values.push((key, doc_value)); @@ -916,7 +916,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = DOC_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); let iter = iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -968,7 +968,7 @@ impl Collection { } let iter = db.iterator_cf_opt( - cf, + &cf, read_opts, IteratorMode::From(prefix, Direction::Forward), ); @@ -1007,7 +1007,7 @@ impl Collection { if let Some(cf) = db.cf_handle(&self.name) { let prefix = DOC_PREFIX.as_bytes(); let count = db - .prefix_iterator_cf(cf, prefix) + .prefix_iterator_cf(&cf, prefix) .take_while(|r| r.as_ref().is_ok_and(|(k, _)| k.starts_with(prefix))) .count(); @@ -1033,7 +1033,7 @@ impl Collection { let prefix = DOC_PREFIX.as_bytes(); let actual_count = self .db - .prefix_iterator_cf(cf, prefix) + .prefix_iterator_cf(&cf, prefix) .take_while(|r| r.as_ref().is_ok_and(|(k, _)| k.starts_with(prefix))) .count(); @@ -1084,7 +1084,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = DOC_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); let mut keys_to_delete = Vec::new(); diff --git a/src/storage/collection/fulltext.rs b/src/storage/collection/fulltext.rs index e8837484..04265d84 100644 --- a/src/storage/collection/fulltext.rs +++ b/src/storage/collection/fulltext.rs @@ -17,7 +17,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = FT_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -35,7 +35,7 @@ impl Collection { pub(crate) fn get_fulltext_index(&self, name: &str) -> Option { let db = &self.db; let cf = db.cf_handle(&self.name)?; - db.get_cf(cf, Self::ft_meta_key(name)) + db.get_cf(&cf, Self::ft_meta_key(name)) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -78,7 +78,7 @@ impl Collection { let cf = db .cf_handle(&self.name) .expect("Column family should exist"); - db.put_cf(cf, Self::ft_meta_key(&name), &index_bytes) + db.put_cf(&cf, Self::ft_meta_key(&name), &index_bytes) .map_err(|e| { DbError::InternalError(format!("Failed to create fulltext index: {}", e)) })?; @@ -104,7 +104,7 @@ impl Collection { for term in &terms { if term.len() >= min_length { let term_key = Self::ft_term_key(&name, term, &doc.key); - batch.put_cf(cf, term_key, doc.key.as_bytes()); + batch.put_cf(&cf, term_key, doc.key.as_bytes()); } } @@ -112,7 +112,7 @@ impl Collection { let ngrams = generate_ngrams(text, NGRAM_SIZE); for ngram in &ngrams { let ngram_key = Self::ft_ngram_key(&name, ngram, &doc.key); - batch.put_cf(cf, ngram_key, doc.key.as_bytes()); + batch.put_cf(&cf, ngram_key, doc.key.as_bytes()); } count += 1; } @@ -151,7 +151,7 @@ impl Collection { .expect("Column family should exist"); // Delete metadata - db.delete_cf(cf, Self::ft_meta_key(name)) + db.delete_cf(&cf, Self::ft_meta_key(name)) .map_err(|e| DbError::InternalError(format!("Failed to drop fulltext index: {}", e)))?; let mut batch = WriteBatch::default(); @@ -159,11 +159,11 @@ impl Collection { // Delete ngrams let prefix = format!("{}{}:", FT_PREFIX, name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter { if let Ok((key, _)) = result { if key.starts_with(prefix.as_bytes()) { - batch.delete_cf(cf, key); + batch.delete_cf(&cf, key); count += 1; } else { break; @@ -180,11 +180,11 @@ impl Collection { // Delete terms let term_prefix = format!("{}{}:", FT_TERM_PREFIX, name); - let iter = db.prefix_iterator_cf(cf, term_prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, term_prefix.as_bytes()); for result in iter { if let Ok((key, _)) = result { if key.starts_with(term_prefix.as_bytes()) { - batch.delete_cf(cf, key); + batch.delete_cf(&cf, key); count += 1; } else { break; // Fixed from original loop which implied break @@ -234,14 +234,14 @@ impl Collection { for term in &terms { if term.len() >= index.min_length { let term_key = Self::ft_term_key(&index.name, term, doc_key); - batch.put_cf(cf, term_key, doc_key.as_bytes()); + batch.put_cf(&cf, term_key, doc_key.as_bytes()); } } let ngrams = generate_ngrams(text, NGRAM_SIZE); for ngram in &ngrams { let ngram_key = Self::ft_ngram_key(&index.name, ngram, doc_key); - batch.put_cf(cf, ngram_key, doc_key.as_bytes()); + batch.put_cf(&cf, ngram_key, doc_key.as_bytes()); } } } @@ -277,14 +277,14 @@ impl Collection { for term in &terms { if term.len() >= index.min_length { let term_key = Self::ft_term_key(&index.name, term, doc_key); - batch.delete_cf(cf, term_key); + batch.delete_cf(&cf, term_key); } } let ngrams = generate_ngrams(text, NGRAM_SIZE); for ngram in &ngrams { let ngram_key = Self::ft_ngram_key(&index.name, ngram, doc_key); - batch.delete_cf(cf, ngram_key); + batch.delete_cf(&cf, ngram_key); } } } @@ -339,7 +339,7 @@ impl Collection { if term.len() >= index.min_length { // Exact term lookup let prefix = format!("{}{}:{}:", FT_TERM_PREFIX, index.name, term); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; diff --git a/src/storage/collection/geo.rs b/src/storage/collection/geo.rs index 73dee6d3..5720a186 100644 --- a/src/storage/collection/geo.rs +++ b/src/storage/collection/geo.rs @@ -13,7 +13,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = GEO_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -31,7 +31,7 @@ impl Collection { pub(crate) fn get_geo_index(&self, name: &str) -> Option { let db = &self.db; let cf = db.cf_handle(&self.name)?; - db.get_cf(cf, Self::geo_meta_key(name)) + db.get_cf(&cf, Self::geo_meta_key(name)) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -59,7 +59,7 @@ impl Collection { let cf = db .cf_handle(&self.name) .expect("Column family should exist"); - db.put_cf(cf, Self::geo_meta_key(&name), &index_bytes) + db.put_cf(&cf, Self::geo_meta_key(&name), &index_bytes) .map_err(|e| { DbError::InternalError(format!("Failed to create geo index: {}", e)) })?; @@ -82,7 +82,7 @@ impl Collection { if val.contains_key("lat") || val.contains_key("latitude") { let entry_key = Self::geo_entry_key(&name, &doc.key); let geo_data = serde_json::to_vec(&doc_value[&field])?; - db.put_cf(cf, entry_key, &geo_data).map_err(|e| { + db.put_cf(&cf, entry_key, &geo_data).map_err(|e| { DbError::InternalError(format!("Failed to build geo index: {}", e)) })?; count += 1; @@ -114,17 +114,17 @@ impl Collection { .expect("Column family should exist"); // Delete metadata - db.delete_cf(cf, Self::geo_meta_key(name)) + db.delete_cf(&cf, Self::geo_meta_key(name)) .map_err(|e| DbError::InternalError(format!("Failed to drop geo index: {}", e)))?; // Delete entries let prefix = format!("{}{}:", GEO_PREFIX, name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(prefix.as_bytes()) { - db.delete_cf(cf, &key).map_err(|e| { + db.delete_cf(&cf, &key).map_err(|e| { DbError::InternalError(format!("Failed to drop geo index entry: {}", e)) })?; } else { @@ -147,7 +147,7 @@ impl Collection { .expect("Column family should exist"); let prefix = format!("{}{}:", GEO_PREFIX, idx.name); let count = db - .prefix_iterator_cf(cf, prefix.as_bytes()) + .prefix_iterator_cf(&cf, prefix.as_bytes()) .take_while(|r| { r.as_ref() .is_ok_and(|(k, _)| k.starts_with(prefix.as_bytes())) @@ -183,7 +183,7 @@ impl Collection { .expect("Column family should exist"); let prefix = format!("{}{}:", GEO_PREFIX, index.name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let mut matches = Vec::new(); @@ -246,7 +246,7 @@ impl Collection { .expect("Column family should exist"); let prefix = format!("{}{}:", GEO_PREFIX, index.name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let mut matches = Vec::new(); diff --git a/src/storage/collection/indexes.rs b/src/storage/collection/indexes.rs index 08653600..eb6c8bfe 100644 --- a/src/storage/collection/indexes.rs +++ b/src/storage/collection/indexes.rs @@ -21,7 +21,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = IDX_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -39,7 +39,7 @@ impl Collection { pub(crate) fn get_index(&self, name: &str) -> Option { let db = &self.db; let cf = db.cf_handle(&self.name)?; - db.get_cf(cf, Self::idx_meta_key(name)) + db.get_cf(&cf, Self::idx_meta_key(name)) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -71,7 +71,7 @@ impl Collection { let cf = db .cf_handle(&self.name) .expect("Column family should exist"); - db.put_cf(cf, Self::idx_meta_key(&name), &index_bytes) + db.put_cf(&cf, Self::idx_meta_key(&name), &index_bytes) .map_err(|e| DbError::InternalError(format!("Failed to create index: {}", e)))?; } @@ -96,7 +96,7 @@ impl Collection { if !field_values.iter().all(|v| v.is_null()) { let entry_key = Self::idx_entry_key(&name, &field_values, &doc.key); - batch.put_cf(cf, entry_key, doc.key.as_bytes()); + batch.put_cf(&cf, entry_key, doc.key.as_bytes()); indexed_count += 1; // If bloom/cuckoo filter, also update in-memory filter @@ -169,16 +169,16 @@ impl Collection { const BATCH_SIZE_LIMIT: usize = 10000; // Flush every 10k entries // Delete index metadata - batch.delete_cf(cf, Self::idx_meta_key(name)); + batch.delete_cf(&cf, Self::idx_meta_key(name)); // Delete all index entries let prefix = format!("{}{}:", IDX_PREFIX, name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(prefix.as_bytes()) { - batch.delete_cf(cf, &key); + batch.delete_cf(&cf, &key); deleted_count += 1; // Flush batch periodically to avoid excessive memory usage @@ -259,11 +259,11 @@ impl Collection { // Clear regular indexes for index in &indexes { let prefix = format!("{}{}:", IDX_PREFIX, index.name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(prefix.as_bytes()) { - batch.delete_cf(cf, &key); + batch.delete_cf(&cf, &key); } else { break; } @@ -273,11 +273,11 @@ impl Collection { // Clear geo indexes for geo_index in &geo_indexes { let prefix = format!("{}{}:", GEO_PREFIX, geo_index.name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(prefix.as_bytes()) { - batch.delete_cf(cf, &key); + batch.delete_cf(&cf, &key); } else { break; } @@ -287,22 +287,22 @@ impl Collection { // Clear fulltext indexes for ft_index in &ft_indexes { let ngram_prefix = format!("{}{}:", FT_PREFIX, ft_index.name); - let iter = db.prefix_iterator_cf(cf, ngram_prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, ngram_prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(ngram_prefix.as_bytes()) { - batch.delete_cf(cf, &key); + batch.delete_cf(&cf, &key); } else { break; } } let term_prefix = format!("{}{}:", FT_TERM_PREFIX, ft_index.name); - let iter = db.prefix_iterator_cf(cf, term_prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, term_prefix.as_bytes()); for result in iter.flatten() { let (key, _) = result; if key.starts_with(term_prefix.as_bytes()) { - batch.delete_cf(cf, &key); + batch.delete_cf(&cf, &key); } else { break; } @@ -325,7 +325,7 @@ impl Collection { .expect("Column family should exist"); let prefix = DOC_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); // Build all index types in a single pass with periodic batch flushing // This reduces memory usage and I/O compared to loading all docs then multiple passes @@ -376,7 +376,7 @@ impl Collection { if !field_values.iter().all(|v| v.is_null()) { let entry_key = Self::idx_entry_key(&index.name, &field_values, &doc.key); - batch.put_cf(cf, entry_key, doc.key.as_bytes()); + batch.put_cf(&cf, entry_key, doc.key.as_bytes()); regular_count += 1; // Flush batch periodically @@ -395,7 +395,7 @@ impl Collection { if !field_value.is_null() { let entry_key = Self::geo_entry_key(&geo_index.name, &doc.key); if let Ok(geo_data) = serde_json::to_vec(&field_value) { - batch.put_cf(cf, entry_key, &geo_data); + batch.put_cf(&cf, entry_key, &geo_data); geo_count += 1; // Flush batch periodically @@ -419,7 +419,7 @@ impl Collection { if term.len() >= ft_index.min_length { let term_key = Self::ft_term_key(&ft_index.name, term, &doc.key); - batch.put_cf(cf, term_key, doc.key.as_bytes()); + batch.put_cf(&cf, term_key, doc.key.as_bytes()); ft_count += 1; } } @@ -428,7 +428,7 @@ impl Collection { for ngram in &ngrams { let ngram_key = Self::ft_ngram_key(&ft_index.name, ngram, &doc.key); - batch.put_cf(cf, ngram_key, doc.key.as_bytes()); + batch.put_cf(&cf, ngram_key, doc.key.as_bytes()); ft_count += 1; } @@ -480,7 +480,7 @@ impl Collection { // Re-index all documents - stream from storage let prefix = DOC_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for result in iter.flatten() { let (key, value) = result; @@ -553,7 +553,7 @@ impl Collection { if !field_values.iter().all(|v| v.is_null()) { let entry_key = Self::idx_entry_key(&index.name, &field_values, &doc.key); - batch.put_cf(cf, entry_key, doc.key.as_bytes()); + batch.put_cf(&cf, entry_key, doc.key.as_bytes()); if index.index_type == IndexType::Bloom { for value in &field_values { @@ -580,7 +580,7 @@ impl Collection { if !field_value.is_null() { let entry_key = Self::geo_entry_key(&geo_index.name, &doc.key); if let Ok(geo_data) = serde_json::to_vec(&field_value) { - batch.put_cf(cf, entry_key, &geo_data); + batch.put_cf(&cf, entry_key, &geo_data); } } } @@ -602,14 +602,14 @@ impl Collection { if term.len() >= ft_index.min_length { let term_key = Self::ft_term_key(&ft_index.name, term, &doc.key); - batch.put_cf(cf, term_key, doc.key.as_bytes()); + batch.put_cf(&cf, term_key, doc.key.as_bytes()); } } let ngrams = generate_ngrams(text, NGRAM_SIZE); for ngram in &ngrams { let ngram_key = Self::ft_ngram_key(&ft_index.name, ngram, &doc.key); - batch.put_cf(cf, ngram_key, doc.key.as_bytes()); + batch.put_cf(&cf, ngram_key, doc.key.as_bytes()); } } } @@ -631,7 +631,7 @@ impl Collection { // Count entries let prefix = format!("{}{}:", IDX_PREFIX, name); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let count = iter .filter(|r| { r.as_ref() @@ -687,7 +687,7 @@ impl Collection { // Key format: idx::: // We want to check if ANY doc_key exists for this (index_name, value_part) combo let prefix = format!("{}{}:{}:", IDX_PREFIX, index.name, value_part); - let mut iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let mut iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); // Check if any OTHER document already has this value if let Some(Ok((key, value))) = iter.next() { @@ -739,7 +739,7 @@ impl Collection { if !field_values.iter().all(|v| v.is_null()) { let entry_key = Self::idx_entry_key(&index.name, &field_values, doc_key); - db.put_cf(cf, entry_key, doc_key.as_bytes()).map_err(|e| { + db.put_cf(&cf, entry_key, doc_key.as_bytes()).map_err(|e| { DbError::InternalError(format!("Failed to update index: {}", e)) })?; @@ -768,7 +768,7 @@ impl Collection { if !field_value.is_null() { let entry_key = Self::geo_entry_key(&geo_index.name, doc_key); let geo_data = serde_json::to_vec(&field_value)?; - db.put_cf(cf, entry_key, &geo_data).map_err(|e| { + db.put_cf(&cf, entry_key, &geo_data).map_err(|e| { DbError::InternalError(format!("Failed to update geo index: {}", e)) })?; } @@ -806,7 +806,7 @@ impl Collection { // Remove old entry if !old_values.iter().all(|v| v.is_null()) { let old_entry_key = Self::idx_entry_key(&index.name, &old_values, doc_key); - db.delete_cf(cf, old_entry_key).map_err(|e| { + db.delete_cf(&cf, old_entry_key).map_err(|e| { DbError::InternalError(format!("Failed to update index: {}", e)) })?; } @@ -814,7 +814,7 @@ impl Collection { // Add new entry if !new_values.iter().all(|v| v.is_null()) { let new_entry_key = Self::idx_entry_key(&index.name, &new_values, doc_key); - db.put_cf(cf, new_entry_key, doc_key.as_bytes()) + db.put_cf(&cf, new_entry_key, doc_key.as_bytes()) .map_err(|e| { DbError::InternalError(format!("Failed to update index: {}", e)) })?; @@ -835,11 +835,11 @@ impl Collection { if !new_field.is_null() { let geo_data = serde_json::to_vec(&new_field)?; - db.put_cf(cf, entry_key, &geo_data).map_err(|e| { + db.put_cf(&cf, entry_key, &geo_data).map_err(|e| { DbError::InternalError(format!("Failed to update geo index: {}", e)) })?; } else { - db.delete_cf(cf, entry_key).map_err(|e| { + db.delete_cf(&cf, entry_key).map_err(|e| { DbError::InternalError(format!("Failed to update geo index: {}", e)) })?; } @@ -870,7 +870,7 @@ impl Collection { if !field_values.iter().all(|v| v.is_null()) { let entry_key = Self::idx_entry_key(&index.name, &field_values, doc_key); - db.delete_cf(cf, entry_key).map_err(|e| { + db.delete_cf(&cf, entry_key).map_err(|e| { DbError::InternalError(format!("Failed to update index: {}", e)) })?; } @@ -886,7 +886,7 @@ impl Collection { for geo_index in geo_indexes { let entry_key = Self::geo_entry_key(&geo_index.name, doc_key); - db.delete_cf(cf, entry_key).map_err(|e| { + db.delete_cf(&cf, entry_key).map_err(|e| { DbError::InternalError(format!("Failed to update geo index: {}", e)) })?; } @@ -901,7 +901,7 @@ impl Collection { // Try to find index by checking idx_meta entries let prefix = IDX_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for result in iter.flatten() { let (key, value) = result; @@ -928,7 +928,7 @@ impl Collection { let cf = db.cf_handle(&self.name)?; let prefix = IDX_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for result in iter.flatten() { let (key, value) = result; @@ -988,7 +988,7 @@ impl Collection { if forward { let mode = IteratorMode::From(seek_key.as_bytes(), Direction::Forward); - let iter = db.iterator_cf(cf, mode); + let iter = db.iterator_cf(&cf, mode); for result in iter { if let Ok((k, v)) = result { @@ -1012,7 +1012,7 @@ impl Collection { // For reverse, we might land ON the key or BEFORE it. // If we land on it, it matches value. let mode = IteratorMode::From(seek_key.as_bytes(), Direction::Reverse); - let iter = db.iterator_cf(cf, mode); + let iter = db.iterator_cf(&cf, mode); for result in iter { if let Ok((k, v)) = result { @@ -1037,7 +1037,7 @@ impl Collection { return Some(Vec::new()); } - let results = db.multi_get_cf(doc_keys.iter().map(|k| (cf, k.as_slice()))); + let results = db.multi_get_cf(doc_keys.iter().map(|k| (&cf, k.as_slice()))); let docs: Vec = results .into_iter() .filter_map(|r| r.ok()) @@ -1074,7 +1074,7 @@ impl Collection { // Prefix for the lookup let prefix = format!("{}{}:{}:", IDX_PREFIX, index.name, value_str); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); // Collect document keys from index let doc_keys: Vec> = iter @@ -1097,7 +1097,7 @@ impl Collection { } // Use multi_get for batch retrieval - let results = db.multi_get_cf(doc_keys.iter().map(|k| (cf, k.as_slice()))); + let results = db.multi_get_cf(doc_keys.iter().map(|k| (&cf, k.as_slice()))); let docs: Vec = results .into_iter() @@ -1171,7 +1171,7 @@ impl Collection { let db = &self.db; let cf = db.cf_handle(&self.name)?; let prefix = format!("{}{}:{}:", IDX_PREFIX, index.name, value_part); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let doc_keys: Vec> = iter .filter_map(|r| r.ok()) @@ -1186,7 +1186,7 @@ impl Collection { return Some((index, Vec::new())); } - let results = db.multi_get_cf(doc_keys.iter().map(|k| (cf, k.as_slice()))); + let results = db.multi_get_cf(doc_keys.iter().map(|k| (&cf, k.as_slice()))); let docs: Vec = results .into_iter() .filter_map(|r| r.ok()) @@ -1217,7 +1217,7 @@ impl Collection { let cf = db.cf_handle(&self.name)?; let prefix = format!("{}{}:{}:", IDX_PREFIX, index.name, value_str); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let doc_keys: Vec> = iter .filter_map(|r| r.ok()) @@ -1233,7 +1233,7 @@ impl Collection { return Some(Vec::new()); } - let results = db.multi_get_cf(doc_keys.iter().map(|k| (cf, k.as_slice()))); + let results = db.multi_get_cf(doc_keys.iter().map(|k| (&cf, k.as_slice()))); let docs: Vec = results .into_iter() .filter_map(|r| r.ok()) @@ -1259,13 +1259,13 @@ impl Collection { let iter = if ascending { let mode = IteratorMode::From(prefix, Direction::Forward); - db.iterator_cf(cf, mode) + db.iterator_cf(&cf, mode) } else { // For descending, we seek past the end of the prefix let mut seek_key = prefix.to_vec(); seek_key.push(0xFF); let mode = IteratorMode::From(&seek_key, Direction::Reverse); - db.iterator_cf(cf, mode) + db.iterator_cf(&cf, mode) }; let docs: Vec = iter @@ -1288,12 +1288,12 @@ impl Collection { let iter = if ascending { let mode = IteratorMode::From(prefix_bytes, Direction::Forward); - db.iterator_cf(cf, mode) + db.iterator_cf(&cf, mode) } else { let mut seek_key = prefix.as_bytes().to_vec(); seek_key.push(0xFF); let mode = IteratorMode::From(&seek_key, Direction::Reverse); - db.iterator_cf(cf, mode) + db.iterator_cf(&cf, mode) }; let doc_keys: Vec = iter @@ -1336,7 +1336,7 @@ impl Collection { .cf_handle(&self.name) .ok_or(DbError::InternalError("CF not found".into()))?; let key = format!("{}{}", BLO_IDX_PREFIX, index_name); - let new_filter = if let Ok(Some(bytes)) = db.get_cf(cf, key.as_bytes()) { + let new_filter = if let Ok(Some(bytes)) = db.get_cf(&cf, key.as_bytes()) { // Deserialize bloom filter from bytes if let Ok(filter) = serde_json::from_slice::(&bytes) { filter @@ -1360,7 +1360,7 @@ impl Collection { .ok_or(DbError::InternalError("CF not found".into()))?; let key = format!("{}{}", BLO_IDX_PREFIX, index_name); let bytes = serde_json::to_vec(filter)?; - db.put_cf(cf, key.as_bytes(), &bytes) + db.put_cf(&cf, key.as_bytes(), &bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(()) } @@ -1392,7 +1392,7 @@ impl Collection { let db = &self.db; if let Some(cf) = db.cf_handle(&self.name) { let key = format!("{}{}", CFO_IDX_PREFIX, index_name); - if let Ok(Some(_bytes)) = db.get_cf(cf, key.as_bytes()) { + if let Ok(Some(_bytes)) = db.get_cf(&cf, key.as_bytes()) { // CuckooFilter doesn't implement Serialize/Deserialize due to DefaultHasher, // so we create a new empty filter instead of deserializing // This means bloom filters are rebuilt on restart - acceptable for the use case diff --git a/src/storage/collection/mod.rs b/src/storage/collection/mod.rs index 2f06aaf0..c7b36bf5 100644 --- a/src/storage/collection/mod.rs +++ b/src/storage/collection/mod.rs @@ -1,6 +1,6 @@ +use crate::storage::RocksDb as DB; use dashmap::DashMap; use parking_lot::RwLock; -use rust_rocksdb::DB; use serde_json::Value; use std::collections::hash_map::DefaultHasher; use std::sync::atomic::{AtomicBool, AtomicUsize}; diff --git a/src/storage/collection/schema.rs b/src/storage/collection/schema.rs index 1efbb4b8..af5de301 100644 --- a/src/storage/collection/schema.rs +++ b/src/storage/collection/schema.rs @@ -81,7 +81,7 @@ impl Collection { .expect("Column family should exist"); let schema_bytes = serde_json::to_vec(&schema)?; - db.put_cf(cf, SCHEMA_KEY.as_bytes(), &schema_bytes) + db.put_cf(&cf, SCHEMA_KEY.as_bytes(), &schema_bytes) .map_err(|e| DbError::InternalError(format!("Failed to set schema: {}", e)))?; Ok(()) @@ -92,7 +92,7 @@ impl Collection { let db = &self.db; let cf = db.cf_handle(&self.name)?; - db.get_cf(cf, SCHEMA_KEY.as_bytes()) + db.get_cf(&cf, SCHEMA_KEY.as_bytes()) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -107,7 +107,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); - db.delete_cf(cf, SCHEMA_KEY.as_bytes()) + db.delete_cf(&cf, SCHEMA_KEY.as_bytes()) .map_err(|e| DbError::InternalError(format!("Failed to remove schema: {}", e)))?; Ok(()) diff --git a/src/storage/collection/ttl.rs b/src/storage/collection/ttl.rs index 059eac58..0ed1eac4 100644 --- a/src/storage/collection/ttl.rs +++ b/src/storage/collection/ttl.rs @@ -18,7 +18,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = TTL_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -36,7 +36,7 @@ impl Collection { pub fn get_ttl_index(&self, name: &str) -> Option { let db = &self.db; let cf = db.cf_handle(&self.name)?; - db.get_cf(cf, Self::ttl_meta_key(name)) + db.get_cf(&cf, Self::ttl_meta_key(name)) .ok() .flatten() .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -68,7 +68,7 @@ impl Collection { let cf = db .cf_handle(&self.name) .expect("Column family should exist"); - db.put_cf(cf, Self::ttl_meta_key(&name), &index_bytes) + db.put_cf(&cf, Self::ttl_meta_key(&name), &index_bytes) .map_err(|e| { DbError::InternalError(format!("Failed to create TTL index: {}", e)) })?; @@ -99,7 +99,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); - db.delete_cf(cf, Self::ttl_meta_key(name)) + db.delete_cf(&cf, Self::ttl_meta_key(name)) .map_err(|e| DbError::InternalError(format!("Failed to drop TTL index: {}", e)))?; Ok(()) @@ -236,7 +236,7 @@ impl Collection { let mut expired_doc_keys: Vec = Vec::new(); let mut expired_expiry_keys: Vec> = Vec::new(); - let iter = db.prefix_iterator_cf(cf, prefix.as_slice()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_slice()); for result in iter.flatten() { let (key_bytes, _value) = result; @@ -288,12 +288,12 @@ impl Collection { let base_idx = deleted_count; for (i, key) in chunk.iter().enumerate() { // Delete document - batch.delete_cf(cf, Self::doc_key(key)); + batch.delete_cf(&cf, Self::doc_key(key)); // Delete expiry index entry let expiry_idx = base_idx.saturating_add(i); if let Some(expiry_key) = expired_expiry_keys.get(expiry_idx) { - batch.delete_cf(cf, expiry_key); + batch.delete_cf(&cf, expiry_key); } deleted_count += 1; diff --git a/src/storage/collection/txn.rs b/src/storage/collection/txn.rs index e3d2d641..cad21f5a 100644 --- a/src/storage/collection/txn.rs +++ b/src/storage/collection/txn.rs @@ -152,28 +152,28 @@ impl Collection { // Serialize and add document to batch let document = Document::with_key(&self.name, key.clone(), data.clone()); let doc_bytes = serialize_doc(&document)?; - batch.put_cf(cf, Self::doc_key(key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(key), &doc_bytes); // Compute and add index entries to batch let (regular_entries, geo_entries) = self.compute_index_entries_for_insert(key, data)?; for (entry_key, entry_value) in regular_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add fulltext entries let fulltext_entries = self.compute_fulltext_entries_for_insert(key, data); for (entry_key, entry_value) in fulltext_entries { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and add TTL expiry entries let ttl_expiry_entries = self.compute_ttl_expiry_entries_for_insert(key, data); for (entry_key, _entry_value) in ttl_expiry_entries { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } // Queue change event for after commit @@ -193,7 +193,7 @@ impl Collection { // Serialize and add document to batch let document = Document::with_key(&self.name, key.clone(), new_data.clone()); let doc_bytes = serialize_doc(&document)?; - batch.put_cf(cf, Self::doc_key(key), &doc_bytes); + batch.put_cf(&cf, Self::doc_key(key), &doc_bytes); // Compute and apply index updates atomically let (entries_to_add, keys_to_remove, geo_entries_to_add, geo_keys_to_remove) = @@ -201,41 +201,41 @@ impl Collection { // Remove old index entries for key_to_remove in keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for geo_key in geo_keys_to_remove { - batch.delete_cf(cf, geo_key); + batch.delete_cf(&cf, geo_key); } // Add new index entries for (entry_key, entry_value) in entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } for (entry_key, entry_value) in geo_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply fulltext updates let fulltext_keys_to_remove = self.compute_fulltext_entries_for_delete(key, old_data); for key_to_remove in fulltext_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } let fulltext_entries_to_add = self.compute_fulltext_entries_for_insert(key, new_data); for (entry_key, entry_value) in fulltext_entries_to_add { - batch.put_cf(cf, entry_key, entry_value); + batch.put_cf(&cf, entry_key, entry_value); } // Compute and apply TTL expiry updates let (ttl_entries_to_add, ttl_keys_to_remove) = self.compute_ttl_expiry_entries_for_update(key, old_data, new_data); for key_to_remove in ttl_keys_to_remove { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for (entry_key, _entry_value) in ttl_entries_to_add { - batch.put_cf(cf, entry_key, Vec::new()); + batch.put_cf(&cf, entry_key, Vec::new()); } // Queue change event for after commit @@ -248,28 +248,28 @@ impl Collection { } Operation::Delete { key, old_data, .. } => { // Delete document from batch - batch.delete_cf(cf, Self::doc_key(key)); + batch.delete_cf(&cf, Self::doc_key(key)); // Compute and remove index entries let (regular_keys, geo_keys) = self.compute_index_entries_for_delete(key, old_data)?; for key_to_remove in regular_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } for key_to_remove in geo_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and remove fulltext entries let fulltext_keys = self.compute_fulltext_entries_for_delete(key, old_data); for key_to_remove in fulltext_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Compute and remove TTL expiry entries let ttl_keys = self.compute_ttl_expiry_entries_for_delete(key, old_data); for key_to_remove in ttl_keys { - batch.delete_cf(cf, key_to_remove); + batch.delete_cf(&cf, key_to_remove); } // Queue change event for after commit diff --git a/src/storage/collection/vector.rs b/src/storage/collection/vector.rs index 3eeac9e1..f8144566 100644 --- a/src/storage/collection/vector.rs +++ b/src/storage/collection/vector.rs @@ -27,7 +27,7 @@ impl Collection { .cf_handle(&self.name) .expect("Column family should exist"); let prefix = VEC_META_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, value)| { @@ -64,7 +64,7 @@ impl Collection { // Check metadata first let meta_key = Self::vec_meta_key(name); - if db.get_cf(cf, &meta_key)?.is_none() { + if db.get_cf(&cf, &meta_key)?.is_none() { return Err(DbError::InvalidDocument(format!( "Vector Index '{}' not found", name @@ -73,7 +73,7 @@ impl Collection { // Load data let data_key = Self::vec_data_key(name); - if let Some(bytes) = db.get_cf(cf, &data_key)? { + if let Some(bytes) = db.get_cf(&cf, &data_key)? { match super::vector::VectorIndex::deserialize(&bytes) { Ok(index) => { let index_arc = Arc::new(index); @@ -108,7 +108,7 @@ impl Collection { let cf = db .cf_handle(&self.name) .expect("Column family should exist"); - db.put_cf(cf, Self::vec_meta_key(&name), &config_bytes) + db.put_cf(&cf, Self::vec_meta_key(&name), &config_bytes) .map_err(|e| { DbError::InternalError(format!("Failed to create vector index: {}", e)) })?; @@ -154,10 +154,10 @@ impl Collection { self.vector_indexes.remove(name); // Remove from disk - db.delete_cf(cf, Self::vec_meta_key(name)).map_err(|e| { + db.delete_cf(&cf, Self::vec_meta_key(name)).map_err(|e| { DbError::InternalError(format!("Failed to delete vector config: {}", e)) })?; - db.delete_cf(cf, Self::vec_data_key(name)) + db.delete_cf(&cf, Self::vec_data_key(name)) .map_err(|e| DbError::InternalError(format!("Failed to delete vector data: {}", e)))?; Ok(()) @@ -231,7 +231,7 @@ impl Collection { let db = &self.db; let cf = db.cf_handle(&self.name).unwrap(); let config_bytes = serde_json::to_vec(config)?; - db.put_cf(cf, Self::vec_meta_key(name), &config_bytes) + db.put_cf(&cf, Self::vec_meta_key(name), &config_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; } @@ -261,7 +261,7 @@ impl Collection { let name = entry.key(); let index_arc = entry.value(); let bytes = index_arc.serialize()?; - db.put_cf(cf, Self::vec_data_key(name), &bytes) + db.put_cf(&cf, Self::vec_data_key(name), &bytes) .map_err(|e| { DbError::InternalError(format!( "Failed to persist vector index {}: {}", diff --git a/src/storage/columnar.rs b/src/storage/columnar.rs index de21684c..2bf0f40b 100644 --- a/src/storage/columnar.rs +++ b/src/storage/columnar.rs @@ -8,7 +8,9 @@ use fastbloom::BloomFilter; use lz4_flex::{compress_prepend_size, decompress_size_prepended}; -use rust_rocksdb::{ColumnFamily, DB}; +use rust_rocksdb::AsColumnFamilyRef; + +use super::RocksDb as DB; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; @@ -255,14 +257,17 @@ impl ColumnarCollection { }; // Store metadata (lock-free, RocksDB is thread-safe) - let cf = db.cf_handle(&cf_name).ok_or_else(|| { - DbError::CollectionNotFound(format!("Columnar CF '{}' not found", cf_name)) - })?; + // Scope the CF handle so its borrow of `db` ends before `db` is moved + { + let cf = db.cf_handle(&cf_name).ok_or_else(|| { + DbError::CollectionNotFound(format!("Columnar CF '{}' not found", cf_name)) + })?; - let meta_key = format!("{}{}", COL_META_PREFIX, name); - let meta_bytes = serde_json::to_vec(&meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) - .map_err(|e| DbError::InternalError(e.to_string()))?; + let meta_key = format!("{}{}", COL_META_PREFIX, name); + let meta_bytes = serde_json::to_vec(&meta)?; + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) + .map_err(|e| DbError::InternalError(e.to_string()))?; + } Ok(Self { name, @@ -277,21 +282,23 @@ impl ColumnarCollection { let cf_name = format!("{}:_columnar_{}", db_name, name); // Lock-free read - RocksDB is thread-safe - let cf = db.cf_handle(&cf_name).ok_or_else(|| { - DbError::CollectionNotFound(format!("Columnar collection '{}' not found", name)) - })?; - - let meta_key = format!("{}{}", COL_META_PREFIX, name); - let meta_bytes = db - .get_cf(cf, meta_key.as_bytes()) - .map_err(|e| DbError::InternalError(e.to_string()))? - .ok_or_else(|| { - DbError::CollectionNotFound(format!( - "Columnar collection metadata '{}' not found", - name - )) + // Scope the CF handle so its borrow of `db` ends before `db` is moved + let meta_bytes = { + let cf = db.cf_handle(&cf_name).ok_or_else(|| { + DbError::CollectionNotFound(format!("Columnar collection '{}' not found", name)) })?; + let meta_key = format!("{}{}", COL_META_PREFIX, name); + db.get_cf(&cf, meta_key.as_bytes()) + .map_err(|e| DbError::InternalError(e.to_string()))? + .ok_or_else(|| { + DbError::CollectionNotFound(format!( + "Columnar collection metadata '{}' not found", + name + )) + })? + }; + let meta = serde_json::from_slice::(&meta_bytes)?; Ok(Self { @@ -333,18 +340,18 @@ impl ColumnarCollection { let value_bytes = serde_json::to_vec(value)?; let stored_bytes = self.compress_data(&value_bytes, &meta.compression); - db.put_cf(cf, col_key.as_bytes(), &stored_bytes) + db.put_cf(&cf, col_key.as_bytes(), &stored_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; } // Update indexes for indexed columns (using UUID string) - self.update_indexes_for_row_uuid(db, cf, &meta.columns, obj, &row_uuid)?; + self.update_indexes_for_row_uuid(db, &cf, &meta.columns, obj, &row_uuid)?; // Also store full row for reconstruction let row_key = format!("{}{}", COL_ROW_PREFIX, row_uuid); let row_bytes = serde_json::to_vec(row)?; let stored_row = self.compress_data(&row_bytes, &meta.compression); - db.put_cf(cf, row_key.as_bytes(), &stored_row) + db.put_cf(&cf, row_key.as_bytes(), &stored_row) .map_err(|e| DbError::InternalError(e.to_string()))?; inserted_ids.push(row_uuid); @@ -357,7 +364,7 @@ impl ColumnarCollection { let meta_key = format!("{}{}", COL_META_PREFIX, self.name); let meta_bytes = serde_json::to_vec(&*meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(inserted_ids) @@ -378,7 +385,7 @@ impl ColumnarCollection { // Check if row already exists (idempotency for replication) let row_key = format!("{}{}", COL_ROW_PREFIX, row_uuid); if db - .get_cf(cf, row_key.as_bytes()) + .get_cf(&cf, row_key.as_bytes()) .map_err(|e| DbError::InternalError(e.to_string()))? .is_some() { @@ -394,17 +401,17 @@ impl ColumnarCollection { let value_bytes = serde_json::to_vec(value)?; let stored_bytes = self.compress_data(&value_bytes, &meta.compression); - db.put_cf(cf, col_key.as_bytes(), &stored_bytes) + db.put_cf(&cf, col_key.as_bytes(), &stored_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; } // Update indexes for indexed columns - self.update_indexes_for_row_uuid(db, cf, &meta.columns, obj, row_uuid)?; + self.update_indexes_for_row_uuid(db, &cf, &meta.columns, obj, row_uuid)?; // Store full row for reconstruction let row_bytes = serde_json::to_vec(&row)?; let stored_row = self.compress_data(&row_bytes, &meta.compression); - db.put_cf(cf, row_key.as_bytes(), &stored_row) + db.put_cf(&cf, row_key.as_bytes(), &stored_row) .map_err(|e| DbError::InternalError(e.to_string()))?; // Update metadata @@ -413,7 +420,7 @@ impl ColumnarCollection { let meta_key = format!("{}{}", COL_META_PREFIX, self.name); let meta_bytes = serde_json::to_vec(&*meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(true) @@ -436,7 +443,7 @@ impl ColumnarCollection { // Check if row exists let row_key = format!("{}{}", COL_ROW_PREFIX, row_uuid); if db - .get_cf(cf, row_key.as_bytes()) + .get_cf(&cf, row_key.as_bytes()) .map_err(|e| DbError::InternalError(e.to_string()))? .is_none() { @@ -446,12 +453,12 @@ impl ColumnarCollection { // Delete column values for col_def in &meta.columns { let col_key = format!("{}{}:{}", COL_DATA_PREFIX, col_def.name, row_uuid); - db.delete_cf(cf, col_key.as_bytes()) + db.delete_cf(&cf, col_key.as_bytes()) .map_err(|e| DbError::InternalError(e.to_string()))?; } // Delete full row - db.delete_cf(cf, row_key.as_bytes()) + db.delete_cf(&cf, row_key.as_bytes()) .map_err(|e| DbError::InternalError(e.to_string()))?; // Update metadata @@ -462,7 +469,7 @@ impl ColumnarCollection { let meta_key = format!("{}{}", COL_META_PREFIX, self.name); let meta_bytes = serde_json::to_vec(&*meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(true) @@ -476,7 +483,7 @@ impl ColumnarCollection { })?; let prefix = COL_ROW_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); let mut uuids = Vec::new(); for item in iter.flatten() { @@ -537,7 +544,7 @@ impl ColumnarCollection { for row_uuid in &uuids { let col_key = format!("{}{}:{}", COL_DATA_PREFIX, column, row_uuid); - match db.get_cf(cf, col_key.as_bytes()) { + match db.get_cf(&cf, col_key.as_bytes()) { Ok(Some(bytes)) => { let decompressed = self.decompress_data(&bytes, &meta.compression)?; let value: Value = serde_json::from_slice(&decompressed)?; @@ -601,7 +608,7 @@ impl ColumnarCollection { for col in columns { let col_key = format!("{}{}:{}", COL_DATA_PREFIX, col, row_uuid); - let value = match db.get_cf(cf, col_key.as_bytes()) { + let value = match db.get_cf(&cf, col_key.as_bytes()) { Ok(Some(bytes)) => { let decompressed = self.decompress_data(&bytes, &meta.compression)?; serde_json::from_slice(&decompressed)? @@ -637,7 +644,7 @@ impl ColumnarCollection { let col_key = format!("{}{}:{}", COL_DATA_PREFIX, column, row_uuid); - match db.get_cf(cf, col_key.as_bytes()) { + match db.get_cf(&cf, col_key.as_bytes()) { Ok(Some(bytes)) => { let decompressed = self.decompress_data(&bytes, &meta.compression)?; let value: Value = serde_json::from_slice(&decompressed)?; @@ -681,7 +688,7 @@ impl ColumnarCollection { // Optimization: Use prefix iterator directly to avoid loading all values into memory // This relies on the fact that aggregation is commutative/associative and order doesn't matter let prefix = format!("{}{}:", COL_DATA_PREFIX, column); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); match op { AggregateOp::Count => { @@ -947,7 +954,7 @@ impl ColumnarCollection { // Iterate over all column data to calculate sizes let prefix = COL_DATA_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for (_, value) in iter.flatten() { compressed_size += value.len() as u64; @@ -994,25 +1001,25 @@ impl ColumnarCollection { // Delete all row data (col: prefix) let prefix = COL_DATA_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, prefix); + let iter = db.prefix_iterator_cf(&cf, prefix); for (key, _) in iter.flatten() { - db.delete_cf(cf, &key) + db.delete_cf(&cf, &key) .map_err(|e| DbError::InternalError(format!("Failed to delete: {}", e)))?; } // Delete all row entries (col_row: prefix) let row_prefix = COL_ROW_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, row_prefix); + let iter = db.prefix_iterator_cf(&cf, row_prefix); for (key, _) in iter.flatten() { - db.delete_cf(cf, &key) + db.delete_cf(&cf, &key) .map_err(|e| DbError::InternalError(format!("Failed to delete: {}", e)))?; } // Delete all index data (idx: prefix) let idx_prefix = b"idx:"; - let iter = db.prefix_iterator_cf(cf, idx_prefix); + let iter = db.prefix_iterator_cf(&cf, idx_prefix); for (key, _) in iter.flatten() { - db.delete_cf(cf, &key) + db.delete_cf(&cf, &key) .map_err(|e| DbError::InternalError(format!("Failed to delete: {}", e)))?; } @@ -1022,7 +1029,7 @@ impl ColumnarCollection { // Save updated metadata let meta_bytes = serde_json::to_vec(&*meta) .map_err(|e| DbError::InternalError(format!("Failed to serialize meta: {}", e)))?; - db.put_cf(cf, b"meta", &meta_bytes) + db.put_cf(&cf, b"meta", &meta_bytes) .map_err(|e| DbError::InternalError(format!("Failed to save meta: {}", e)))?; Ok(()) @@ -1030,14 +1037,10 @@ impl ColumnarCollection { /// Drop the entire columnar collection (removes everything including schema) pub fn drop(&self) -> DbResult<()> { - // Safety: We're dropping the CF which requires mutable access to DB - // We use unsafe here since DB is behind an Arc - let db_ptr = Arc::as_ptr(&self.db) as *mut DB; - unsafe { - (*db_ptr) - .drop_cf(&self.cf_name) - .map_err(|e| DbError::InternalError(format!("Failed to drop CF: {}", e)))?; - } + // MultiThreaded mode: drop_cf takes &self and synchronizes internally + self.db + .drop_cf(&self.cf_name) + .map_err(|e| DbError::InternalError(format!("Failed to drop CF: {}", e)))?; Ok(()) } @@ -1075,7 +1078,7 @@ impl ColumnarCollection { // Build index from existing data by iterating over actual row UUIDs // Get all row UUIDs from the col_row: prefix let row_prefix = COL_ROW_PREFIX.as_bytes(); - let iter = db.prefix_iterator_cf(cf, row_prefix); + let iter = db.prefix_iterator_cf(&cf, row_prefix); let mut row_uuids = Vec::new(); for (key, _) in iter.flatten() { @@ -1091,7 +1094,7 @@ impl ColumnarCollection { // Build index using UUIDs, but storing row position for bitmap for (row_id, row_uuid) in row_uuids.iter().enumerate() { let col_key = format!("{}{}:{}", COL_DATA_PREFIX, column, row_uuid); - if let Ok(Some(bytes)) = db.get_cf(cf, col_key.as_bytes()) { + if let Ok(Some(bytes)) = db.get_cf(&cf, col_key.as_bytes()) { if let Ok(decompressed) = self.decompress_data(&bytes, &compression) { if let Ok(value) = serde_json::from_slice::(&decompressed) { match index_type { @@ -1099,21 +1102,21 @@ impl ColumnarCollection { let idx_key = self.encode_bitmap_key(column, &value); self.append_row_to_bitmap_index( db, - cf, + &cf, &idx_key, row_id as u64, &compression, )?; } ColumnarIndexType::MinMax => { - self.update_minmax_index(db, cf, column, &value, row_id as u64)?; + self.update_minmax_index(db, &cf, column, &value, row_id as u64)?; } ColumnarIndexType::Bloom => { - self.update_bloom_index(db, cf, column, &value, row_id as u64)?; + self.update_bloom_index(db, &cf, column, &value, row_id as u64)?; } _ => { let idx_key = self.encode_index_key(column, &value); - self.append_row_to_index(db, cf, &idx_key, row_id as u64)?; + self.append_row_to_index(db, &cf, &idx_key, row_id as u64)?; } } } @@ -1129,7 +1132,7 @@ impl ColumnarCollection { }; let idx_meta_key = format!("{}{}", COL_IDX_META_PREFIX, column); let idx_meta_bytes = serde_json::to_vec(&idx_meta)?; - db.put_cf(cf, idx_meta_key.as_bytes(), &idx_meta_bytes) + db.put_cf(&cf, idx_meta_key.as_bytes(), &idx_meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; // Mark column as indexed @@ -1139,7 +1142,7 @@ impl ColumnarCollection { // Update collection metadata let meta_key = format!("{}{}", COL_META_PREFIX, self.name); let meta_bytes = serde_json::to_vec(&*meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(()) @@ -1172,18 +1175,18 @@ impl ColumnarCollection { // Delete all index entries for this column let prefix = format!("{}{}:", COL_IDX_PREFIX, column); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); for (key, _) in iter.flatten() { if !key.starts_with(prefix.as_bytes()) { break; } - db.delete_cf(cf, &key) + db.delete_cf(&cf, &key) .map_err(|e| DbError::InternalError(e.to_string()))?; } // Delete index metadata let idx_meta_key = format!("{}{}", COL_IDX_META_PREFIX, column); - db.delete_cf(cf, idx_meta_key.as_bytes()) + db.delete_cf(&cf, idx_meta_key.as_bytes()) .map_err(|e| DbError::InternalError(e.to_string()))?; // Mark column as not indexed @@ -1192,7 +1195,7 @@ impl ColumnarCollection { // Update collection metadata let meta_key = format!("{}{}", COL_META_PREFIX, self.name); let meta_bytes = serde_json::to_vec(&*meta)?; - db.put_cf(cf, meta_key.as_bytes(), &meta_bytes) + db.put_cf(&cf, meta_key.as_bytes(), &meta_bytes) .map_err(|e| DbError::InternalError(e.to_string()))?; Ok(()) @@ -1213,7 +1216,7 @@ impl ColumnarCollection { for col_def in &meta.columns { if col_def.indexed { let idx_meta_key = format!("{}{}", COL_IDX_META_PREFIX, col_def.name); - if let Ok(Some(bytes)) = db.get_cf(cf, idx_meta_key.as_bytes()) { + if let Ok(Some(bytes)) = db.get_cf(&cf, idx_meta_key.as_bytes()) { if let Ok(idx_meta) = serde_json::from_slice::(&bytes) { indexes.push(idx_meta); } @@ -1240,7 +1243,7 @@ impl ColumnarCollection { // Check index type if let Some(cf) = self.db.cf_handle(&self.cf_name) { let idx_meta_key = format!("{}{}", COL_IDX_META_PREFIX, column); - if let Ok(Some(bytes)) = self.db.get_cf(cf, idx_meta_key.as_bytes()) { + if let Ok(Some(bytes)) = self.db.get_cf(&cf, idx_meta_key.as_bytes()) { if let Ok(idx_meta) = serde_json::from_slice::(&bytes) { return idx_meta.index_type == ColumnarIndexType::Sorted; } @@ -1269,7 +1272,7 @@ impl ColumnarCollection { let idx_key = self.encode_bitmap_key(column, value); let mut row_ids = Vec::new(); - if let Ok(Some(bytes)) = db.get_cf(cf, idx_key.as_bytes()) { + if let Ok(Some(bytes)) = db.get_cf(&cf, idx_key.as_bytes()) { if let Ok(bitmap) = self.decompress_data(&bytes, &CompressionType::Lz4) { for (byte_idx, &byte) in bitmap.iter().enumerate() { if byte == 0 { @@ -1309,7 +1312,7 @@ impl ColumnarCollection { for chunk_id in 0..chunk_count { let idx_key = self.encode_minmax_key(column, chunk_id); - if let Ok(Some(bytes)) = db.get_cf(cf, idx_key.as_bytes()) { + if let Ok(Some(bytes)) = db.get_cf(&cf, idx_key.as_bytes()) { if let Ok(chunk) = serde_json::from_slice::(&bytes) { let mut matches = false; match filter { @@ -1364,7 +1367,7 @@ impl ColumnarCollection { // If bloom filter missing, assume match (safe default) let mut matches = true; - if let Ok(Some(bytes)) = db.get_cf(cf, idx_key.as_bytes()) { + if let Ok(Some(bytes)) = db.get_cf(&cf, idx_key.as_bytes()) { if let Ok(filter) = serde_json::from_slice::(&bytes) { matches = filter.contains(&val_str); } @@ -1388,7 +1391,7 @@ impl ColumnarCollection { })?; let idx_key = self.encode_index_key(column, value); - match db.get_cf(cf, idx_key.as_bytes()) { + match db.get_cf(&cf, idx_key.as_bytes()) { Ok(Some(bytes)) => { let row_ids: Vec = serde_json::from_slice(&bytes)?; Ok(row_ids) @@ -1406,7 +1409,7 @@ impl ColumnarCollection { })?; let prefix = format!("{}{}:", COL_IDX_PREFIX, column); - let iter = db.prefix_iterator_cf(cf, prefix.as_bytes()); + let iter = db.prefix_iterator_cf(&cf, prefix.as_bytes()); let mut result = Vec::new(); @@ -1551,7 +1554,7 @@ impl ColumnarCollection { fn append_row_to_index( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, idx_key: &str, row_id: u64, ) -> DbResult<()> { @@ -1574,7 +1577,7 @@ impl ColumnarCollection { fn append_row_to_bitmap_index( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, idx_key: &str, row_id: u64, _compression: &CompressionType, @@ -1608,7 +1611,7 @@ impl ColumnarCollection { fn update_bloom_index( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, column: &str, value: &Value, row_id: u64, @@ -1635,7 +1638,7 @@ impl ColumnarCollection { fn update_minmax_index( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, column: &str, value: &Value, row_id: u64, @@ -1674,7 +1677,7 @@ impl ColumnarCollection { fn update_indexes_for_row_uuid( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, columns: &[ColumnDef], row: &serde_json::Map, row_uuid: &str, @@ -1709,7 +1712,7 @@ impl ColumnarCollection { fn append_uuid_to_index( &self, db: &DB, - cf: &ColumnFamily, + cf: &impl AsColumnFamilyRef, idx_key: &str, row_uuid: &str, ) -> DbResult<()> { diff --git a/src/storage/database.rs b/src/storage/database.rs index 933bc84e..b02aa9fc 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -1,9 +1,10 @@ +use super::RocksDb as DB; use dashmap::DashMap; -use rust_rocksdb::{Options, DB}; use std::sync::{Arc, RwLock}; use super::collection::Collection; use super::columnar::*; +use super::engine::tuned_cf_options; use crate::error::{DbError, DbResult}; use serde_json::Value; @@ -63,20 +64,20 @@ impl Database { return Err(DbError::CollectionAlreadyExists(collection_name)); } - let db_ptr = Arc::as_ptr(&self.db) as *mut DB; - unsafe { - (*db_ptr) - .create_cf(&cf_name, &Options::default()) - .map_err(|e| { - DbError::InternalError(format!("Failed to create collection: {}", e)) - })?; - } + // Use the shared tuned options so collections get LZ4 compression, + // the shared block cache, and bloom filters (Options::default() + // would silently skip all of that) + self.db + .create_cf(&cf_name, &tuned_cf_options()) + .map_err(|e| { + DbError::InternalError(format!("Failed to create collection: {}", e)) + })?; } // Persist collection type (lock-free, thread-safe) if let Some(cf) = self.db.cf_handle(&cf_name) { self.db - .put_cf(cf, "_stats:type".as_bytes(), type_.as_bytes()) + .put_cf(&cf, "_stats:type".as_bytes(), type_.as_bytes()) .map_err(|e| { DbError::InternalError(format!("Failed to set collection type: {}", e)) })?; @@ -94,16 +95,10 @@ impl Database { return Err(DbError::CollectionNotFound(collection_name.to_string())); } - // Drop column family - requires exclusive lock - { - let _cf_guard = self.cf_lock.write().unwrap(); - let db_ptr = Arc::as_ptr(&self.db) as *mut DB; - unsafe { - (*db_ptr).drop_cf(&cf_name).map_err(|e| { - DbError::InternalError(format!("Failed to delete collection: {}", e)) - })?; - } - } + // MultiThreaded mode: drop_cf takes &self and synchronizes internally + self.db + .drop_cf(&cf_name) + .map_err(|e| DbError::InternalError(format!("Failed to delete collection: {}", e)))?; // Remove from cache self.collections.remove(collection_name); @@ -115,13 +110,12 @@ impl Database { pub fn list_collections(&self) -> Vec { let prefix = format!("{}:", self.name); - // Iterate through all column families (lock-free, DB::list_cf is thread-safe) + // Use the live in-memory CF list — DB::list_cf would re-read the + // MANIFEST from disk on every call let mut collections = Vec::new(); - for cf_name in DB::list_cf(&Options::default(), self.db.path()).unwrap_or_default() { - if cf_name.starts_with(&prefix) { - if let Some(name) = cf_name.strip_prefix(&prefix) { - collections.push(name.to_string()); - } + for cf_name in self.db.cf_names() { + if let Some(name) = cf_name.strip_prefix(&prefix) { + collections.push(name.to_string()); } } collections diff --git a/src/storage/engine.rs b/src/storage/engine.rs index d91caf2f..186cd666 100644 --- a/src/storage/engine.rs +++ b/src/storage/engine.rs @@ -1,10 +1,10 @@ use dashmap::DashMap; -use rust_rocksdb::{ - BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBCompressionType, Options, DB, -}; +use rust_rocksdb::{BlockBasedOptions, Cache, ColumnFamilyDescriptor, DBCompressionType, Options}; + +use super::RocksDb as DB; use std::collections::HashMap; use std::path::Path; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, OnceLock, RwLock}; use super::collection::Collection; use super::database::Database; @@ -15,6 +15,46 @@ use crate::transaction::manager::TransactionManager; /// Metadata column family name const META_CF: &str = "_meta"; +/// Shared block cache used by all column families (and all DB instances). +/// Without an explicit table factory, each CF gets its own private default +/// cache and no bloom filter — with thousands of CFs that wastes memory and +/// bypasses the cache/bloom tuning entirely. +fn shared_block_cache() -> &'static Cache { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| Cache::new_lru_cache(512 * 1024 * 1024)) +} + +/// Create optimized column family options +/// Used for ALL column families — including those created via `Database` — +/// to ensure consistent compression, caching, and performance settings +pub(crate) fn tuned_cf_options() -> Options { + let mut opts = Options::default(); + + // Enable LZ4 compression for this column family + opts.set_compression_type(DBCompressionType::Lz4); + + // Level compaction is the default and works well for most workloads + // Optimize for SSD storage with fast sequential I/O + opts.set_target_file_size_base(64 * 1024 * 1024); // 64MB base file size + opts.set_target_file_size_multiplier(2); + + // Write buffer settings + opts.set_write_buffer_size(64 * 1024 * 1024); // 64MB memtable + opts.set_max_write_buffer_number(3); + opts.set_min_write_buffer_number_to_merge(1); + + // Optimize for SSD storage - parallel compactions + opts.set_max_subcompactions(4); + + // Shared block cache + bloom filter for faster point lookups + let mut block_opts = BlockBasedOptions::default(); + block_opts.set_block_cache(shared_block_cache()); + block_opts.set_bloom_filter(10.0, false); + opts.set_block_based_table_factory(&block_opts); + + opts +} + /// The main storage engine backed by RocksDB /// /// Uses lock-free reads - RocksDB's DB type is thread-safe for concurrent reads. @@ -62,30 +102,6 @@ impl std::fmt::Debug for StorageEngine { } impl StorageEngine { - /// Create optimized column family options - /// Used for all column families to ensure consistent compression and performance settings - fn create_cf_options() -> Options { - let mut opts = Options::default(); - - // Enable LZ4 compression for this column family - opts.set_compression_type(DBCompressionType::Lz4); - - // Level compaction is the default and works well for most workloads - // Optimize for SSD storage with fast sequential I/O - opts.set_target_file_size_base(64 * 1024 * 1024); // 64MB base file size - opts.set_target_file_size_multiplier(2); - - // Write buffer settings - opts.set_write_buffer_size(64 * 1024 * 1024); // 64MB memtable - opts.set_max_write_buffer_number(3); - opts.set_min_write_buffer_number_to_merge(1); - - // Optimize for SSD storage - parallel compactions - opts.set_max_subcompactions(4); - - opts - } - /// Create a new storage engine pub fn new>(data_dir: P) -> DbResult { let path = data_dir.as_ref().to_path_buf(); @@ -100,11 +116,10 @@ impl StorageEngine { opts.set_compression_type(DBCompressionType::Lz4); opts.set_compression_options(-14, -1, 0, 0); - // Block cache - use 512MB or 30% of available memory + // Block cache - shared 512MB cache across all CFs and DB instances // Improves read performance by caching frequently accessed blocks - let cache = Cache::new_lru_cache(512 * 1024 * 1024); let mut block_opts = BlockBasedOptions::default(); - block_opts.set_block_cache(&cache); + block_opts.set_block_cache(shared_block_cache()); // Enable bloom filter for faster point lookups block_opts.set_bloom_filter(10.0, false); opts.set_block_based_table_factory(&block_opts); @@ -158,7 +173,7 @@ impl StorageEngine { // All column families inherit compression and performance settings let cf_descriptors: Vec = cf_names .iter() - .map(|name| ColumnFamilyDescriptor::new(name, Self::create_cf_options())) + .map(|name| ColumnFamilyDescriptor::new(name, tuned_cf_options())) .collect(); // Open database with column families @@ -306,7 +321,7 @@ impl StorageEngine { let meta_cf = self.db.cf_handle(META_CF).expect("META_CF should exist"); let db_key = format!("db:{}", name); self.db - .put_cf(meta_cf, db_key.as_bytes(), b"1") + .put_cf(&meta_cf, db_key.as_bytes(), b"1") .map_err(|e| DbError::InternalError(format!("Failed to create database: {}", e)))?; Ok(()) @@ -334,7 +349,7 @@ impl StorageEngine { let meta_cf = self.db.cf_handle(META_CF).expect("META_CF should exist"); let db_key = format!("db:{}", name); self.db - .delete_cf(meta_cf, db_key.as_bytes()) + .delete_cf(&meta_cf, db_key.as_bytes()) .map_err(|e| DbError::InternalError(format!("Failed to delete database: {}", e)))?; // Remove from cache @@ -352,7 +367,7 @@ impl StorageEngine { }; let prefix = b"db:"; - let iter = self.db.prefix_iterator_cf(meta_cf, prefix); + let iter = self.db.prefix_iterator_cf(&meta_cf, prefix); iter.filter_map(|result| { result.ok().and_then(|(key, _)| { @@ -394,7 +409,7 @@ impl StorageEngine { let type_ = collection_type.unwrap_or_else(|| "document".to_string()); // Create the column family - requires exclusive lock - let opts = Self::create_cf_options(); + let opts = tuned_cf_options(); { let _cf_guard = self.cf_lock.write().unwrap(); @@ -404,20 +419,16 @@ impl StorageEngine { return Err(DbError::CollectionAlreadyExists(name)); } - // Safety: We use cf_lock to ensure exclusive access for CF operations - // This is a safe workaround for RocksDB's create_cf requiring &mut self - let db_ptr = Arc::as_ptr(&self.db) as *mut DB; - unsafe { - (*db_ptr).create_cf(&name, &opts).map_err(|e| { - DbError::InternalError(format!("Failed to create collection: {}", e)) - })?; - } + // MultiThreaded mode: create_cf takes &self and synchronizes internally + self.db.create_cf(&name, &opts).map_err(|e| { + DbError::InternalError(format!("Failed to create collection: {}", e)) + })?; } // Persist collection type (lock-free, thread-safe) if let Some(cf) = self.db.cf_handle(&name) { self.db - .put_cf(cf, "_stats:type".as_bytes(), type_.as_bytes()) + .put_cf(&cf, "_stats:type".as_bytes(), type_.as_bytes()) .map_err(|e| { DbError::InternalError(format!("Failed to set collection type: {}", e)) })?; @@ -465,26 +476,20 @@ impl StorageEngine { return Err(DbError::CollectionNotFound(name.to_string())); } - // Drop the column family - requires exclusive lock - { - let _cf_guard = self.cf_lock.write().unwrap(); - // Safety: We use cf_lock to ensure exclusive access for CF operations - let db_ptr = Arc::as_ptr(&self.db) as *mut DB; - unsafe { - (*db_ptr).drop_cf(name).map_err(|e| { - DbError::InternalError(format!("Failed to delete collection: {}", e)) - })?; - } - } + // MultiThreaded mode: drop_cf takes &self and synchronizes internally + self.db + .drop_cf(name) + .map_err(|e| DbError::InternalError(format!("Failed to delete collection: {}", e)))?; Ok(()) } /// List all collection names pub fn list_collections(&self) -> Vec { - // Get all column family names, excluding internal ones - DB::list_cf(&Options::default(), &self.path) - .unwrap_or_default() + // Use the live in-memory CF list — DB::list_cf would re-read the + // MANIFEST from disk on every call + self.db + .cf_names() .into_iter() .filter(|name| name != "default" && name != META_CF) .collect() diff --git a/src/storage/mod.rs b/src/storage/mod.rs index c19e34db..3ff9bae8 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,3 +1,11 @@ +/// RocksDB handle type used throughout the codebase. +/// +/// `MultiThreaded` mode synchronizes the column-family handle map internally +/// (`create_cf`/`drop_cf` take `&self`), so CF creation/drop no longer needs +/// `unsafe` mutable casts of `Arc` — which raced against the lock-free +/// `cf_handle()` reads done all over the codebase. +pub type RocksDb = rust_rocksdb::DBWithThreadMode; + pub mod codec; pub mod collection; pub mod columnar; diff --git a/src/sync/log.rs b/src/sync/log.rs index c9ef9e4d..8af54106 100644 --- a/src/sync/log.rs +++ b/src/sync/log.rs @@ -6,7 +6,8 @@ use std::collections::VecDeque; use std::sync::{Arc, RwLock}; -use rust_rocksdb::{Direction, IteratorMode, Options, WriteBatch, DB}; +use crate::storage::RocksDb as DB; +use rust_rocksdb::{Direction, IteratorMode, Options, WriteBatch}; use serde::{Deserialize, Serialize}; use super::protocol::{Operation, SyncEntry}; From ac967517d34d954b0f0a502df2142a80bb222cd7 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 5 Jun 2026 09:18:44 +0200 Subject: [PATCH 07/12] perf(storage): make database drop instant via background column-family drops Every drop_cf rewrites + fsyncs the entire OPTIONS file (one section per CF) under the DB mutex, so delete_database blocked for collections x ~400ms on instances with many CFs (measured: 18s for a 25-collection database at ~1800 CFs). delete_database now atomically deletes the db:{name} meta key and persists a pending_drop:{cf} marker per collection in one WriteBatch, then returns (~1ms); a background thread performs the expensive drops, paced 25ms apart so foreground CF ops don't starve behind the queue. Markers are resumed by initialize() so drops interrupted by a crash or restart complete eventually. Recreating the same database/collection while its old CF is still doomed is handled by claiming: create_collection atomically takes ownership of a Pending CF and drops + recreates it fresh, or waits if the dropper is mid-drop on that exact CF. Doomed CFs are filtered from list_collections, get_collection, delete_collection, and is_columnar_collection. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/storage/database.rs | 64 ++++++++-- src/storage/engine.rs | 66 +++++++--- src/storage/mod.rs | 1 + src/storage/pending_drops.rs | 224 ++++++++++++++++++++++++++++++++++ tests/storage_engine_tests.rs | 43 +++++++ 5 files changed, 373 insertions(+), 25 deletions(-) create mode 100644 src/storage/pending_drops.rs diff --git a/src/storage/database.rs b/src/storage/database.rs index b02aa9fc..c042d2ac 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -1,10 +1,12 @@ use super::RocksDb as DB; use dashmap::DashMap; use std::sync::{Arc, RwLock}; +use std::time::Duration; use super::collection::Collection; use super::columnar::*; use super::engine::tuned_cf_options; +use super::pending_drops::{Claim, PendingCfDrops}; use crate::error::{DbError, DbResult}; use serde_json::Value; @@ -20,6 +22,8 @@ pub struct Database { cf_lock: Arc>, /// Cached collection handles (DashMap for lock-free concurrent access) collections: Arc>, + /// Column families scheduled for background drop — treated as deleted + pending_cf_drops: Arc, } impl std::fmt::Debug for Database { @@ -32,12 +36,13 @@ impl std::fmt::Debug for Database { impl Database { /// Create a new database handle - pub fn new(name: String, db: Arc) -> Self { + pub fn new(name: String, db: Arc, pending_cf_drops: Arc) -> Self { Self { name, db, cf_lock: Arc::new(RwLock::new(())), collections: Arc::new(DashMap::new()), + pending_cf_drops, } } @@ -58,10 +63,35 @@ impl Database { { let _cf_guard = self.cf_lock.write().unwrap(); - // Check inside lock to avoid TOCTOU race when multiple threads - // try to create the same collection concurrently - if self.db.cf_handle(&cf_name).is_some() { - return Err(DbError::CollectionAlreadyExists(collection_name)); + // The CF may be a leftover from a dropped database still awaiting + // its background drop — claim it and recreate fresh instead of + // failing with "already exists". + match self.pending_cf_drops.claim_for_recreate(&cf_name) { + Claim::Claimed => { + if self.db.cf_handle(&cf_name).is_some() { + if let Err(e) = self.db.drop_cf(&cf_name) { + self.pending_cf_drops.release_claim(&cf_name); + return Err(DbError::InternalError(format!( + "Failed to reclaim pending collection: {}", + e + ))); + } + } + self.pending_cf_drops.complete(&self.db, &cf_name); + } + Claim::InProgress => { + // The background dropper is dropping this exact CF right + // now — wait for it to finish, then create fresh below. + self.pending_cf_drops + .wait_until_dropped(&cf_name, Duration::from_secs(30))?; + } + Claim::NotPending => { + // Check inside lock to avoid TOCTOU race when multiple + // threads try to create the same collection concurrently + if self.db.cf_handle(&cf_name).is_some() { + return Err(DbError::CollectionAlreadyExists(collection_name)); + } + } } // Use the shared tuned options so collections get LZ4 compression, @@ -90,6 +120,11 @@ impl Database { pub fn delete_collection(&self, collection_name: &str) -> DbResult<()> { let cf_name = self.collection_cf_name(collection_name); + // Already scheduled for background drop — logically gone + if self.pending_cf_drops.contains(&cf_name) { + return Err(DbError::CollectionNotFound(collection_name.to_string())); + } + // Check if collection exists (lock-free read) if self.db.cf_handle(&cf_name).is_none() { return Err(DbError::CollectionNotFound(collection_name.to_string())); @@ -114,6 +149,10 @@ impl Database { // MANIFEST from disk on every call let mut collections = Vec::new(); for cf_name in self.db.cf_names() { + // Skip CFs awaiting their background drop — logically deleted + if self.pending_cf_drops.contains(&cf_name) { + continue; + } if let Some(name) = cf_name.strip_prefix(&prefix) { collections.push(name.to_string()); } @@ -130,6 +169,11 @@ impl Database { let cf_name = self.collection_cf_name(collection_name); + // A CF awaiting its background drop is logically deleted + if self.pending_cf_drops.contains(&cf_name) { + return Err(DbError::CollectionNotFound(collection_name.to_string())); + } + // Check if collection exists (lock-free read) if self.db.cf_handle(&cf_name).is_none() { return Err(DbError::CollectionNotFound(collection_name.to_string())); @@ -336,7 +380,7 @@ impl Database { /// Check if a collection is a columnar collection pub fn is_columnar_collection(&self, collection_name: &str) -> bool { let cf_name = self.columnar_cf_name(collection_name); - self.db.cf_handle(&cf_name).is_some() + self.db.cf_handle(&cf_name).is_some() && !self.pending_cf_drops.contains(&cf_name) } /// List all columnar collections in this database @@ -369,7 +413,7 @@ mod tests { #[test] fn test_create_collection() { let (db, _dir) = create_test_db(); - let database = Database::new("testdb".to_string(), db); + let database = Database::new("testdb".to_string(), db, PendingCfDrops::new()); assert!(database .create_collection("users".to_string(), None) @@ -380,7 +424,7 @@ mod tests { #[test] fn test_create_duplicate_collection() { let (db, _dir) = create_test_db(); - let database = Database::new("testdb".to_string(), db); + let database = Database::new("testdb".to_string(), db, PendingCfDrops::new()); database .create_collection("users".to_string(), None) @@ -393,7 +437,7 @@ mod tests { #[test] fn test_delete_collection() { let (db, _dir) = create_test_db(); - let database = Database::new("testdb".to_string(), db); + let database = Database::new("testdb".to_string(), db, PendingCfDrops::new()); database .create_collection("users".to_string(), None) @@ -405,7 +449,7 @@ mod tests { #[test] fn test_list_collections() { let (db, _dir) = create_test_db(); - let database = Database::new("testdb".to_string(), db); + let database = Database::new("testdb".to_string(), db, PendingCfDrops::new()); database .create_collection("users".to_string(), None) diff --git a/src/storage/engine.rs b/src/storage/engine.rs index 186cd666..0ea6e2df 100644 --- a/src/storage/engine.rs +++ b/src/storage/engine.rs @@ -8,12 +8,13 @@ use std::sync::{Arc, OnceLock, RwLock}; use super::collection::Collection; use super::database::Database; +use super::pending_drops::PendingCfDrops; use crate::cluster::ClusterConfig; use crate::error::{DbError, DbResult}; use crate::transaction::manager::TransactionManager; /// Metadata column family name -const META_CF: &str = "_meta"; +pub(crate) const META_CF: &str = "_meta"; /// Shared block cache used by all column families (and all DB instances). /// Without an explicit table factory, each CF gets its own private default @@ -75,6 +76,8 @@ pub struct StorageEngine { cluster_config: Option, /// Transaction manager (optionally initialized, uses RwLock for interior mutability) transaction_manager: RwLock>>, + /// Column families scheduled for background drop (see `pending_drops`) + pending_cf_drops: Arc, } impl Clone for StorageEngine { @@ -89,6 +92,7 @@ impl Clone for StorageEngine { transaction_manager: RwLock::new( self.transaction_manager.read().ok().and_then(|t| t.clone()), ), + pending_cf_drops: self.pending_cf_drops.clone(), } } } @@ -188,6 +192,7 @@ impl StorageEngine { databases: Arc::new(DashMap::new()), cluster_config: None, transaction_manager: RwLock::new(None), + pending_cf_drops: PendingCfDrops::new(), }) } @@ -247,6 +252,16 @@ impl StorageEngine { // This ensures counts are accurate after crashes or unclean shutdowns self.recalculate_all_counts(); + // Resume column-family drops interrupted by a previous shutdown/crash + let resumed = self.pending_cf_drops.resume_from_meta(&self.db); + if !resumed.is_empty() { + tracing::info!( + "Resuming {} interrupted column-family drops in the background", + resumed.len() + ); + PendingCfDrops::spawn_dropper(self.db.clone(), self.pending_cf_drops.clone(), resumed); + } + Ok(()) } @@ -327,7 +342,14 @@ impl StorageEngine { Ok(()) } - /// Delete a database and all its collections + /// Delete a database and all its collections. + /// + /// The database is removed from metadata immediately; the per-collection + /// column-family drops run on a background thread. Each `drop_cf` + /// rewrites + fsyncs the entire OPTIONS file (one section per CF), so on + /// an instance with many CFs dropping a database inline would block the + /// request for `collections × hundreds-of-ms` (measured: 18s for 25 + /// collections at ~1800 CFs). See `storage::pending_drops`. pub fn delete_database(&self, name: &str) -> DbResult<()> { // Prevent deletion of _system database if name == "_system" { @@ -336,25 +358,35 @@ impl StorageEngine { )); } - // Get database to ensure it exists - let database = self.get_database(name)?; - - // Delete all collections in the database - let collections = database.list_collections(); - for collection_name in collections { - database.delete_collection(&collection_name)?; + // Ensure the database exists + if !self.list_databases().contains(&name.to_string()) { + return Err(DbError::CollectionNotFound(format!( + "Database '{}' not found", + name + ))); } - // Remove database metadata (RocksDB is thread-safe for writes) - let meta_cf = self.db.cf_handle(META_CF).expect("META_CF should exist"); + // Every CF belonging to this database (document + columnar), minus + // any already scheduled by a previous drop of the same name + let prefix = format!("{}:", name); + let doomed: Vec = self + .db + .cf_names() + .into_iter() + .filter(|cf| cf.starts_with(&prefix) && !self.pending_cf_drops.contains(cf)) + .collect(); + + // Atomically delete the `db:{name}` metadata key and persist a + // `pending_drop:` marker per CF, then drop the CFs in the background. + // Markers survive a crash and are resumed by `initialize`. let db_key = format!("db:{}", name); - self.db - .delete_cf(&meta_cf, db_key.as_bytes()) - .map_err(|e| DbError::InternalError(format!("Failed to delete database: {}", e)))?; + self.pending_cf_drops.schedule(&self.db, &db_key, &doomed)?; // Remove from cache self.databases.remove(name); + PendingCfDrops::spawn_dropper(self.db.clone(), self.pending_cf_drops.clone(), doomed); + Ok(()) } @@ -395,7 +427,11 @@ impl StorageEngine { } // Create and cache the database - let database = Database::new(name.to_string(), self.db.clone()); + let database = Database::new( + name.to_string(), + self.db.clone(), + self.pending_cf_drops.clone(), + ); self.databases.insert(name.to_string(), database.clone()); Ok(database) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 3ff9bae8..9cb01c4e 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -16,6 +16,7 @@ pub mod engine; pub mod geo; pub mod http_client; pub mod index; +pub mod pending_drops; pub mod query_cache; pub mod schema; pub mod serializer; diff --git a/src/storage/pending_drops.rs b/src/storage/pending_drops.rs new file mode 100644 index 00000000..9447e523 --- /dev/null +++ b/src/storage/pending_drops.rs @@ -0,0 +1,224 @@ +//! Pending column-family drop registry. +//! +//! Dropping a RocksDB column family rewrites + fsyncs the entire OPTIONS +//! file (one section per CF) under the DB mutex, so with thousands of CFs +//! each `drop_cf` costs hundreds of milliseconds. `delete_database` would +//! otherwise block for `collections × that cost` (measured: 18s for a +//! 25-collection database on a 1794-CF instance). +//! +//! Instead, `delete_database` removes the database from `_meta` immediately +//! (making it invisible to all metadata-driven paths) and schedules the CF +//! drops here; a background thread performs the expensive drops while the +//! request returns instantly. +//! +//! Each scheduled drop is persisted as a `pending_drop:{cf}` marker in the +//! `_meta` CF — written in the same atomic batch that deletes the `db:{name}` +//! key — so drops interrupted by a crash or restart are resumed on startup. +//! +//! Recreating a database/collection with the same name while its old CF is +//! still doomed is handled by *claiming*: the creator atomically takes +//! ownership of a `Pending` CF, drops it synchronously, and recreates it +//! fresh. If the background dropper is mid-`drop_cf` on that exact CF +//! (`Dropping`), the creator waits for it to finish instead. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use dashmap::DashMap; +use rust_rocksdb::WriteBatch; + +use super::engine::META_CF; +use super::RocksDb as DB; +use crate::error::{DbError, DbResult}; + +/// `_meta` key prefix for persisted drop markers. +const MARKER_PREFIX: &str = "pending_drop:"; + +#[derive(Clone, Copy, PartialEq)] +enum DropState { + /// Scheduled, not yet started — a recreate may claim it. + Pending, + /// `drop_cf` is executing right now (by the dropper or a claimant). + Dropping, +} + +/// Outcome of [`PendingCfDrops::claim_for_recreate`]. +pub enum Claim { + /// Caller now owns the doomed CF: drop it, then call `complete`. + Claimed, + /// The background dropper is mid-drop on this CF — wait for it. + InProgress, + /// The CF is not scheduled for drop. + NotPending, +} + +/// In-memory registry of CFs awaiting a background drop, backed by +/// persisted `pending_drop:{cf}` markers in `_meta` for crash recovery. +#[derive(Default)] +pub struct PendingCfDrops { + states: DashMap, +} + +impl PendingCfDrops { + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// True if this CF is scheduled (or mid-drop) — callers must treat such + /// collections as already deleted. + pub fn contains(&self, cf_name: &str) -> bool { + self.states.contains_key(cf_name) + } + + /// Atomically persist drop markers for `cfs` and delete the database's + /// `db:{name}` metadata key (`db_meta_key`) in one write batch, then + /// register the CFs in memory. + pub fn schedule(&self, db: &DB, db_meta_key: &str, cfs: &[String]) -> DbResult<()> { + let meta_cf = db + .cf_handle(META_CF) + .ok_or_else(|| DbError::InternalError("_meta column family missing".to_string()))?; + + let mut batch = WriteBatch::default(); + batch.delete_cf(&meta_cf, db_meta_key.as_bytes()); + for cf in cfs { + batch.put_cf( + &meta_cf, + format!("{}{}", MARKER_PREFIX, cf).as_bytes(), + b"1", + ); + } + db.write(&batch).map_err(|e| { + DbError::InternalError(format!("Failed to schedule collection drops: {}", e)) + })?; + + for cf in cfs { + self.states.insert(cf.clone(), DropState::Pending); + } + Ok(()) + } + + /// Load persisted markers left by a previous run (crash / restart during + /// a background drop) and re-register them. Returns the CFs to drop. + pub fn resume_from_meta(&self, db: &DB) -> Vec { + let meta_cf = match db.cf_handle(META_CF) { + Some(cf) => cf, + None => return vec![], + }; + + let iter = db.prefix_iterator_cf(&meta_cf, MARKER_PREFIX.as_bytes()); + let cfs: Vec = iter + .filter_map(|result| { + result.ok().and_then(|(key, _)| { + let key_str = String::from_utf8(key.to_vec()).ok()?; + key_str.strip_prefix(MARKER_PREFIX).map(|s| s.to_string()) + }) + }) + .collect(); + + for cf in &cfs { + self.states.insert(cf.clone(), DropState::Pending); + } + cfs + } + + /// Attempt to take ownership of a doomed CF so it can be dropped + /// synchronously and recreated fresh (collection re-created with the + /// same name before the background drop got to it). + pub fn claim_for_recreate(&self, cf_name: &str) -> Claim { + use dashmap::mapref::entry::Entry; + match self.states.entry(cf_name.to_string()) { + Entry::Occupied(mut entry) => match entry.get() { + DropState::Pending => { + *entry.get_mut() = DropState::Dropping; + Claim::Claimed + } + DropState::Dropping => Claim::InProgress, + }, + Entry::Vacant(_) => Claim::NotPending, + } + } + + /// Hand a claimed CF back (claimant failed to drop it) so the background + /// dropper or a restart retries it. + pub fn release_claim(&self, cf_name: &str) { + self.states.insert(cf_name.to_string(), DropState::Pending); + } + + /// Mark the drop finished: delete the persisted marker and forget the CF. + pub fn complete(&self, db: &DB, cf_name: &str) { + if let Some(meta_cf) = db.cf_handle(META_CF) { + let _ = db.delete_cf(&meta_cf, format!("{}{}", MARKER_PREFIX, cf_name).as_bytes()); + } + self.states.remove(cf_name); + } + + /// Block until the background dropper finishes this CF (used when a + /// recreate races the in-flight `drop_cf` of the same name). + pub fn wait_until_dropped(&self, cf_name: &str, timeout: Duration) -> DbResult<()> { + let start = Instant::now(); + while self.states.contains_key(cf_name) { + if start.elapsed() > timeout { + return Err(DbError::InternalError(format!( + "Timed out waiting for pending drop of collection '{}'", + cf_name + ))); + } + std::thread::sleep(Duration::from_millis(10)); + } + Ok(()) + } + + /// Dropper-side claim: only proceeds on CFs still `Pending` (a recreate + /// may have claimed them in the meantime). + fn begin_drop(&self, cf_name: &str) -> bool { + use dashmap::mapref::entry::Entry; + match self.states.entry(cf_name.to_string()) { + Entry::Occupied(mut entry) if *entry.get() == DropState::Pending => { + *entry.get_mut() = DropState::Dropping; + true + } + _ => false, + } + } + + /// Drop the scheduled CFs on a background thread. Each successful drop + /// removes its persisted marker; failed drops stay marked so they are + /// retried on the next startup. + pub fn spawn_dropper(db: Arc, registry: Arc, cfs: Vec) { + if cfs.is_empty() { + return; + } + std::thread::spawn(move || { + let start = Instant::now(); + let total = cfs.len(); + let mut dropped = 0usize; + for (i, cf) in cfs.iter().enumerate() { + // Breathe between drops: each drop_cf holds the DB mutex for + // its full OPTIONS rewrite, and back-to-back drops starve + // concurrent foreground CF ops (an immediate recreate of the + // same database would otherwise wait for the whole queue). + if i > 0 { + std::thread::sleep(Duration::from_millis(25)); + } + if !registry.begin_drop(cf) { + continue; // claimed by a concurrent recreate + } + if db.cf_handle(cf).is_some() { + if let Err(e) = db.drop_cf(cf) { + tracing::warn!("Background drop of column family '{}' failed: {}", cf, e); + registry.release_claim(cf); + continue; + } + dropped += 1; + } + registry.complete(&db, cf); + } + tracing::info!( + "Background-dropped {}/{} column families in {:.2?}", + dropped, + total, + start.elapsed() + ); + }); + } +} diff --git a/tests/storage_engine_tests.rs b/tests/storage_engine_tests.rs index c7b006fe..ac8c2dc4 100644 --- a/tests/storage_engine_tests.rs +++ b/tests/storage_engine_tests.rs @@ -146,6 +146,49 @@ fn test_delete_database_with_collections() { assert!(engine.get_database("dbwithcol").is_err()); } +#[test] +fn test_recreate_database_after_delete() { + let (engine, _tmp) = create_test_engine(); + + engine.create_database("recreated".to_string()).unwrap(); + let db = engine.get_database("recreated").unwrap(); + db.create_collection("users".to_string(), None).unwrap(); + let users = db.get_collection("users").unwrap(); + users + .insert(json!({"_key": "alice", "name": "Alice"})) + .unwrap(); + + // Drop and immediately recreate the same database + collection. The old + // CF may still be awaiting its background drop — the recreate must claim + // it and come back empty, not fail with "already exists" or show stale + // documents. + engine.delete_database("recreated").unwrap(); + assert!(engine.get_database("recreated").is_err()); + + engine.create_database("recreated".to_string()).unwrap(); + let db = engine.get_database("recreated").unwrap(); + + // The doomed collection must not be visible on the fresh database + assert!(!db.list_collections().contains(&"users".to_string())); + assert!(db.get_collection("users").is_err()); + + db.create_collection("users".to_string(), None).unwrap(); + let users = db.get_collection("users").unwrap(); + assert_eq!(users.count(), 0, "reclaimed collection must be empty"); +} + +#[test] +fn test_delete_database_twice_returns_not_found() { + let (engine, _tmp) = create_test_engine(); + + engine.create_database("twice".to_string()).unwrap(); + engine.delete_database("twice").unwrap(); + + // The logical delete is immediate even though CF drops run in the + // background — a second delete must already see the database as gone. + assert!(engine.delete_database("twice").is_err()); +} + // ============================================================================ // Collection Operations Tests // ============================================================================ From 164be4116fc68fb26633d750b19ed353d24badff Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 5 Jun 2026 09:20:58 +0200 Subject: [PATCH 08/12] chore: release v0.26.2 Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae10e071..42ccbea7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4414,7 +4414,7 @@ dependencies = [ [[package]] name = "solidb" -version = "0.26.1" +version = "0.26.2" dependencies = [ "anyhow", "argon2", diff --git a/Cargo.toml b/Cargo.toml index b3283459..c8a732b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["clients/rust-client", "sdbql-core", "benchmarks"] [package] name = "solidb" -version = "0.26.1" +version = "0.26.2" edition = "2021" default-run = "solidb" description = "A lightweight, high-performance structured database server written in Rust." From 47a440d0a8a0f0d62cab5ffd9ce07ecebb33be36 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 5 Jun 2026 11:39:42 +0200 Subject: [PATCH 09/12] fix(tests): remove UB single-threaded DB cast in explain_columnar_test The test cast Database::db_arc()'s Arc pointer to the single-threaded rust_rocksdb::DB type and called create_cf through it. Since the switch to DBWithThreadMode (7aa803c) the layouts differ, so the unsafe mutation was type confusion and segfaulted in CI (SIGSEGV). MultiThreaded create_cf takes &self, so the unsafe block is unnecessary: call it directly on the Arc. Also drop the stale single-threaded create_test_db helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/explain_columnar_test.rs | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/tests/explain_columnar_test.rs b/tests/explain_columnar_test.rs index 376dce0b..3d87f74f 100644 --- a/tests/explain_columnar_test.rs +++ b/tests/explain_columnar_test.rs @@ -1,36 +1,21 @@ use solidb::storage::columnar::{ColumnDef, ColumnType, ColumnarCollection, CompressionType}; use rust_rocksdb::Options; -use rust_rocksdb::DB; use solidb::sdbql::{parse, QueryExecutor}; use solidb::storage::engine::StorageEngine; -use std::sync::{Arc, RwLock}; use tempfile::TempDir; -#[allow(dead_code)] -fn create_test_db() -> (Arc>, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let db = DB::open_default(temp_dir.path()).unwrap(); - (Arc::new(RwLock::new(db)), temp_dir) -} - #[test] fn test_explain_columnar_query() { let _dir = TempDir::new().unwrap(); // Create StorageEngine wrapper (this opens the DB) let storage = StorageEngine::new(_dir.path()).unwrap(); - // database.db_arc() will provide the handle - - // Check if StorageEngine has a method to get the DB Arc. - // Step 393 showed `db: Arc>` is a field, but no public getter was visible in the snippet. - // However, Database struct has `db_arc()`. - // StorageEngine usually has `get_database`. - // Let's create the database first + // Create the database first storage.create_database("testdb".to_string()).unwrap(); let database = storage.get_database("testdb").unwrap(); - let db_arc = database.db_arc(); // valid way to get it + let db_arc = database.db_arc(); // Create a regular collection first (required for executor to find the collection) // The collection name must match the name used in the query ("metrics") @@ -38,16 +23,12 @@ fn test_explain_columnar_query() { .create_collection("metrics".to_string(), None) .unwrap(); - // Manually create the column family for the columnar collection - // Using unsafe pattern similar to Database::create_collection + // Manually create the column family for the columnar collection. + // MultiThreaded mode: create_cf takes &self and synchronizes internally. { let cf_name = "testdb:_columnar_metrics"; - // Check if exists first to be safe, though unexpected in fresh db if db_arc.cf_handle(cf_name).is_none() { - let db_ptr = Arc::as_ptr(&db_arc) as *mut DB; - unsafe { - (*db_ptr).create_cf(cf_name, &Options::default()).unwrap(); - } + db_arc.create_cf(cf_name, &Options::default()).unwrap(); } } From fbdbf8bc9864840e07c6f0e1a5ebbc8f6e7169f6 Mon Sep 17 00:00:00 2001 From: Olivier Bonnaure Date: Fri, 5 Jun 2026 11:39:48 +0200 Subject: [PATCH 10/12] feat(admin): add Soli admin app, task tickets, and rebuilt layouts.css Embed the admin/ Soli application (controllers, views, specs) into the main repository, add the tasks/ ticket workflow directory, and include a TailwindCSS rebuild of www/public/layouts.css. Co-Authored-By: Claude Opus 4.8 (1M context) --- admin/.env.test | 22 + admin/.gitignore | 34 + admin/AGENTS.md | 5 + admin/CLAUDE.md | 513 ++ admin/README.md | 92 + admin/app/assets/css/application.css | 112 + admin/app/controllers/CLAUDE.md | 527 ++ admin/app/controllers/api_keys_controller.sl | 75 + .../app/controllers/collections_controller.sl | 247 + admin/app/controllers/cron_controller.sl | 106 + admin/app/controllers/databases_controller.sl | 59 + admin/app/controllers/documents_controller.sl | 191 + admin/app/controllers/env_vars_controller.sl | 74 + admin/app/controllers/home_controller.sl | 25 + .../controllers/live_queries_controller.sl | 41 + .../materialized_views_controller.sl | 124 + admin/app/controllers/query_controller.sl | 124 + admin/app/controllers/queues_controller.sl | 108 + admin/app/controllers/roles_controller.sl | 102 + admin/app/controllers/scripts_controller.sl | 127 + admin/app/controllers/triggers_controller.sl | 119 + admin/app/controllers/users_controller.sl | 88 + admin/app/helpers/application_helper.sl | 82 + admin/app/helpers/solidb_helper.sl | 83 + admin/app/middleware/CLAUDE.md | 259 + admin/app/middleware/cors.sl | 24 + admin/app/models/CLAUDE.md | 460 ++ admin/app/services/admin_context.sl | 14 + admin/app/services/solidb_client.sl | 172 + admin/app/services/solidb_endpoints.sl | 192 + admin/app/views/CLAUDE.md | 409 ++ admin/app/views/api_keys/index.html.slv | 91 + admin/app/views/collections/_indexes.html.slv | 178 + admin/app/views/collections/_stats.html.slv | 45 + admin/app/views/collections/index.html.slv | 218 + admin/app/views/cron/index.html.slv | 119 + admin/app/views/databases/index.html.slv | 65 + admin/app/views/documents/index.html.slv | 217 + admin/app/views/env_vars/index.html.slv | 96 + admin/app/views/home/index.html.slv | 78 + admin/app/views/layouts/application.html.slv | 45 + admin/app/views/live_queries/show.html.slv | 185 + .../views/materialized_views/index.html.slv | 118 + admin/app/views/query/_explain.html.slv | 150 + admin/app/views/query/_results.html.slv | 34 + admin/app/views/query/_slow_badge.html.slv | 3 + admin/app/views/query/show.html.slv | 155 + admin/app/views/query/slow.html.slv | 61 + admin/app/views/queues/_jobs.html.slv | 63 + admin/app/views/queues/index.html.slv | 106 + admin/app/views/roles/index.html.slv | 88 + admin/app/views/roles/show.html.slv | 58 + admin/app/views/scripts/_form.html.slv | 68 + admin/app/views/scripts/edit.html.slv | 15 + admin/app/views/scripts/index.html.slv | 64 + admin/app/views/scripts/new.html.slv | 14 + admin/app/views/scripts/show.html.slv | 25 + admin/app/views/shared/_banners.html.slv | 29 + .../app/views/shared/_confirm_modal.html.slv | 39 + admin/app/views/shared/_db_picker.html.slv | 28 + admin/app/views/shared/_nav.html.slv | 80 + admin/app/views/triggers/index.html.slv | 208 + admin/app/views/users/index.html.slv | 100 + admin/config/application.sl | 103 + admin/config/routes.sl | 104 + admin/db/migrations/CLAUDE.md | 274 + admin/docs/ai-agents.md | 75 + admin/docs/authentication.md | 138 + admin/docs/blog/advanced-modeling.md | 76 + admin/docs/blog/ai-coding-agents.md | 153 + admin/docs/blog/background-jobs-and-cron.md | 481 ++ admin/docs/blog/benchmarks-reality.md | 84 + .../blog/competing-with-big-frameworks.md | 124 + admin/docs/blog/dev-bar.md | 101 + admin/docs/blog/e2e-testing.md | 114 + admin/docs/blog/event-streaming-with-es.md | 642 +++ admin/docs/blog/github-oauth.md | 294 + admin/docs/blog/google-oauth.md | 221 + admin/docs/blog/htmx-datatable.md | 282 + admin/docs/blog/htmx-integration.md | 174 + admin/docs/blog/index.md | 124 + admin/docs/blog/liveview.md | 313 + admin/docs/blog/no-build-no-dependency.md | 117 + admin/docs/blog/password-validation.md | 183 + admin/docs/blog/pattern-matching.md | 88 + admin/docs/blog/scaffolds.md | 63 + admin/docs/blog/sendgrid-email-jobs.md | 169 + admin/docs/blog/similar-search.md | 404 ++ admin/docs/blog/soli-minimal-lang.md | 92 + admin/docs/blog/spreadsheet-functions.md | 172 + admin/docs/blog/totp-authentication.md | 173 + .../docs/blog/uploads-and-image-transforms.md | 123 + admin/docs/blog/web-push-notifications.md | 248 + admin/docs/builtins.md | 5047 +++++++++++++++++ admin/docs/builtins/vapid.md | 206 + admin/docs/client-interactivity.md | 250 + admin/docs/configuration.md | 189 + admin/docs/controllers.md | 682 +++ admin/docs/database.md | 328 ++ admin/docs/editor-integration.md | 129 + admin/docs/engines.md | 92 + admin/docs/error-pages.md | 250 + admin/docs/formatting.md | 88 + admin/docs/installation.md | 162 + admin/docs/introduction.md | 90 + admin/docs/jobs.md | 225 + admin/docs/live-reload.md | 93 + admin/docs/liveview.md | 181 + admin/docs/metaprogramming.md | 101 + admin/docs/middleware.md | 135 + admin/docs/migrations.md | 306 + admin/docs/models.md | 1479 +++++ admin/docs/request-params.md | 207 + admin/docs/routing.md | 234 + admin/docs/scaffold.md | 213 + admin/docs/sessions.md | 184 + admin/docs/soli-language.md | 3953 +++++++++++++ admin/docs/state-machines.md | 338 ++ admin/docs/testing-assertions.md | 345 ++ admin/docs/testing-e2e.md | 837 +++ admin/docs/testing-guide.md | 837 +++ admin/docs/testing-quick-reference.md | 113 + admin/docs/testing.md | 442 ++ admin/docs/validation.md | 307 + admin/docs/views.md | 862 +++ admin/package-lock.json | 1028 ++++ admin/package.json | 12 + admin/public/css/application.css | 2080 +++++++ admin/public/js/admin-drop.js | 79 + admin/public/js/admin-editor.js | 233 + admin/public/js/admin-json.js | 65 + admin/public/js/alpine.min.js | 6 + admin/public/js/htmx.min.js | 2 + admin/soli.toml | 4 + admin/tailwind.config.js | 35 + admin/tests/CLAUDE.md | 312 + admin/tests/api_keys_controller_spec.sl | 40 + admin/tests/collections_controller_spec.sl | 190 + admin/tests/cron_controller_spec.sl | 61 + admin/tests/databases_controller_spec.sl | 44 + admin/tests/documents_controller_spec.sl | 149 + admin/tests/env_vars_controller_spec.sl | 48 + .../tests/helpers/application_helper_spec.sl | 67 + admin/tests/helpers/solidb_helper_spec.sl | 59 + admin/tests/home_controller_spec.sl | 25 + admin/tests/live_queries_controller_spec.sl | 27 + .../materialized_views_controller_spec.sl | 101 + admin/tests/query_controller_spec.sl | 100 + admin/tests/queues_controller_spec.sl | 72 + admin/tests/roles_controller_spec.sl | 65 + admin/tests/scripts_controller_spec.sl | 69 + admin/tests/solidb_client_spec.sl | 145 + admin/tests/solidb_endpoints_spec.sl | 82 + admin/tests/triggers_controller_spec.sl | 87 + admin/tests/users_controller_spec.sl | 49 + admin/tests/zz_dump_spec.sl | 37 + admin/tests/zz_probe_spec.sl | 23 + ...login-rate-limit-shared-loopback-bucket.md | 31 + ...ow-database-drop-per-cf-options-rewrite.md | 91 + ...cate-leaves-blob-chunks-and-index-state.md | 34 + www/public/layouts.css | 2 +- 161 files changed, 37620 insertions(+), 1 deletion(-) create mode 100644 admin/.env.test create mode 100644 admin/.gitignore create mode 100644 admin/AGENTS.md create mode 100644 admin/CLAUDE.md create mode 100644 admin/README.md create mode 100644 admin/app/assets/css/application.css create mode 100644 admin/app/controllers/CLAUDE.md create mode 100644 admin/app/controllers/api_keys_controller.sl create mode 100644 admin/app/controllers/collections_controller.sl create mode 100644 admin/app/controllers/cron_controller.sl create mode 100644 admin/app/controllers/databases_controller.sl create mode 100644 admin/app/controllers/documents_controller.sl create mode 100644 admin/app/controllers/env_vars_controller.sl create mode 100644 admin/app/controllers/home_controller.sl create mode 100644 admin/app/controllers/live_queries_controller.sl create mode 100644 admin/app/controllers/materialized_views_controller.sl create mode 100644 admin/app/controllers/query_controller.sl create mode 100644 admin/app/controllers/queues_controller.sl create mode 100644 admin/app/controllers/roles_controller.sl create mode 100644 admin/app/controllers/scripts_controller.sl create mode 100644 admin/app/controllers/triggers_controller.sl create mode 100644 admin/app/controllers/users_controller.sl create mode 100644 admin/app/helpers/application_helper.sl create mode 100644 admin/app/helpers/solidb_helper.sl create mode 100644 admin/app/middleware/CLAUDE.md create mode 100644 admin/app/middleware/cors.sl create mode 100644 admin/app/models/CLAUDE.md create mode 100644 admin/app/services/admin_context.sl create mode 100644 admin/app/services/solidb_client.sl create mode 100644 admin/app/services/solidb_endpoints.sl create mode 100644 admin/app/views/CLAUDE.md create mode 100644 admin/app/views/api_keys/index.html.slv create mode 100644 admin/app/views/collections/_indexes.html.slv create mode 100644 admin/app/views/collections/_stats.html.slv create mode 100644 admin/app/views/collections/index.html.slv create mode 100644 admin/app/views/cron/index.html.slv create mode 100644 admin/app/views/databases/index.html.slv create mode 100644 admin/app/views/documents/index.html.slv create mode 100644 admin/app/views/env_vars/index.html.slv create mode 100644 admin/app/views/home/index.html.slv create mode 100644 admin/app/views/layouts/application.html.slv create mode 100644 admin/app/views/live_queries/show.html.slv create mode 100644 admin/app/views/materialized_views/index.html.slv create mode 100644 admin/app/views/query/_explain.html.slv create mode 100644 admin/app/views/query/_results.html.slv create mode 100644 admin/app/views/query/_slow_badge.html.slv create mode 100644 admin/app/views/query/show.html.slv create mode 100644 admin/app/views/query/slow.html.slv create mode 100644 admin/app/views/queues/_jobs.html.slv create mode 100644 admin/app/views/queues/index.html.slv create mode 100644 admin/app/views/roles/index.html.slv create mode 100644 admin/app/views/roles/show.html.slv create mode 100644 admin/app/views/scripts/_form.html.slv create mode 100644 admin/app/views/scripts/edit.html.slv create mode 100644 admin/app/views/scripts/index.html.slv create mode 100644 admin/app/views/scripts/new.html.slv create mode 100644 admin/app/views/scripts/show.html.slv create mode 100644 admin/app/views/shared/_banners.html.slv create mode 100644 admin/app/views/shared/_confirm_modal.html.slv create mode 100644 admin/app/views/shared/_db_picker.html.slv create mode 100644 admin/app/views/shared/_nav.html.slv create mode 100644 admin/app/views/triggers/index.html.slv create mode 100644 admin/app/views/users/index.html.slv create mode 100644 admin/config/application.sl create mode 100644 admin/config/routes.sl create mode 100644 admin/db/migrations/CLAUDE.md create mode 100644 admin/docs/ai-agents.md create mode 100644 admin/docs/authentication.md create mode 100644 admin/docs/blog/advanced-modeling.md create mode 100644 admin/docs/blog/ai-coding-agents.md create mode 100644 admin/docs/blog/background-jobs-and-cron.md create mode 100644 admin/docs/blog/benchmarks-reality.md create mode 100644 admin/docs/blog/competing-with-big-frameworks.md create mode 100644 admin/docs/blog/dev-bar.md create mode 100644 admin/docs/blog/e2e-testing.md create mode 100644 admin/docs/blog/event-streaming-with-es.md create mode 100644 admin/docs/blog/github-oauth.md create mode 100644 admin/docs/blog/google-oauth.md create mode 100644 admin/docs/blog/htmx-datatable.md create mode 100644 admin/docs/blog/htmx-integration.md create mode 100644 admin/docs/blog/index.md create mode 100644 admin/docs/blog/liveview.md create mode 100644 admin/docs/blog/no-build-no-dependency.md create mode 100644 admin/docs/blog/password-validation.md create mode 100644 admin/docs/blog/pattern-matching.md create mode 100644 admin/docs/blog/scaffolds.md create mode 100644 admin/docs/blog/sendgrid-email-jobs.md create mode 100644 admin/docs/blog/similar-search.md create mode 100644 admin/docs/blog/soli-minimal-lang.md create mode 100644 admin/docs/blog/spreadsheet-functions.md create mode 100644 admin/docs/blog/totp-authentication.md create mode 100644 admin/docs/blog/uploads-and-image-transforms.md create mode 100644 admin/docs/blog/web-push-notifications.md create mode 100644 admin/docs/builtins.md create mode 100644 admin/docs/builtins/vapid.md create mode 100644 admin/docs/client-interactivity.md create mode 100644 admin/docs/configuration.md create mode 100644 admin/docs/controllers.md create mode 100644 admin/docs/database.md create mode 100644 admin/docs/editor-integration.md create mode 100644 admin/docs/engines.md create mode 100644 admin/docs/error-pages.md create mode 100644 admin/docs/formatting.md create mode 100644 admin/docs/installation.md create mode 100644 admin/docs/introduction.md create mode 100644 admin/docs/jobs.md create mode 100644 admin/docs/live-reload.md create mode 100644 admin/docs/liveview.md create mode 100644 admin/docs/metaprogramming.md create mode 100644 admin/docs/middleware.md create mode 100644 admin/docs/migrations.md create mode 100644 admin/docs/models.md create mode 100644 admin/docs/request-params.md create mode 100644 admin/docs/routing.md create mode 100644 admin/docs/scaffold.md create mode 100644 admin/docs/sessions.md create mode 100644 admin/docs/soli-language.md create mode 100644 admin/docs/state-machines.md create mode 100644 admin/docs/testing-assertions.md create mode 100644 admin/docs/testing-e2e.md create mode 100644 admin/docs/testing-guide.md create mode 100644 admin/docs/testing-quick-reference.md create mode 100644 admin/docs/testing.md create mode 100644 admin/docs/validation.md create mode 100644 admin/docs/views.md create mode 100644 admin/package-lock.json create mode 100644 admin/package.json create mode 100644 admin/public/css/application.css create mode 100644 admin/public/js/admin-drop.js create mode 100644 admin/public/js/admin-editor.js create mode 100644 admin/public/js/admin-json.js create mode 100644 admin/public/js/alpine.min.js create mode 100644 admin/public/js/htmx.min.js create mode 100644 admin/soli.toml create mode 100644 admin/tailwind.config.js create mode 100644 admin/tests/CLAUDE.md create mode 100644 admin/tests/api_keys_controller_spec.sl create mode 100644 admin/tests/collections_controller_spec.sl create mode 100644 admin/tests/cron_controller_spec.sl create mode 100644 admin/tests/databases_controller_spec.sl create mode 100644 admin/tests/documents_controller_spec.sl create mode 100644 admin/tests/env_vars_controller_spec.sl create mode 100644 admin/tests/helpers/application_helper_spec.sl create mode 100644 admin/tests/helpers/solidb_helper_spec.sl create mode 100644 admin/tests/home_controller_spec.sl create mode 100644 admin/tests/live_queries_controller_spec.sl create mode 100644 admin/tests/materialized_views_controller_spec.sl create mode 100644 admin/tests/query_controller_spec.sl create mode 100644 admin/tests/queues_controller_spec.sl create mode 100644 admin/tests/roles_controller_spec.sl create mode 100644 admin/tests/scripts_controller_spec.sl create mode 100644 admin/tests/solidb_client_spec.sl create mode 100644 admin/tests/solidb_endpoints_spec.sl create mode 100644 admin/tests/triggers_controller_spec.sl create mode 100644 admin/tests/users_controller_spec.sl create mode 100644 admin/tests/zz_dump_spec.sl create mode 100644 admin/tests/zz_probe_spec.sl create mode 100644 tasks/todo/login-rate-limit-shared-loopback-bucket.md create mode 100644 tasks/todo/slow-database-drop-per-cf-options-rewrite.md create mode 100644 tasks/todo/truncate-leaves-blob-chunks-and-index-state.md diff --git a/admin/.env.test b/admin/.env.test new file mode 100644 index 00000000..24fbbcdd --- /dev/null +++ b/admin/.env.test @@ -0,0 +1,22 @@ +# Test environment -- `soli test` reads this instead of .env. +# Use a dedicated database so tests don't touch development data; the test +# runner drops & recreates this database per worker on each run. + +# 127.0.0.1, not localhost: see .env. +SOLIDB_HOST=http://127.0.0.1:6745 +SOLIDB_DATABASE=solidb_admin_test +SOLIDB_USERNAME=admin +SOLIDB_PASSWORD=admin + +# HTTP.request()'s SSRF guard blocks loopback; the specs exercise the real +# local SoliDB API, so allow it (same as .env). +SOLI_DEV_ALLOW_SSRF=1 + +# The test client and the test server are colocated and trusted; the Origin +# check would reject every POST otherwise. +SOLI_DISABLE_CSRF=true + +# .env sets a LAN SOLIDB_PUBLIC_WS_URL for remote browsers; it leaks into the +# test process env. Force it empty so public_ws_url() derives from SOLIDB_HOST +# (blank counts as unset). +SOLIDB_PUBLIC_WS_URL= diff --git a/admin/.gitignore b/admin/.gitignore new file mode 100644 index 00000000..e1a5811f --- /dev/null +++ b/admin/.gitignore @@ -0,0 +1,34 @@ +# Dependencies +node_modules/ + +# Build artifacts +/target/ +*.o +*.so + +# Process files +*.pid + +# Logs +*.log +logs/ + +# Environment +.env +.env.local +.env.*.local + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Coverage +coverage/ +.coverage diff --git a/admin/AGENTS.md b/admin/AGENTS.md new file mode 100644 index 00000000..6bf1e381 --- /dev/null +++ b/admin/AGENTS.md @@ -0,0 +1,5 @@ +# AGENTS.md + +This project's agent guidance lives in `CLAUDE.md` (root) and per-directory `CLAUDE.md` files under `app/`, `tests/`, and `db/migrations/`. They apply to all coding agents — Claude Code, Cursor, Aider, Copilot CLI, Codex CLI, etc. + +Start by reading `CLAUDE.md`. The "For AI agents — read this first" section at the top is the contract: verification loop, generator-first, footguns. diff --git a/admin/CLAUDE.md b/admin/CLAUDE.md new file mode 100644 index 00000000..ac9b9919 --- /dev/null +++ b/admin/CLAUDE.md @@ -0,0 +1,513 @@ +# Soli Lang + +Soli is a dynamically-typed, high-performance web framework written in Rust. This file orients an AI assistant (and future you) to how *this* application is laid out and what the language syntax actually looks like. + +## For AI agents — read this first + +You are working in a Soli MVC app. Soli looks like Ruby/JS but has its own quirks; skim the **Footgun cheatsheet** below before generating code. Per-directory `CLAUDE.md` files in `app/controllers/`, `app/models/`, `app/views/`, `app/middleware/`, `tests/`, and `db/migrations/` give you the local rules — Claude Code loads them automatically when you work in those directories. + +### Verification loop (mandatory before reporting done) + +1. `soli lint ` — naming, smells, undefined-locals. +2. `soli test tests/.sl` — narrow, fast feedback. +3. `soli test --coverage --coverage-min 90.0` — full sweep before handing off. +4. `soli serve . --dev`, then hit the route in a browser if you changed a UI/route — confirm 200 and that the page renders. + +If a step fails, fix the root cause. Don't weaken assertions, lower the coverage bar, or `--no-verify` past hooks. The `/soli-verify` slash command bundles steps 1-3. + +### Reach for generators first + +| Task | Command | +|------------------------------------------|-----------------------------------------------| +| New controller (+ views + spec) | `soli generate controller posts` | +| New model | `soli generate model post` | +| New migration | `soli generate migration create_posts` | +| Full RESTful resource end-to-end | `/soli-resource posts` (slash command) | + +Generators encode the naming, location, and boilerplate the framework expects. Hand-writing diverges and triggers lint failures. + +## Footgun cheatsheet (Soli ≠ Ruby ≠ JS) + +| You'd type… | In Soli it's… | Why | +|--------------------------------------------|--------------------------------------------|------------------------------------------------------------------------------| +| `// comment` | `# comment` | `//` was standardized away — lint flags it. | +| `${name}` / `#{name}` in a string | `\(name)` | Backslash-paren is the only interpolation form. | +| `@"multi\nline"` raw string | `[[multi\nline]]` or `""" ... """` | `@"..."` doesn't exist; `@` is only for `@sdbql{...}` query blocks. | +| `if (x) { … }` | `if x … end` | C-style parses, but Ruby-style is the convention here. | +| `xs.forEach(…)` | `xs.each do \|x\| … end` or `for x in xs` | No `forEach`. | +| `x \|\| default` | `x ?? default` | `\|\|` returns the wrong side when `x` is `0` or `""` (those are TRUTHY). | +| `if (xs.length)` | `if xs.length() > 0` | `0` and `""` are truthy in Soli — only `false` and `null` are falsy. | +| `import "../models/post.sl"` in controller | nothing — already auto-loaded | Triggers `style/redundant-model-import` lint. | +| Building URLs by hand | `posts_path()`, `post_path(post)` | Named helpers come from `resources(...)` in `config/routes.sl`. | +| Overriding `Model.all` / `Model.find` | don't | Inherited from `Model`; the framework relies on it. | +| `if x == nil \|\| x == ""` | `if x.blank?` | `.blank?` covers both nil and empty string in one call. | +| `if x == nil` / `if x != nil` | `if x.nil?` / `unless x.nil?` | `.nil?` reads as the question; reserve `==`/`!=` for value comparisons. | +| `user == nil ? nil : user._key` | `user&._key` | Safe navigation short-circuits to `nil` if the receiver is `nil`. | +| `if s != "a" && s != "b" && s != "c"` | `unless ["a", "b", "c"].includes?(s)` | Intent is membership check, not a pile of `&&`. | +| `x = x \|\| default` | `x \|\|= default` | `\|\|=` is a single operator for "set if nil/false". | + +## Recipes + +### Add a RESTful resource end-to-end + +1. `soli generate model post` → fill fields, validations, associations. +2. `soli generate migration create_posts` → fill `up`/`down`, then `soli db:migrate up`. +3. `soli generate controller posts` → fill `index`/`show`/`create`/etc. +4. In `config/routes.sl` add `resources("posts")`. +5. Edit `app/views/posts/*.html.slv`. +6. Add specs in `tests/posts_controller_spec.sl`. +7. Run the verification loop. + +(Or just: `/soli-resource post` — bundles steps 1-4 and stubs 5-6.) + +### Add an authenticated route + +Wrap the routes in a `middleware("authenticate", -> { … })` block in `config/routes.sl`. The `authenticate` middleware in `app/middleware/auth.sl` is `scope_only`, so unscoped routes are unaffected. + +### Debug a request live + +Run `soli serve . --dev`. The dev bar shows the AQL queries (`dev_queries()`) issued for the request, with bind vars and durations. + +### Add a partial + +- File: `app/views//_name.html.slv` (leading underscore is mandatory). +- Render: `<%- partial("dir/name", { "key": value }) %>` — use `<%-` (raw output), not `<%=`, since the partial returns HTML that must not be re-escaped. +- Inside the partial: read via `key` (or `locals["key"]` if it collides with a builtin/helper). + +## Project Structure + +``` +app/ +├── controllers/ # Request handlers (one class per resource, < Controller) +├── helpers/ # View helper functions +├── middleware/ # Request/response filters (per-file `# order:` directives) +├── models/ # Data models (< Model — ORM is inherited) +└── views/ # ERB-style templates with .html.slv extension +config/ +└── routes.sl # URL routing +db/ +└── migrations/ # Database migrations +public/ # Static assets (CSS/JS compiled into here) +tests/ # *_spec.sl test files +``` + +## Naming Conventions + +| Type | Convention | Example | +|-----------|------------------------|------------------------| +| Files | `snake_case.sl` | `posts_controller.sl` | +| Classes | `PascalCase` | `PostsController` | +| Functions | `snake_case` | `get_user_by_id` | +| Constants | `SCREAMING_SNAKE_CASE` | `MAX_SIZE` | + +**Use intelligible variable names** — no single-letter or cryptic short names. The name should make the intent obvious without scanning back for the assignment. + +```soli +# Bad — what is p? r? pg? qb? +let p = params +let r = users_result(p["q"], p["sort"]) +let pg = r["pagination"] +let qb = User.where(...) + +# Good — read top-to-bottom and the meaning is clear. +let search_query = params["q"] +let sort_column = params["sort"] +let result = users_result(search_query, sort_column) +let pagination = result["pagination"] +let query_builder = User.where(...) +``` + +Short names are only acceptable for true conventions: loop indices (`i`, `j`), block parameters whose role is obvious from context (`fn(x) x * 2`), and well-known math symbols inside their natural domain. + +## Syntax basics + +Soli supports both Ruby-style (`def`/`end`, `class X < Y ... end`, `if cond ... end`) and C-style (`fn`/`{ }`, `class X extends Y { ... }`, `if cond { ... }`); they parse to the same AST. **The convention in this project is Ruby-style** for class declarations and control flow (`class Demo < Test ... end`, `if cond ... end`). Reserve `fn { ... }` for free-standing functions and lambdas. Match this style when writing new code. + +```soli +# Variables +name = "Alice" # `let` is optional — bare assignment creates the binding +let age: Int = 30 # Use `let` when you want a type annotation, or to + # forward-declare before a branch that assigns it +const MAX = 100 # Immutable + +# Prefer the bare `name = value` form. Reach for `let` only when it earns +# its keep: a type annotation, or a hoisted declaration before `if`/`match`. + +# Free-standing functions +fn add(a: Int, b: Int) -> Int { + return a + b; +} + +# Implicit return: the last expression in a block is returned +fn greet(name) { + "Hello, " + name + "!" +} + +# Lambdas +let double = fn(x) { return x * 2; }; +let halve = |x| { return x / 2; }; + +# String interpolation +let msg = "Hi \(name), age \(age)" + +# Multiline / raw strings (NO @"..." — that form does not exist) +let lua_raw = [[ + Raw text. No escape processing. + Good for queries with embedded "double quotes". +]] +let triple = """ + Raw, multiline. Closes on """. + Good for content with ] or ]] inside. +""" +let single_raw = r"C:\Users\name" # raw, single-line only + +# Collection iteration — Ruby-style block, no parens before `do` +[1, 2, 3].map do |x| x * 2 end +[1, 2, 3].filter do |x| x > 2 end + +# Pipelines (when chaining multiple stages) +[1, 2, 3] |> map(fn(x) x * 2) |> filter(fn(x) x > 2) + +# Pattern matching +let label = match value { + 42 => "the answer", + n if n > 0 => "positive", + [first, ...rest] => "head: " + str(first), + _ => "other" +}; + +# Postfix conditionals (idiomatic) +print("adult") if age >= 18 +let data = fetch() rescue null # returns null if fetch() throws + +# Concise defaults and guards +this.balance ||= 0 # ||= sets when nil/false +this.email = this.email.trim().downcase() unless this.email.blank? # .blank? covers nil + "" +unless ["up", "late", "overdue"].includes?(this.status) # membership check + add_error("invalid status") +end +``` + +## Routes (`config/routes.sl`) + +```soli +# Basic routes +get("/", "home#index", name: "root") +get("/about", "pages#about", name: "about") +post("/users", "users#create") + +# RESTful resources — registers index/show/new/create/edit/update/destroy +# plus path/url helpers: posts_path(), post_path(post), new_post_path(), +# edit_post_path(post), and *_url variants. +resources("posts") + +# Scoped middleware — only runs for routes inside the block +middleware("authenticate", -> { + get("/admin", "admin#index") + resources("admin/users") +}) +``` + +Use the named helpers (`posts_path`, `root_path`, etc.) in controllers and views — never concatenate URLs by hand. + +## Controllers + +Controllers are classes that inherit from `Controller`. Action methods take a request hash and return a response. + +```soli +# app/controllers/posts_controller.sl +class PostsController < Controller + static + this.layout = "application" + end + + # GET /posts + def index(req) + let posts = Post.all() + return render("posts/index", { "posts": posts, "title": "Posts" }) + end + + # GET /posts/:id — Model.find raises on miss; framework maps to 404 + def show(req) + let post = Post.find(req.params["id"]) + return render("posts/show", { "post": post }) + end + + # POST /posts + def create(req) + let permitted = this._permit_params(req.params) + let post = Post.create(permitted) + if post._errors + return render("posts/new", { "post": post }) + end + return redirect(post_path(post)) + end + + # Mass-assignment protection — whitelist allowed fields + def _permit_params(params) + return { "title": params["title"], "body": params["body"] } + end +end +``` + +### Request access + +- `req.params["id"]` — route + query + body params merged +- `req["json"]` — parsed JSON body +- `req["headers"]`, `req["cookies"]`, `req["method"]` +- Bare `params` is also available globally inside actions (= `req.params`) + +### Response shapes + +- `render("view/name", {...})` — render `app/views/view/name.html.slv` with the given locals +- `redirect("/path")` or `redirect(post_path(post))` — HTTP redirect +- `{"status": 422, "headers": {...}, "body": "..."}` — raw response + +## Models + +Models inherit from `Model`; CRUD methods come with the inheritance — don't redefine them. + +```soli +# app/models/post.sl +class Post < Model + # Inherited from Model: + # Post.all() Post.find(id) Post.find_by(field, val) + # Post.where({...}) Post.create({...}) post.save() post.delete() + # + # `Post.find(id)` RAISES RecordNotFound on miss — the framework converts + # that to a 404 automatically. Don't add `if post.nil? { 404 }` after it; + # that branch is unreachable. Use `find_by` / `first_by` when you want + # the "or nil" shape instead. + # + # Add associations and validations declaratively: + belongs_to("user") + has_many("comments") + + validates("title", { "presence": true, "min_length": 3 }) + validates("body", { "presence": true }) + + before_save("normalize_title") + + def normalize_title + this.title = this.title.trim() + end +end +``` + +`Model.create(...)` always returns an instance. On validation/database failure, the instance has `_errors` populated — check `if post._errors` and re-render the form. Don't write fake `static` shims around the inherited CRUD. + +### Raw queries (SDBQL) + +Drop down to raw SDBQL only when the ORM doesn't cover the case. **Always parameterize** — never concatenate user input. + +```soli +# `@sdbql{}` block — preferred for multi-line queries. +# `#{expr}` is bound as a parameter, not interpolated as text. +let min_age = 18 +let users = @sdbql{ + FOR u IN users + FILTER u.age >= #{min_age} + SORT u.name ASC + LIMIT 50 + RETURN u +} +``` + +## Views (`.html.slv`) + +```erb +

    <%= title %>

    + +<% for post in posts %> +
    +

    <%= h(post.title) %>

    + <%= post.body %> +
    +<% end %> + +<%= link_to("New post", new_post_path()) %> +``` + +Always use `h()` to escape user-supplied content — XSS is the default risk. + +## Middleware + +A middleware file declares one function. Per-file directive comments at the top configure how the framework wires it up: + +```soli +# app/middleware/auth.sl + +# order: 20 +# scope_only: true — only runs when wrapped in `middleware("authenticate", -> { ... })` + +def authenticate(req) + let key = req["headers"]["X-Api-Key"] ?? "" + if key == "" + return { + "continue": false, + "response": { "status": 401, "body": "Unauthorized" } + } + end + return { "continue": true, "request": req } +end +``` + +| Directive | Meaning | +|----------------------|--------------------------------------------------------| +| `# order: N` | Lower runs first. Default 100. | +| `# global_only: true` | Always runs; cannot be scoped. | +| `# scope_only: true` | Only runs when explicitly scoped via `middleware(...)`. | + +Returning `{"continue": false, "response": {...}}` short-circuits with that response. Returning `{"continue": true, "request": req}` proceeds to the next middleware / handler. + +## Testing + +Specs live in `tests/` and run with `soli test`. Use the BDD DSL with `describe` / `test` / `before_each`. Controller tests get an E2E client (`get`, `post`, `put`, `delete`, `assigns()`, `view_path()`, `as_guest()`). + +```soli +# tests/posts_controller_spec.sl +describe("PostsController", fn() { + before_each(fn() { + as_guest(); + }); + + describe("GET /posts", fn() { + test("returns list of posts", fn() { + let response = get("/posts"); + assert_eq(res_status(response), 200); + assert_hash_has_key(assigns(), "posts"); + }); + }); + + describe("POST /posts", fn() { + test("creates with valid data", fn() { + let response = post("/posts", { "title": "Hello", "body": "World" }); + assert_eq(res_status(response), 302); + }); + + test("rejects invalid data", fn() { + let response = post("/posts", {}); + assert_eq(res_status(response), 422); + }); + }); +}); +``` + +### Test coverage requirement + +**Every new feature must ship with tests achieving >90% coverage of the changed code.** Run coverage locally before opening a PR: + +```bash +soli test --coverage # generate report +soli test --coverage --coverage-min 90.0 # fail if under 90% +``` + +This applies to controllers, models, middleware, helpers, and any new library code. Don't merge a feature whose coverage report is missing or below the threshold — write the tests first if it helps you design the API. + +## SOLID Principles + +Apply these for maintainable code. + +```soli +# Single Responsibility — one reason to change per class +class UserValidator + def validate(user) end +end + +class UserRepository + def save(user) end +end + +# Open/Closed — extend via subclasses, don't edit the base +class Shape + def area -> Float + 0.0 + end +end + +class Circle < Shape + radius: Float + + def area -> Float + 3.14159 * this.radius * this.radius + end +end + +# Liskov — subclasses must honor the parent's contract. +# Don't override a method to throw where the parent returns. + +# Interface Segregation — many small interfaces beat one fat one +interface Printable + def print() +end + +interface Exportable + def export() +end + +# Dependency Inversion — depend on abstractions +interface UserRepository + def find(id: Int) -> User +end + +class UserService + repo: UserRepository + + def get(id) + this.repo.find(id) + end +end +``` + +## Linting + +```bash +soli lint # lint entire project +soli lint app/controllers/ # lint a directory +soli lint path/to/file.sl # lint a single file +``` + +Key rules: + +- `naming/snake-case`, `naming/pascal-case` +- `style/empty-block`, `style/line-length` (≤120 chars) +- `style/redundant-model-import` — don't `import "../models/*.sl"` inside `app/controllers/`; models are auto-loaded +- `smell/unreachable-code`, `smell/empty-catch`, `smell/duplicate-methods`, `smell/dangerous-server-builtin` (flags `db_query_raw` / `Trusted.*` / `System.shell` / backticks in `app/controllers/`, `app/middleware/`, `app/views/`) +- `smell/deep-nesting` (≤4 levels) +- `smell/undefined-local` — reads of a name never assigned in scope (catches typos) + +## Common commands + +```bash +soli serve . --dev # dev server, hot reload, dev bar enabled +soli serve . --port 5011 # run without --dev (still single-process) + +soli generate controller posts # scaffold controller + spec + views +soli generate model post # scaffold model +soli generate migration create_posts # scaffold migration + +soli db:migrate up # run pending migrations +soli db:migrate down # roll back last migration +soli db:migrate status # show migration state + +soli test # run all tests in tests/ +soli test --coverage --coverage-min 90.0 +soli lint # static analysis +``` + +## Conventions to follow + +1. **Prefer Ruby-style** for classes and control flow — `class Demo < Test ... end`, `def name(args) ... end`, `if cond ... end`. Reserve `fn { }` for free-standing functions and lambdas. +2. **Use type annotations** on public function signatures — they catch errors and document intent. +3. **Prefer immutability** — `const` for values that never change. +4. **Chain collection methods** instead of writing manual loops. +5. **Use named parameters** when a function has multiple optional args. +6. **Use named route helpers** (`posts_path`, `root_path`) — never hand-built URL strings. +7. **Validate at the model**, not in the controller — keep controllers thin. +8. **Return errors early** — don't pile `if`s; bail with a 422/redirect at the first invalid branch. +9. **Use `.blank?` for nil/empty checks** — replaces `x == nil || x == ""`. +10. **Use `.nil?` over `== nil`** — `if x.nil?` / `unless x.nil?` reads as a question; keep `==`/`!=` for value comparisons. +11. **Use `&.` to short-circuit on nil** — `user&._key` replaces `user == nil ? nil : user._key`; chain it (`user&.address&.city`) instead of nested guards. +12. **Use `||=` for falsey defaults** — `this.balance ||= 0` instead of `if this.balance == nil`. +13. **Use `.includes?` for membership checks** — replaces chained `||` comparisons. +14. **Test new features to >90% coverage** — non-negotiable, see above. diff --git a/admin/README.md b/admin/README.md new file mode 100644 index 00000000..a559e252 --- /dev/null +++ b/admin/README.md @@ -0,0 +1,92 @@ +# admin + +A Soli MVC application. + +## Getting Started + +### Development Server + +Start the development server with hot reload: + +```bash +soli serve . --dev +``` + +Your app will be available at [http://localhost:5011](http://localhost:5011) + +### Production Server + +Start the production server: + +```bash +soli serve . --port 5011 +``` + +Or run as a daemon: + +```bash +soli serve . -d +``` + +## Project Structure + +``` +admin/ +├── app/ +│ ├── assets/ +│ │ └── css/ +│ │ └── application.css # Source CSS with Tailwind directives +│ ├── controllers/ # Request handlers +│ ├── models/ # Data models +│ └── views/ # HTML templates +│ ├── home/ # Home page views +│ └── layouts/ # Layout templates +├── config/ +│ └── routes.sl # Route definitions +├── db/ +│ └── migrations/ # Database migrations +├── public/ # Static assets (compiled output) +│ ├── css/ +│ │ └── application.css # Compiled CSS (generated) +│ ├── js/ +│ └── images/ +├── tests/ # Test files +├── package.json # npm dependencies +└── tailwind.config.js # Tailwind configuration +``` + +## Database Migrations + +Generate a new migration: + +```bash +soli db:migrate generate create_users +``` + +Run pending migrations: + +```bash +soli db:migrate up +``` + +Rollback last migration: + +```bash +soli db:migrate down +``` + +Check migration status: + +```bash +soli db:migrate status +``` + +## Documentation + +- [Soli MVC Documentation](https://soli.solisoft.net/docs) +- [Soli Language Reference](https://soli.solisoft.net/docs/soli-language) +- [Tailwind CSS](https://tailwindcss.com/docs) + +## License + +MIT diff --git a/admin/app/assets/css/application.css b/admin/app/assets/css/application.css new file mode 100644 index 00000000..94c9c318 --- /dev/null +++ b/admin/app/assets/css/application.css @@ -0,0 +1,112 @@ +/* Tailwind CSS directives */ +@tailwind base; +@tailwind components; +@tailwind utilities; + +/* Base styles — SoliDB admin ops console (dark zinc + terminal teal) */ +@layer base { + html { + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + scroll-behavior: smooth; + } + + body { + @apply bg-zinc-950 text-zinc-100 leading-relaxed; + } + + h1 { @apply text-2xl font-semibold tracking-tight; } + h2 { @apply text-xl font-semibold tracking-tight; } + h3 { @apply text-lg font-medium; } + + img, video { @apply max-w-full h-auto; } +} + +@layer components { + /* Faint blueprint grid behind the content area. */ + .console-grid { + background-image: + linear-gradient(rgba(63, 63, 70, 0.16) 1px, transparent 1px), + linear-gradient(90deg, rgba(63, 63, 70, 0.16) 1px, transparent 1px); + background-size: 32px 32px; + } + + /* Uppercase mono section label — the console's signature typographic move. */ + .console-label { + @apply font-mono text-[10px] uppercase tracking-[0.2em] text-zinc-500; + } + + /* No backdrop-blur here: backdrop-filter makes the card the containing + block for position:fixed descendants, which would scope/crop the + modals that live inside cards (e.g. per-row document edit). */ + .console-card { + @apply border border-zinc-800 bg-zinc-900/80; + } + + .console-input { + @apply w-full border border-zinc-800 bg-zinc-950 px-3 py-2 text-sm text-zinc-100 + placeholder-zinc-600 outline-none transition-colors + focus:border-teal-500/60 focus:ring-1 focus:ring-teal-500/30; + } + + .btn-primary { + @apply inline-flex items-center gap-2 border border-teal-500/50 bg-teal-500/10 px-3 py-1.5 + font-mono text-xs uppercase tracking-wider text-teal-300 transition-all + hover:bg-teal-500/20 hover:border-teal-400; + } + + .btn-ghost { + @apply inline-flex items-center gap-2 border border-zinc-700 bg-transparent px-3 py-1.5 + font-mono text-xs uppercase tracking-wider text-zinc-400 transition-all + hover:border-zinc-500 hover:text-zinc-200; + } + + .btn-danger { + @apply inline-flex items-center gap-2 border border-red-900 bg-transparent px-3 py-1.5 + font-mono text-xs uppercase tracking-wider text-red-400/80 transition-all + hover:border-red-500 hover:bg-red-500/10 hover:text-red-300; + } + + .console-table { + @apply w-full text-left text-sm; + } + + .console-table thead th { + @apply border-b border-zinc-800 px-4 py-2 font-mono text-[10px] uppercase tracking-[0.2em] + text-zinc-500 font-normal; + } + + .console-table tbody td { + @apply border-b border-zinc-800/60 px-4 py-2.5 align-middle; + } + + .console-table tbody tr { + @apply transition-colors hover:bg-zinc-900/80; + } +} + +/* Alpine: hide x-cloak'd elements until Alpine boots. */ +[x-cloak] { display: none !important; } + +/* HTMX swap transition — content eases in on every fragment swap. */ +#content.htmx-swapping { opacity: 0; } +#content { transition: opacity 120ms ease-out; } + +/* Custom scrollbar — teal-tinted to match the accent. */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: rgba(20, 184, 166, 0.25); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb:hover { + background: rgba(20, 184, 166, 0.45); +} diff --git a/admin/app/controllers/CLAUDE.md b/admin/app/controllers/CLAUDE.md new file mode 100644 index 00000000..4cbe5b12 --- /dev/null +++ b/admin/app/controllers/CLAUDE.md @@ -0,0 +1,527 @@ +# Controllers + +This directory holds request handlers. **One file per resource**: +`posts_controller.sl` defines `class PostsController < Controller`. Filenames +are `snake_case.sl`; class names are `PascalCase` ending in `Controller`. + +Controllers stay thin: they pull params off the request, ask a model to do +the work, and return a response. Validation, persistence, and business rules +belong on the model — not here. + +## The Controller contract + +Inherit from `Controller`. The class body holds action methods (one per route) +plus a `static { ... }` block for layout and lifecycle hooks. + +```soli +class PostsController < Controller + static { + this.layout = "application" + } + + def index + @posts = Post.all + @title = "Posts" + end +end +``` + +The class itself uses Ruby-style `class X < Y ... end`, and methods use +`def name ... end` — but the `static` block **requires braces** (`static { ... }`). +The Ruby-style `static ... end` form does not parse. + +Free-function actions (no class wrapper) also work — see +`www/app/controllers/docs_controller.sl` for a real example — but the class +form is recommended for anything stateful or with hooks. + +## The `static { }` block + +Set the layout, register `before_action` / `after_action` hooks. The runtime +calls to `before_action` / `after_action` are no-ops — the controller registry +**textually scans the class body** at `soli serve` startup to wire hooks up, +so the syntax has to match what the scanner expects. + +```soli +class PostsController < Controller + static { + this.layout = "application" + + # Runs on every action. + this.before_action = fn(req) { + @current_user = session_get("user_id") + req + } + + # Runs only on the listed actions. + this.before_action(:show, :edit, :update, :delete) = fn(req) { + @post = Post.find(params["id"]) + req + } + + # after_action receives the response too. + this.after_action = fn(req, response) { + response + } + } +end +``` + +A `before_action` returning `req` proceeds; returning a response hash (one +with a `"status"` key) short-circuits and that response is returned to the +client. + +## Reading the request + +Inside an action, `req` is the request hash. The framework also exposes a few +globals so you don't have to dig: + +| Read | What it gives you | +|-----------------------|----------------------------------------------------------------| +| `params` | Merged route + query + JSON body (= `req["all"]`). Most common.| +| `params["id"]` | Path segment from `/posts/:id`. | +| `req["json"]` | Parsed JSON body (when the request had one). | +| `req["query"]` | Just the URL query string params. | +| `req["headers"]` | Lowercased header hash. `req["headers"]["user-agent"]`. | +| `req["method"]` | `"GET"`, `"POST"`, ... | +| `req["path"]` | Request path. | +| `req["files"]` | Array of uploaded files (multipart only). See **Handling file uploads**. | +| `cookies` | **Global** read-only hash of parsed cookies. `cookies.theme`. | + +`params` reads route params, query params, and parsed body fields with the +same key — write `params["id"]` whether the value came from `/posts/:id`, +`?id=42`, or `{"id": 42}` in the JSON body. + +## Response shapes + +Soli supports an **implicit render** that covers 80% of cases. Only reach for +explicit response builders when you need something special. + +### 1. Implicit render (preferred for the common case) + +If an action returns anything that is **not** a response hash (i.e. doesn't +have a `"status"` key), the framework auto-renders the default template at +`app/views//.html.slv`. `PostsController#show` → +`posts/show`. So an action that just sets up view state can be a one-liner: + +```soli +def show + @post = Post.find(params["id"]) +end +``` + +That's it. No `render(...)` call, no return statement. Every `@field` on the +instance is auto-injected as a view local (see "@-variables" below). + +### 2. Explicit render — non-default template or extra locals + +```soli +def create + permitted = this._permit_params(params) + @post = Post.create(permitted) + if @post._errors + return render("posts/new", { "title": "New post" }) # re-render form + end + redirect(post_path(@post)) +end +``` + +### 3. JSON + +```soli +def show + @post = Post.find(params["id"]) + render_json({ "id": @post.id, "title": @post.title }) +end +``` + +`render_json` sets `Content-Type: application/json` and serializes the hash +for you. + +### 4. Plain text + +```soli +def health + render_text("OK") +end +``` + +### 5. Redirect + +```soli +redirect("/posts") # 302 to a path +redirect(post_path(post)) # use named-route helpers, not hand-built URLs +redirect(:back) # back to the Referer if safe +redirect_external(url) # opt-in to redirect to a different host +``` + +### 6. Short-circuit with `halt` + +```soli +def admin + halt(403, "Forbidden") unless current_user.admin + @users = User.all +end +``` + +`halt(status, body)` immediately returns that response and skips the rest of +the action. + +### 7. Raw hash — when you need full control + +```soli +def webhook + return { + "status": 202, + "headers": { "Content-Type": "application/json", "X-Request-Id": req["id"] }, + "body": "{\"ok\":true}" + } +end +``` + +Any hash with a `"status"` key is treated as a final response and bypasses +auto-render. + +### 8. Content negotiation with `respond_to` + +```soli +def show + @post = Post.find(params["id"]) + respond_to(req, { + "html": fn() { render("posts/show") }, + "json": fn() { render_json({ "id": @post.id, "title": @post.title }) } + }) +end +``` + +## `@`-variables are injected into views + +Every non-underscore-prefixed instance field you set on the controller is +auto-exposed as a top-level view local. `@post = Post.find(...)` makes `post` +available in the template. + +```soli +def index + @posts = Post.all + @title = "Posts" + @filter = params["filter"] ?? "all" +end +``` + +In `app/views/posts/index.html.slv`: + +```erb +

    <%= @title %>

    +

    Showing: <%= @filter %>

    +<% for post in @posts %> +
  • <%= h(post.title) %>
    • +<% end %> +``` + +(Both `@title` and bare `title` resolve to the same value — `@` is the +canonical form.) + +**Underscore-prefixed fields are private.** `@_internal_state = ...` is *not* +exposed to the view — useful for state shared between hooks and actions that +shouldn't leak into templates. + +Because of this, you usually don't pass a data hash to `render` at all — set +`@fields` and let the framework do the rest. Reach for `render(view, {...})` +only when you need to render a *different* view than the default, or when you +want to override a field's name for the template. + +## Full CRUD sample + +```soli +# app/controllers/posts_controller.sl + +class PostsController < Controller + static { + this.layout = "application" + + # Look up @post once for every action that needs it. + this.before_action(:show, :edit, :update, :delete) = fn(req) { + @post = Post.find(params["id"]) + req + } + } + + # GET /posts — implicit render of posts/index + def index + @posts = Post.all + @title = "All posts" + end + + # GET /posts/:id — @post set by before_action, implicit render of posts/show + def show + @title = "Post: #{@post.title}" + end + + # GET /posts/new — implicit render of posts/new + def new + @post = Post.new + @title = "New post" + end + + # POST /posts + def create + permitted = this._permit_params(params) + @post = Post.create(permitted) + if @post._errors + @title = "New post" + return render("posts/new") # explicit: re-render the form view + end + redirect(post_path(@post)) + end + + # GET /posts/:id/edit — implicit render of posts/edit + def edit + @title = "Edit #{@post.title}" + end + + # PATCH/PUT /posts/:id + def update + permitted = this._permit_params(params) + @post.update(permitted) + if @post._errors + return render("posts/edit") + end + redirect(post_path(@post)) + end + + # DELETE /posts/:id + def delete + @post.delete + redirect(posts_path()) + end + + # Mass-assignment guard — whitelist the fields users can write. + def _permit_params(params) + { + "title": params["title"], + "body": params["body"] + } + end +end +``` + +Notes on the sample: + +- `before_action(:show, ...)` does the `Post.find` once instead of repeating + it in four actions. +- `_permit_params` is a private helper (the leading `_` makes it + non-routable). Only its return value is passed to `Model.create` / `update`. +- `index` / `show` / `new` / `edit` rely on **implicit render** — they just + set `@fields` and exit. +- `create` and `update` use **explicit render** for the validation-failure + re-render, because they need to render a *different* template than the + default for the action. +- All redirects use **named helpers** (`post_path(post)`, `posts_path()`) — + never hand-built URL strings. + +## Validation re-render flow + +`Model.create(attrs)` and `instance.save()` always return; on failure they +populate `_errors` on the returned instance. The controller checks `_errors`, +re-renders the form view passing the invalid instance, and the view displays +the errors. + +```soli +@post = Post.create(permitted) +if @post._errors + return render("posts/new") # view reads @post._errors to show messages +end +redirect(post_path(@post)) +``` + +**Don't wrap `Model.find` in nil-checks or `try/catch`.** On miss it raises +`RecordNotFound`, which the framework converts to a 404 automatically — so a +manual `if post.nil? ... end` branch is unreachable. Use `find_by(field, val)` +or `first_by(...)` when you want the "or nil" shape: + +```soli +@post = Post.find(params["id"]) # raises → 404 +@draft = Post.find_by("slug", params["slug"]) # nil on miss +``` + +## Handling file uploads + +For `multipart/form-data` requests, the framework parses every file part into +`req["files"]` — an array of hashes. Use the `find_uploaded_file(req, "field")` +helper to pull one by form field name; it returns `nil` if no file was +attached under that name or the request wasn't multipart. + +```soli +photo = find_uploaded_file(params, "photo") +# nil, or: +# { +# "name": "photo", # form field name +# "filename": "vacation.jpg", # client-supplied filename +# "content_type": "image/jpeg", +# "size": 184_213, # bytes +# "data": "" +# } +``` + +**Don't read the bytes yourself.** When the field is declared with +`uploader(...)` on the model, hand the file straight to the auto-generated +`attach_` method — it runs the configured MIME/size validations and +stores the blob in SoliDB for you: + +```soli +def create + @contact = Contact.create(this._permit_params(params)) + if @contact._errors + return render("contacts/new") + end + + photo = find_uploaded_file(params, "photo") + if !photo.nil? && !@contact.attach_photo(photo) + # attach_ populates @contact._errors on failure (bad MIME, + # too large, or storage error). Re-render with the same flow you + # use for validation errors. + return render("contacts/new") + end + + redirect(contact_path(@contact)) +end +``` + +For multi-file fields (`uploader("attachments", { "multiple": true, ... })`), +iterate `req["files"]` directly and attach one by one: + +```soli +def upload_batch + @document = Document.find(params["id"]) + for file in (req["files"] ?? []) + next unless file["name"] == "attachments" + @document.attach_attachments(file) # array column; each call pushes one blob + end + redirect(document_path(@document)) +end +``` + +The whole upload contract (declarations, options, routes, cleanup) is in +`app/models/CLAUDE.md` → **Attachments and uploads**. Don't re-implement +blob storage in the controller. + +### Form markup + +The HTML form needs `enctype="multipart/form-data"` and one `` +per uploader field. Anything posted under a name that doesn't match an +uploader is just ignored. + +```erb +
    + + + +
    +``` + +The cap on `req["files"]` array length is `SOLI_MAX_UPLOAD_FILES` (default 32 +per request); excess files are dropped before the action runs. + +## Cookies and sessions + +Cookies are a read-only global; write them with `set_cookie`: + +```soli +@theme = cookies["theme"] ?? "light" # read +set_cookie("theme", "dark") # write (Path=/) +``` + +Sessions are read/write via builtins (storage backend configured in +`config/application.sl`): + +```soli +session_set("user_id", user.id) +let uid = session_get("user_id") # nil if not set +session_has("user_id") # bool +session_delete("user_id") +session_regenerate # after a successful login (security) +session_destroy # on logout +``` + +## Named route helpers + +`resources("posts")` in `config/routes.sl` auto-registers a family of helpers +as globals. Use them — never concatenate URLs by hand. + +| Route | Path helper | URL helper | +|----------------------|-------------------------|-------------------------| +| `GET /posts` | `posts_path()` | `posts_url()` | +| `GET /posts/new` | `new_post_path()` | `new_post_url()` | +| `GET /posts/:id` | `post_path(post)` | `post_url(post)` | +| `GET /posts/:id/edit` | `edit_post_path(post)` | `edit_post_url(post)` | + +Custom routes named with `name: "..."` get the same treatment: +`get("/about", "pages#about", name: "about")` → `about_path()` / `about_url()`. + +`*_path` returns a relative path; `*_url` is the absolute form (and respects +`enable_trust_proxy` if set in `config/application.sl`). + +## Spec location + +Every controller has a sibling spec at `tests/_controller_spec.sl`. +`soli generate controller posts` scaffolds it for you. Use the E2E client: + +```soli +describe("PostsController") do + before_each() do + as_guest() + end + + test("GET /posts returns 200") do + response = get("/posts") + assert_eq(res_status(response), 200) + assert_hash_has_key(assigns(), "posts") + end + + test("POST /posts with invalid params re-renders new") do + response = post("/posts", {}) + assert_eq(res_status(response), 200) + assert_eq(view_path(), "posts/new.html") + end +end +``` + +E2E helpers: `get` / `post` / `put` / `delete` to make requests; `res_status`, +`assigns()` (the `@field` hash exposed to the view), `view_path()`, +`render_template()`, `as_guest()`. + +## Do / Don't + +| Do | Don't | +|----------------------------------------------------------|------------------------------------------------------------------| +| Use named route helpers — `post_path(post)` | Hand-build URLs — `"/posts/" + str(post.id)` | +| Let `Model.find` raise → 404 | Wrap `Model.find` in `try/catch` or `if record.nil?` | +| Whitelist via `_permit_params` before `Model.create` | Pass `params` (or `req["json"]`) straight to `Model.create` | +| Keep actions thin; push rules to the model | Stuff validation / business logic into controller actions | +| Set `@fields` and let the framework auto-render | Repeat `@field` in `render(...)`'s data hash | +| Use `_`-prefixed methods for non-routable helpers | Expose helper methods as public actions | +| Use `find_by` / `first_by` when you want nil-on-miss | Add `if record.nil?` guards after `find` — they're unreachable | +| | `import "../models/*.sl"` — models are auto-loaded | +| | Use `db_query_raw` / backticks here — push raw SQL to the model | + +## Lints that fire here + +- `style/redundant-model-import` — models in `app/models/*.sl` are auto-loaded; + importing them from a controller triggers this. +- `smell/dangerous-server-builtin` — `db_query_raw`, `Trusted.*`, `System.shell`, + and backtick commands are flagged inside controllers. Use the model layer + or a dedicated service object instead. +- `smell/deep-nesting` — keep actions ≤4 levels of nesting. If you're past + that, the action is doing too much. +- `smell/unreachable-code` — typically catches dead branches after an early + `return` or after a `Model.find` nil-check that can never fire. +- `smell/undefined-local` — flags reads of a name that's never assigned in + the action's scope (catches typos that bypass `let`). +- `naming/pascal-case` — class name must be `PascalCase`. +- `naming/snake-case` — action and helper names must be `snake_case`. + +Run on the directory: + +```bash +soli lint app/controllers/ +soli lint app/controllers/posts_controller.sl +``` diff --git a/admin/app/controllers/api_keys_controller.sl b/admin/app/controllers/api_keys_controller.sl new file mode 100644 index 00000000..a8d3266f --- /dev/null +++ b/admin/app/controllers/api_keys_controller.sl @@ -0,0 +1,75 @@ +# API keys - create / list / revoke server API keys. The raw key is only +# returned by SoliDB at creation time, so it is surfaced once in a banner. + +class ApiKeysController < Controller + static { + this.layout = "application" + } + + # GET /api-keys + def index + @title = "API Keys" + this._reset_banners() + this._load() + end + + # POST /api-keys + def create + name = (params["name"] ?? "").trim() + if name.blank? + return this._respond({ "ok": false, "status": 422, "error": "key name is required" }, "") + end + payload = { "name": name } + roles = this._split_csv(params["roles"] ?? "") + payload["roles"] = roles if roles.length() > 0 + scoped = this._split_csv(params["scoped_databases"] ?? "") + payload["scoped_databases"] = scoped if scoped.length() > 0 + result = SolidbClient.post_api(SolidbEndpoints.api_keys(), payload) + raw_key = (result["data"] ?? {})["key"] ?? "" + return this._respond(result, "api key " + name + " created", raw_key) + end + + # DELETE /api-keys/:id + def delete + key_id = params["id"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.api_key(key_id)) + return this._respond(result, "api key revoked") + end + + def _split_csv(text) + return [] if text.trim().blank? + parts = text.split(",").map do |part| part.trim() end + return parts.filter do |part| !part.blank? end + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.api_keys()) + @keys = (result["data"] ?? {})["keys"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + @created_key = "" + end + + def _respond(result, notice, created_key = "") + @title = "API Keys" + this._reset_banners() + @created_key = created_key if result["ok"] + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("api_keys/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("api_keys/index") + end +end diff --git a/admin/app/controllers/collections_controller.sl b/admin/app/controllers/collections_controller.sl new file mode 100644 index 00000000..4922f95b --- /dev/null +++ b/admin/app/controllers/collections_controller.sl @@ -0,0 +1,247 @@ +# Collections - browse / create / truncate / drop collections in a database. + +class CollectionsController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/collections + def index + this._ctx() + @title = "Collections · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/collections + def create + this._ctx() + name = (params["name"] ?? "").trim() + if name.blank? + return this._respond({ "ok": false, "status": 422, "error": "collection name is required" }, "") + end + return this._create_columnar(name) if params["type"] == "columnar" + payload = this._build_create_payload(name) + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "schema must be a valid JSON object" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.collections(@db), payload) + return this._respond(result, "collection " + name + " created") + end + + # Columnar collections live behind their own API and require column + # definitions: [{ "name": "host", "type": "string" }, ...] + def _create_columnar(name) + columns_text = (params["columns"] ?? "").trim() + columns = JSON.parse(columns_text) rescue nil + if columns.nil? || columns.length() == 0 + return this._respond({ "ok": false, "status": 422, + "error": "columnar collections need a JSON array of column definitions" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.columnar(@db), { "name": name, "columns": columns }) + return this._respond(result, "columnar collection " + name + " created") + end + + # Optional creation settings from the modal; nil when the schema textarea + # holds invalid JSON. + def _build_create_payload(name) + payload = { "name": name } + coll_type = (params["type"] ?? "").trim() + payload["type"] = coll_type unless coll_type.blank? + num_shards = (params["num_shards"] ?? "").trim() + payload["numShards"] = num_shards.to_int() unless num_shards.blank? + shard_key = (params["shard_key"] ?? "").trim() + payload["shardKey"] = shard_key unless shard_key.blank? + replication_factor = (params["replication_factor"] ?? "").trim() + payload["replicationFactor"] = replication_factor.to_int() unless replication_factor.blank? + validation_mode = (params["validation_mode"] ?? "").trim() + payload["validationMode"] = validation_mode unless validation_mode.blank? + schema_text = (params["schema"] ?? "").trim() + if !schema_text.blank? + schema = JSON.parse(schema_text) rescue nil + return nil if schema.nil? + payload["schema"] = schema + end + return payload + end + + # GET /databases/:db/collections/:name/stats - HTMX-loaded detail fragment + def stats + this._ctx() + @collection_name = params["name"] ?? "" + result = SolidbClient.get_api(SolidbEndpoints.collection_stats(@db, @collection_name)) + @stats = result["data"] ?? {} + @stats_error = result["ok"] ? "" : (result["error"] ?? "request failed") + return render("collections/_stats", { "layout": false }) + end + + # PUT /databases/:db/collections/:name/truncate + def truncate + this._ctx() + name = params["name"] ?? "" + result = SolidbClient.put_api(SolidbEndpoints.collection_truncate(@db, name)) + return this._respond(result, "collection " + name + " truncated") + end + + # DELETE /databases/:db/collections/:name (?ctype=columnar for columnar) + def delete + this._ctx() + name = params["name"] ?? "" + if params["ctype"] == "columnar" + result = SolidbClient.delete_api(SolidbEndpoints.columnar_collection(@db, name)) + return this._respond(result, "columnar collection " + name + " dropped") + end + result = SolidbClient.delete_api(SolidbEndpoints.collection(@db, name)) + return this._respond(result, "collection " + name + " dropped") + end + + # GET /databases/:db/collections/:name/indexes - HTMX-loaded panel + def indexes + this._ctx_indexes() + return this._render_indexes({ "ok": true }, "") + end + + # POST /databases/:db/collections/:name/indexes + def create_index + this._ctx_indexes() + index_name = (params["index_name"] ?? "").trim() + fields_text = (params["fields"] ?? "").trim() + if index_name.blank? || fields_text.blank? + return this._render_indexes({ "ok": false, "status": 422, + "error": "index name and fields are required" }, "") + end + index_type = (params["index_type"] ?? "persistent").trim() + result = this._create_index_request(index_name, fields_text, index_type) + return this._render_indexes(result, "index " + index_name + " created") + end + + # PUT /databases/:db/collections/:name/indexes/rebuild + def rebuild_indexes + this._ctx_indexes() + result = SolidbClient.put_api(SolidbEndpoints.collection_indexes_rebuild(@db, @collection_name)) + indexed_count = (result["data"] ?? {})["documents_indexed"] ?? 0 + return this._render_indexes(result, "indexes rebuilt (" + str(indexed_count) + " documents)") + end + + # DELETE /databases/:db/collections/:name/indexes/:index_name + # The unified API drops standard / fulltext / geo / ttl indexes by name. + def delete_index + this._ctx_indexes() + index_name = params["index_name"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.collection_index(@db, @collection_name, index_name)) + return this._render_indexes(result, "index " + index_name + " dropped") + end + + # Geo and TTL indexes are created through their own API families; everything + # else (persistent / hash / fulltext / bloom / cuckoo) goes to /index. + def _create_index_request(index_name, fields_text, index_type) + fields = [] + for part in fields_text.split(",") + cleaned = part.trim() + fields.push(cleaned) unless cleaned.blank? + end + if index_type == "geo" + return SolidbClient.post_api(SolidbEndpoints.geo_indexes(@db, @collection_name), + { "name": index_name, "field": fields[0] }) + end + if index_type == "ttl" + expire_text = (params["expire_after_seconds"] ?? "").trim() + if expire_text.blank? + return { "ok": false, "status": 422, "error": "ttl indexes need expire_after_seconds" } + end + return SolidbClient.post_api(SolidbEndpoints.ttl_indexes(@db, @collection_name), + { "name": index_name, "field": fields[0], + "expire_after_seconds": expire_text.to_int() }) + end + return SolidbClient.post_api(SolidbEndpoints.collection_indexes(@db, @collection_name), + { "name": index_name, "fields": fields, + "type": index_type, "unique": params["unique"] == "true" }) + end + + def _ctx_indexes + this._ctx() + @collection_name = params["name"] ?? "" + end + + def _render_indexes(result, notice) + @indexes_error = result["ok"] ? "" : (result["error"] ?? "request failed") + @indexes_notice = result["ok"] ? notice : "" + this._load_indexes() + return render("collections/_indexes", { "layout": false }) + end + + # Merge the three index families (standard+fulltext, geo, ttl) into one + # display list of { name, fields, type, unique, detail } rows. + def _load_indexes + @indexes = [] + result = SolidbClient.get_api(SolidbEndpoints.collection_indexes(@db, @collection_name)) + if !result["ok"] + @indexes_error = result["error"] ?? "request failed" if @indexes_error.blank? + return + end + for entry in ((result["data"] ?? {})["indexes"] ?? []) + type_name = str(entry["index_type"] ?? "persistent").downcase() + @indexes.push({ "name": entry["name"] ?? "", "fields": entry["fields"] ?? [], + "type": type_name, "unique": entry["unique"] ?? false, + "detail": str(entry["indexed_documents"] ?? 0) + " docs" }) + end + geo_result = SolidbClient.get_api(SolidbEndpoints.geo_indexes(@db, @collection_name)) + for entry in ((geo_result["data"] ?? {})["indexes"] ?? []) + @indexes.push({ "name": entry["name"] ?? "", "fields": [entry["field"] ?? ""], + "type": "geo", "unique": false, + "detail": str(entry["indexed_documents"] ?? 0) + " docs" }) + end + ttl_result = SolidbClient.get_api(SolidbEndpoints.ttl_indexes(@db, @collection_name)) + for entry in ((ttl_result["data"] ?? {})["indexes"] ?? []) + @indexes.push({ "name": entry["name"] ?? "", "fields": [entry["field"] ?? ""], + "type": "ttl", "unique": false, + "detail": "expires after " + str(entry["expire_after_seconds"] ?? 0) + "s" }) + end + end + + # Route context: set explicitly per action (before_action hooks are wired + # by a startup-time scan and are unreliable under dev hot-reload). + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + regular = (result["data"] ?? {})["collections"] ?? [] + # Columnar collections appear in the regular list as internal + # `_columnar_` document collections -- hide those and merge the + # real entries from the columnar API instead. + @collections = regular.filter do |coll| !(coll["name"] ?? "").starts_with("_columnar_") end + columnar_result = SolidbClient.get_api(SolidbEndpoints.columnar(@db)) + columnar_entries = (columnar_result["data"] ?? {})["collections"] ?? [] + for entry in columnar_entries + @collections.push({ "name": entry["name"] ?? "", "type": "columnar", + "count": entry["row_count"] ?? 0, "stats": {} }) + end + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Collections · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("collections/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("collections/index") + end +end diff --git a/admin/app/controllers/cron_controller.sl b/admin/app/controllers/cron_controller.sl new file mode 100644 index 00000000..37a038ed --- /dev/null +++ b/admin/app/controllers/cron_controller.sl @@ -0,0 +1,106 @@ +# Cron - scheduled jobs (cron expression -> script on a queue). + +class CronController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/cron + def index + this._ctx() + @title = "Cron · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/cron + def create + this._ctx() + payload = this._build_payload() + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "params must be a JSON object" }, "") + end + cron_name = payload["name"] ?? "" + if cron_name.blank? || (payload["cron_expression"] ?? "") == "" || (payload["script"] ?? "") == "" + return this._respond({ "ok": false, "status": 422, + "error": "name, cron expression and script are required" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.cron_jobs(@db), payload) + return this._respond(result, "cron job " + cron_name + " created") + end + + # PUT /databases/:db/cron/:id + def update + this._ctx() + payload = this._build_payload() + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "params must be a JSON object" }, "") + end + result = SolidbClient.put_api(SolidbEndpoints.cron_job(@db, params["id"] ?? ""), payload) + return this._respond(result, "cron job " + (payload["name"] ?? "") + " updated") + end + + # DELETE /databases/:db/cron/:id + def delete + this._ctx() + result = SolidbClient.delete_api(SolidbEndpoints.cron_job(@db, params["id"] ?? "")) + return this._respond(result, "cron job deleted") + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + # nil when the params textarea holds invalid JSON. + def _build_payload + job_params = {} + params_text = (params["job_params"] ?? "").trim() + if !params_text.blank? + job_params = JSON.parse(params_text) rescue nil + return nil if job_params.nil? + end + payload = { + "name": (params["cron_name"] ?? "").trim(), + "cron_expression": (params["cron_expression"] ?? "").trim(), + "script": (params["script"] ?? "").trim(), + "params": job_params + } + queue = (params["queue"] ?? "").trim() + payload["queue"] = queue unless queue.blank? + priority = (params["priority"] ?? "").trim() + payload["priority"] = priority.to_int() unless priority.blank? + max_retries = (params["max_retries"] ?? "").trim() + payload["max_retries"] = max_retries.to_int() unless max_retries.blank? + return payload + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.cron_jobs(@db)) + @cron_jobs = result["data"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Cron · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("cron/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("cron/index") + end +end diff --git a/admin/app/controllers/databases_controller.sl b/admin/app/controllers/databases_controller.sl new file mode 100644 index 00000000..9ab1476b --- /dev/null +++ b/admin/app/controllers/databases_controller.sl @@ -0,0 +1,59 @@ +# Databases - list / create / drop databases on the SoliDB server. + +class DatabasesController < Controller + static { + this.layout = "application" + } + + # GET /databases + def index + @title = "Databases" + this._reset_banners() + this._load() + end + + # POST /databases + def create + name = (params["name"] ?? "").trim() + if name.blank? + return this._respond({ "ok": false, "status": 422, "error": "database name is required" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.database_create(), { "name": name }) + return this._respond(result, "database " + name + " created") + end + + # DELETE /databases/:db + def delete + db_name = params["db"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.database(db_name)) + return this._respond(result, "database " + db_name + " dropped") + end + + def _load + @database_names = AdminContext.database_names() + end + + # Reading an unset @field raises in Soli, so banner fields are always + # assigned before the view renders. + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + # Shared tail for mutations: set banners, reload the list, re-render the + # section (fragment for HTMX swaps, full page otherwise). + def _respond(result, notice) + @title = "Databases" + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("databases/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("databases/index") + end +end diff --git a/admin/app/controllers/documents_controller.sl b/admin/app/controllers/documents_controller.sl new file mode 100644 index 00000000..cf66314c --- /dev/null +++ b/admin/app/controllers/documents_controller.sl @@ -0,0 +1,191 @@ +# Documents - browse a collection's documents with an SDBQL filter and +# offset/limit pagination, plus create / edit / delete single documents. + +class DocumentsController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/collections/:name/docs + def index + this._ctx() + @title = @collection_name + " · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/collections/:name/docs + def create + this._ctx() + document = this._parse_document() + if document.nil? + return this._respond({ "ok": false, "status": 422, "error": "document must be a JSON object" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.documents(@db, @collection_name), document) + return this._respond(result, "document created") + end + + # PUT /databases/:db/collections/:name/docs/:key + def update + this._ctx() + document = this._parse_document() + if document.nil? + return this._respond({ "ok": false, "status": 422, "error": "document must be a JSON object" }, "") + end + key = params["key"] ?? "" + result = SolidbClient.put_api(SolidbEndpoints.document(@db, @collection_name, key), document) + return this._respond(result, "document " + key + " updated") + end + + # POST /databases/:db/collections/:name/docs/upload - multipart file upload + # into a blob collection. The file's base64 payload goes through the binary + # driver protocol (store_blob), never through an HTTP string body, so the + # bytes arrive exactly as sent. + def upload + this._ctx() + wants_json = (req["headers"]["accept"] ?? "").includes?("application/json") + file = find_uploaded_file(req, "file") + if file.nil? + return render_json({ "ok": false, "error": "no file selected" }) if wants_json + return this._respond({ "ok": false, "status": 422, "error": "no file selected" }, "") + end + blob_id = nil + try + conn = this._driver() + blob_id = conn.store_blob(@collection_name, file["data"], + file["filename"] ?? "upload.bin", + file["content_type"] ?? "application/octet-stream") + catch upload_error + return render_json({ "ok": false, "error": str(upload_error) }) if wants_json + return this._respond({ "ok": false, "status": 500, "error": str(upload_error) }, "") + end + return render_json({ "ok": true, "blob_id": blob_id }) if wants_json + return this._respond({ "ok": true }, "file " + (file["filename"] ?? "") + " uploaded (" + str(blob_id) + ")") + end + + # GET /databases/:db/collections/:name/docs/:key/blob - stream a blob back + # (inline for preview, attachment with ?download=1). + def blob + this._ctx() + key = params["key"] ?? "" + meta = nil + data_base64 = nil + try + conn = this._driver() + meta = conn.get_blob_metadata(@collection_name, key) + data_base64 = conn.get_blob(@collection_name, key) + catch blob_error + meta = nil + end + # The driver returns null (rather than raising) for unknown keys. + if meta == nil || data_base64 == nil + return { "status": 404, "headers": { "Content-Type": "text/plain" }, "body": "blob not found" } + end + # Blob metadata fields vary by upload path: `name`/`type` are canonical, + # `filename`/`content_type` only present on some. + content_type = meta["content_type"] ?? (meta["type"] ?? "application/octet-stream") + filename = meta["filename"] ?? (meta["name"] ?? key) + disposition = params["download"] == "1" ? "attachment" : "inline" + return { + "status": 200, + "headers": { + "Content-Type": content_type, + "Content-Disposition": disposition + "; filename=\"" + filename + "\"" + }, + "body": Base64.decode(data_base64) + } + end + + # DELETE /databases/:db/collections/:name/docs/:key + def delete + this._ctx() + key = params["key"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.document(@db, @collection_name, key)) + return this._respond(result, "document " + key + " deleted") + end + + def _ctx + @db = params["db"] ?? "" + @collection_name = params["name"] ?? "" + @databases = AdminContext.database_names() + @filter = (params["filter"] ?? "").trim() + @limit = this._page_limit() + offset = (params["offset"] ?? "0").to_int() + offset = 0 if offset < 0 + @offset = offset + this._load_collection_type() + end + + # blob collections get upload/download/preview affordances in the view. + def _load_collection_type + result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + collections = (result["data"] ?? {})["collections"] ?? [] + matching = collections.filter do |coll| coll["name"] == @collection_name end + @collection_type = matching.length() > 0 ? (matching[0]["type"] ?? "document") : "document" + end + + # Authenticated binary-protocol connection (byte-safe for blob payloads). + def _driver + conn = Solidb(SolidbClient.host(), @db) + conn.auth(SolidbClient.username(), SolidbClient.password()) + return conn + end + + def _page_limit + limit = (params["limit"] ?? "25").to_int() + return 25 unless [25, 50, 100].includes?(limit) + return limit + end + + # nil when the document textarea holds invalid JSON. + def _parse_document + document = JSON.parse((params["document"] ?? "").trim()) rescue nil + return document + end + + # Runs the listing query: the filter input is spliced as a FILTER clause on + # `doc`, offset/limit are bound. Fetches limit+1 rows to know has-more. + # Blob collections project metadata only -- their docs embed binary chunk + # data that must never be rendered into HTML. + def _load + return_clause = " LIMIT @offset, @batch RETURN doc" + if @collection_type == "blob" + return_clause = " LIMIT @offset, @batch RETURN { \"_key\": doc._key, \"filename\": doc.filename," + + " \"name\": doc.name, \"size\": doc.size, \"content_type\": doc.content_type," + + " \"type\": doc.type, \"chunks\": doc.chunks, \"created\": doc.created }" + end + query = "FOR doc IN " + @collection_name + query = query + " FILTER " + @filter unless @filter.blank? + query = query + return_clause + payload = { "query": query, "bindVars": { "offset": @offset, "batch": @limit + 1 } } + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), payload) + rows = (result["data"] ?? {})["result"] ?? [] + @has_more = rows.length() > @limit + @documents = @has_more ? rows.slice(0, @limit) : rows + @query_error = "" + if !result["ok"] + @query_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = @collection_name + " · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("documents/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("documents/index") + end +end diff --git a/admin/app/controllers/env_vars_controller.sl b/admin/app/controllers/env_vars_controller.sl new file mode 100644 index 00000000..30aafc59 --- /dev/null +++ b/admin/app/controllers/env_vars_controller.sl @@ -0,0 +1,74 @@ +# Env vars - per-database key/value settings exposed to Lua scripts. +# SoliDB's PUT /env/{key} is an upsert, so create and edit share one action. + +class EnvVarsController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/env + def index + this._ctx() + @title = "Env vars · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/env + def upsert + this._ctx() + env_key = (params["env_key"] ?? "").trim() + env_value = params["env_value"] ?? "" + if env_key.blank? + return this._respond({ "ok": false, "status": 422, "error": "key is required" }, "") + end + result = SolidbClient.put_api(SolidbEndpoints.env_var(@db, env_key), { "value": env_value }) + return this._respond(result, "env var " + env_key + " saved") + end + + # DELETE /databases/:db/env/:key + def delete + this._ctx() + result = SolidbClient.delete_api(SolidbEndpoints.env_var(@db, params["key"] ?? "")) + return this._respond(result, "env var deleted") + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + # SoliDB returns a flat { key: value } hash -- flatten to a sorted array + # of rows so the view stays a dumb loop. + def _load + result = SolidbClient.get_api(SolidbEndpoints.env_vars(@db)) + env_map = result["data"] ?? {} + @env_vars = env_map.keys().sort().map do |env_key| + { "key": env_key, "value": env_map[env_key] ?? "" } + end + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Env vars · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("env_vars/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("env_vars/index") + end +end diff --git a/admin/app/controllers/home_controller.sl b/admin/app/controllers/home_controller.sl new file mode 100644 index 00000000..d9adda60 --- /dev/null +++ b/admin/app/controllers/home_controller.sl @@ -0,0 +1,25 @@ +# Home controller - server dashboard + app health endpoint + +class HomeController < Controller + static { + this.layout = "application" + } + + # GET / - server health, cluster info, database count + def index + @title = "Dashboard" + health_result = SolidbClient.get_api(SolidbEndpoints.health()) + @solidb_down = health_result["status"] == 0 + @server_ok = health_result["ok"] + info_result = SolidbClient.get_api(SolidbEndpoints.cluster_info()) + @cluster = info_result["data"] ?? {} + @database_names = AdminContext.database_names() + @flash_error = "" + @flash_notice = "" + end + + # GET /health - app-level health check (used by the proxy) + def health + render_json({ "status": "ok" }) + end +end diff --git a/admin/app/controllers/live_queries_controller.sl b/admin/app/controllers/live_queries_controller.sl new file mode 100644 index 00000000..3e582b11 --- /dev/null +++ b/admin/app/controllers/live_queries_controller.sl @@ -0,0 +1,41 @@ +# Live queries - stream the SoliDB changefeed over a WebSocket the browser +# opens directly against the server. Tokens are short-lived, so the page +# fetches a fresh one from /databases/:db/live/token on every (re)connect. + +class LiveQueriesController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/live + def show + this._ctx() + @title = "Live · " + @db + @flash_error = "" + @flash_notice = "" + @solidb_down = false + collections_result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + @collections = (collections_result["data"] ?? {})["collections"] ?? [] + @collection_names = @collections.map do |coll| coll["name"] ?? "" end + end + + # GET /databases/:db/live/token - JSON consumed by the page's JS + def token + this._ctx() + live_token = SolidbClient.livequery_token() + if live_token.blank? + return render_json({ "ok": false, "error": "could not mint a livequery token (is SoliDB up?)" }) + end + return render_json({ + "ok": true, + "token": live_token, + "ws_url": SolidbClient.public_ws_url() + "/_api/ws/changefeed", + "database": @db + }) + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end +end diff --git a/admin/app/controllers/materialized_views_controller.sl b/admin/app/controllers/materialized_views_controller.sl new file mode 100644 index 00000000..6f2ade64 --- /dev/null +++ b/admin/app/controllers/materialized_views_controller.sl @@ -0,0 +1,124 @@ +# Materialized views - list / create / refresh / drop per database. +# +# SoliDB manages these through SDBQL statements (CREATE/REFRESH MATERIALIZED +# VIEW) run on the cursor API. Metadata lives in the per-db `_views` system +# collection; the server stores the view query as a serialized AST, so on +# create we annotate the metadata doc with the original SDBQL source +# (query_text) to keep definitions readable in the UI. There is no DROP +# statement: dropping = delete the metadata doc + drop the backing collection. + +class MaterializedViewsController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/views + def index + this._ctx() + @title = "Views · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/views + def create + this._ctx() + name = (params["name"] ?? "").trim() + query_text = (params["query"] ?? "").trim() + if !this._valid_view_name(name) + return this._respond({ "ok": false, "status": 422, + "error": "view name must be an identifier (letters, digits, underscore)" }, "") + end + if query_text.blank? + return this._respond({ "ok": false, "status": 422, "error": "view query is required" }, "") + end + statement = "CREATE MATERIALIZED VIEW " + name + " AS " + query_text + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), { "query": statement }) + this._annotate_query_text(name, query_text) if result["ok"] + return this._respond(result, "view " + name + " created") + end + + # PUT /databases/:db/views/:name/refresh + def refresh + this._ctx() + name = params["name"] ?? "" + if !this._valid_view_name(name) + return this._respond({ "ok": false, "status": 422, "error": "invalid view name" }, "") + end + statement = "REFRESH MATERIALIZED VIEW " + name + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), { "query": statement }) + refreshed_count = (result["data"] ?? {})["inserted"] ?? 0 + return this._respond(result, "view " + name + " refreshed (" + str(refreshed_count) + " documents)") + end + + # DELETE /databases/:db/views/:name + def delete + this._ctx() + name = params["name"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.document(@db, "_views", name)) + if !result["ok"] + return this._respond(result, "") + end + # Backing collection may already be gone - the metadata doc was the + # source of truth, so a miss here is not an error worth surfacing. + SolidbClient.delete_api(SolidbEndpoints.collection(@db, name)) + return this._respond(result, "view " + name + " dropped") + end + + # View names are spliced into SDBQL statements - restrict to identifiers + # so a crafted name can't smuggle extra clauses in. + def _valid_view_name(name) + return false if name.blank? + cleaned = name.gsub("[^A-Za-z0-9_]", "") + return false if cleaned != name + first_char = name.substring(0, 1) + return !"0123456789".includes?(first_char) + end + + # Best-effort: PUT merges fields into the metadata doc, preserving the + # stored AST. A failure here only loses the pretty definition display. + def _annotate_query_text(name, query_text) + SolidbClient.put_api(SolidbEndpoints.document(@db, "_views", name), { "query_text": query_text }) + end + + # Route context: set explicitly per action (before_action hooks are wired + # by a startup-time scan and are unreliable under dev hot-reload). + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + def _load + list_query = "FOR v IN _views FILTER v.type == \"materialized\" SORT v._key ASC " + + "RETURN { name: v._key, created_at: v.created_at, query_text: v.query_text }" + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), { "query": list_query }) + # No _views collection yet just means no views were ever created. + @views = result["ok"] ? ((result["data"] ?? {})["result"] ?? []) : [] + counts = {} + collections_result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + for coll in ((collections_result["data"] ?? {})["collections"] ?? []) + counts[coll["name"] ?? ""] = coll["count"] ?? 0 + end + @view_counts = counts + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Views · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("materialized_views/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("materialized_views/index") + end +end diff --git a/admin/app/controllers/query_controller.sl b/admin/app/controllers/query_controller.sl new file mode 100644 index 00000000..d9cf50d5 --- /dev/null +++ b/admin/app/controllers/query_controller.sl @@ -0,0 +1,124 @@ +# Query console - run SDBQL queries and explain plans against a database. +# Also exposes the slow query log: SoliDB records every query slower than +# 100ms into the per-db _slow_queries system collection. + +class QueryController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/query + def show + this._ctx() + @title = "Query · " + @db + @flash_error = "" + @flash_notice = "" + @solidb_down = false + collections_result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + raw_collections = (collections_result["data"] ?? {})["collections"] ?? [] + @collection_names = raw_collections.map do |coll| coll["name"] ?? "" end + end + + # POST /databases/:db/query/run - returns the results fragment (HTMX) + def run + this._ctx() + payload = this._build_payload() + if payload.nil? + return this._render_results({ "ok": false, "error": "bind vars must be a JSON object" }) + end + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), payload) + return this._render_results(result) + end + + # POST /databases/:db/query/explain - returns the explain fragment (HTMX) + def explain + this._ctx() + payload = this._build_payload() + if payload.nil? + return this._render_explain({ "ok": false, "error": "bind vars must be a JSON object" }) + end + result = SolidbClient.post_api(SolidbEndpoints.explain(@db), payload) + return this._render_explain(result) + end + + # GET /databases/:db/query/slow + def slow + this._ctx() + @title = "Slow queries · " + @db + @flash_error = "" + @flash_notice = "" + @solidb_down = false + this._load_slow() + end + + # GET /databases/:db/query/slow/count - sidebar badge fragment (HTMX) + def slow_count + this._ctx() + @slow_count = this._count_slow() + return render("query/_slow_badge", { "layout": false }) + end + + # PUT /databases/:db/query/slow/clear + def clear_slow + this._ctx() + @title = "Slow queries · " + @db + result = SolidbClient.put_api(SolidbEndpoints.collection_truncate(@db, "_slow_queries")) + @flash_error = result["ok"] ? "" : (result["error"] ?? "request failed") + @flash_notice = result["ok"] ? "slow query log cleared" : "" + @solidb_down = (result["status"] ?? -1) == 0 + this._load_slow() + return render("query/slow", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("query/slow") + end + + # Route context: set explicitly per action (before_action hooks are wired + # by a startup-time scan and are unreliable under dev hot-reload). + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + # nil when the bind-vars textarea holds invalid JSON. + def _build_payload + bind_text = (params["bind_vars"] ?? "").trim() + bind_vars = {} + if !bind_text.blank? + bind_vars = JSON.parse(bind_text) rescue nil + return nil if bind_vars.nil? + end + return { "query": params["query"] ?? "", "bindVars": bind_vars, "batchSize": 200 } + end + + def _render_results(result) + @query_error = result["ok"] ? "" : (result["error"] ?? "request failed") + data = result["data"] ?? {} + @rows = data["result"] ?? [] + @meta = data + return render("query/_results", { "layout": false }) + end + + def _render_explain(result) + @explain_error = result["ok"] ? "" : (result["error"] ?? "request failed") + @plan = result["data"] ?? {} + return render("query/_explain", { "layout": false }) + end + + def _load_slow + statement = "FOR q IN _slow_queries SORT q.timestamp DESC LIMIT 200 RETURN q" + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), { "query": statement }) + @slow_queries = (result["data"] ?? {})["result"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + # 0 when the _slow_queries collection is missing or the query fails -- + # the badge just stays hidden. + def _count_slow + statement = "FOR q IN _slow_queries COLLECT WITH COUNT INTO total RETURN total" + result = SolidbClient.post_api(SolidbEndpoints.cursor(@db), { "query": statement }) + counts = (result["data"] ?? {})["result"] ?? [] + return counts[0] ?? 0 + end +end diff --git a/admin/app/controllers/queues_controller.sl b/admin/app/controllers/queues_controller.sl new file mode 100644 index 00000000..b3f6943d --- /dev/null +++ b/admin/app/controllers/queues_controller.sl @@ -0,0 +1,108 @@ +# Queues - background job queues: stats, job lists, enqueue, cancel, run-now. + +class QueuesController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/queues + def index + this._ctx() + @title = "Queues · " + @db + this._reset_banners() + this._load() + end + + # GET /databases/:db/queues/:name/jobs - HTMX-loaded fragment + def jobs + this._ctx() + @queue_name = params["name"] ?? "" + result = SolidbClient.get_api(SolidbEndpoints.queue_jobs(@db, @queue_name)) + @jobs = (result["data"] ?? {})["jobs"] ?? [] + @jobs_error = result["ok"] ? "" : (result["error"] ?? "request failed") + return render("queues/_jobs", { "layout": false }) + end + + # POST /databases/:db/queues/enqueue (queue name comes from the form) + def enqueue + this._ctx() + queue_name = (params["queue"] ?? "").trim() + queue_name = "default" if queue_name.blank? + payload = this._build_enqueue_payload() + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "params must be a JSON object" }, "") + end + job_script = payload["script"] ?? "" + if job_script.blank? + return this._respond({ "ok": false, "status": 422, "error": "script is required" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.queue_enqueue(@db, queue_name), payload) + return this._respond(result, "job enqueued on " + queue_name) + end + + # DELETE /databases/:db/queues/jobs/:id + def cancel_job + this._ctx() + result = SolidbClient.delete_api(SolidbEndpoints.queue_job(@db, params["id"] ?? "")) + return this._respond(result, "job cancelled") + end + + # POST /databases/:db/queues/jobs/:id/run-now + def run_now + this._ctx() + result = SolidbClient.post_api(SolidbEndpoints.queue_job_run_now(@db, params["id"] ?? "")) + return this._respond(result, "job scheduled to run now") + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + # nil when the params textarea holds invalid JSON. + def _build_enqueue_payload + job_params = {} + params_text = (params["job_params"] ?? "").trim() + if !params_text.blank? + job_params = JSON.parse(params_text) rescue nil + return nil if job_params.nil? + end + payload = { "script": (params["script"] ?? "").trim(), "params": job_params } + priority = (params["priority"] ?? "").trim() + payload["priority"] = priority.to_int() unless priority.blank? + max_retries = (params["max_retries"] ?? "").trim() + payload["max_retries"] = max_retries.to_int() unless max_retries.blank? + run_at = (params["run_at"] ?? "").trim() + payload["run_at"] = run_at unless run_at.blank? + return payload + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.queues(@db)) + @queues = result["data"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Queues · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("queues/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("queues/index") + end +end diff --git a/admin/app/controllers/roles_controller.sl b/admin/app/controllers/roles_controller.sl new file mode 100644 index 00000000..253319d9 --- /dev/null +++ b/admin/app/controllers/roles_controller.sl @@ -0,0 +1,102 @@ +# Roles - RBAC role management (builtin roles are read-only). + +class RolesController < Controller + static { + this.layout = "application" + } + + # GET /roles + def index + @title = "Roles" + this._reset_banners() + this._load() + end + + # GET /roles/:name + def show + @title = "Role · " + (params["name"] ?? "") + this._reset_banners() + result = SolidbClient.get_api(SolidbEndpoints.role(params["name"] ?? "")) + @role = result["data"] ?? {} + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + # POST /roles + def create + name = (params["name"] ?? "").trim() + payload = this._build_payload(name) + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "permissions must be a JSON array" }, "") + end + if name.blank? + return this._respond({ "ok": false, "status": 422, "error": "role name is required" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.roles(), payload) + return this._respond(result, "role " + name + " created") + end + + # PUT /roles/:name + def update + name = params["name"] ?? "" + payload = this._build_payload(name) + if payload.nil? + return this._respond({ "ok": false, "status": 422, "error": "permissions must be a JSON array" }, "") + end + result = SolidbClient.put_api(SolidbEndpoints.role(name), payload) + return this._respond(result, "role " + name + " updated") + end + + # DELETE /roles/:name + def delete + name = params["name"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.role(name)) + return this._respond(result, "role " + name + " deleted") + end + + # nil when the permissions textarea holds invalid JSON. + def _build_payload(name) + permissions = [] + permissions_text = (params["permissions"] ?? "").trim() + if !permissions_text.blank? + permissions = JSON.parse(permissions_text) rescue nil + return nil if permissions.nil? + end + return { + "name": name, + "description": params["description"] ?? "", + "permissions": permissions + } + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.roles()) + @roles = result["data"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Roles" + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("roles/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("roles/index") + end +end diff --git a/admin/app/controllers/scripts_controller.sl b/admin/app/controllers/scripts_controller.sl new file mode 100644 index 00000000..63ad20da --- /dev/null +++ b/admin/app/controllers/scripts_controller.sl @@ -0,0 +1,127 @@ +# Lua scripts - CRUD for the database's custom Lua endpoints. + +class ScriptsController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/scripts + def index + this._ctx() + @title = "Scripts · " + @db + this._reset_banners() + this._load() + end + + # GET /databases/:db/scripts/new + def new + this._ctx() + @title = "New script · " + @db + this._reset_banners() + @script = {} + end + + # POST /databases/:db/scripts + def create + this._ctx() + payload = this._build_payload() + script_name = payload["name"] ?? "" + script_path = payload["path"] ?? "" + if script_name.blank? || script_path.blank? + return this._respond({ "ok": false, "status": 422, "error": "name and path are required" }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.scripts(@db), payload) + return this._respond(result, "script " + payload["name"] + " created") + end + + # GET /databases/:db/scripts/:id + def show + this._ctx() + this._reset_banners() + this._load_script() + @title = "Script · " + (@script["name"] ?? "") + end + + # GET /databases/:db/scripts/:id/edit + def edit + this._ctx() + this._reset_banners() + this._load_script() + @title = "Edit script · " + (@script["name"] ?? "") + end + + # PUT /databases/:db/scripts/:id + def update + this._ctx() + payload = this._build_payload() + result = SolidbClient.put_api(SolidbEndpoints.script(@db, params["id"] ?? ""), payload) + return this._respond(result, "script " + (payload["name"] ?? "") + " updated") + end + + # DELETE /databases/:db/scripts/:id + def delete + this._ctx() + result = SolidbClient.delete_api(SolidbEndpoints.script(@db, params["id"] ?? "")) + return this._respond(result, "script deleted") + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + def _build_payload + methods = [] + methods.push("GET") if params["method_get"] == "on" + methods.push("POST") if params["method_post"] == "on" + methods.push("PUT") if params["method_put"] == "on" + methods.push("DELETE") if params["method_delete"] == "on" + methods = ["GET"] if methods.length() == 0 + return { + "name": (params["name"] ?? "").trim(), + "path": (params["path"] ?? "").trim(), + "methods": methods, + "code": params["code"] ?? "", + "description": params["description"] ?? "", + "service": (params["service"] ?? "").trim() + } + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.scripts(@db)) + @scripts = (result["data"] ?? {})["scripts"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _load_script + result = SolidbClient.get_api(SolidbEndpoints.script(@db, params["id"] ?? "")) + @script = result["data"] ?? {} + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Scripts · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("scripts/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("scripts/index") + end +end diff --git a/admin/app/controllers/triggers_controller.sl b/admin/app/controllers/triggers_controller.sl new file mode 100644 index 00000000..13e9a2f8 --- /dev/null +++ b/admin/app/controllers/triggers_controller.sl @@ -0,0 +1,119 @@ +# Triggers - fire a script or webhook when documents change in a collection. + +class TriggersController < Controller + static { + this.layout = "application" + } + + # GET /databases/:db/triggers + def index + this._ctx() + @title = "Triggers · " + @db + this._reset_banners() + this._load() + end + + # POST /databases/:db/triggers + def create + this._ctx() + payload = this._build_payload() + trigger_name = payload["name"] ?? "" + events = payload["events"] ?? [] + script_target = payload["script_path"] ?? "" + webhook_target = payload["webhook_url"] ?? "" + has_target = script_target != "" || webhook_target != "" + if trigger_name.blank? || (payload["collection"] ?? "") == "" || events.length() == 0 || !has_target + validation_error = "name, collection, at least one event and a script or webhook target are required" + return this._respond({ "ok": false, "status": 422, "error": validation_error }, "") + end + result = SolidbClient.post_api(SolidbEndpoints.triggers(@db), payload) + return this._respond(result, "trigger " + trigger_name + " created") + end + + # POST /databases/:db/triggers/:id/toggle + def toggle + this._ctx() + result = SolidbClient.post_api(SolidbEndpoints.trigger_toggle(@db, params["id"] ?? "")) + now_enabled = (result["data"] ?? {})["enabled"] ?? false + return this._respond(result, now_enabled ? "trigger enabled" : "trigger disabled") + end + + # DELETE /databases/:db/triggers/:id + def delete + this._ctx() + result = SolidbClient.delete_api(SolidbEndpoints.trigger(@db, params["id"] ?? "")) + return this._respond(result, "trigger deleted") + end + + def _ctx + @db = params["db"] ?? "" + @databases = AdminContext.database_names() + end + + def _build_payload + # Checkboxes only submit when checked -- absent means off. + insert_flag = params["event_insert"] ?? "" + update_flag = params["event_update"] ?? "" + delete_flag = params["event_delete"] ?? "" + events = [] + events.push("insert") if insert_flag != "" + events.push("update") if update_flag != "" + events.push("delete") if delete_flag != "" + payload = { + "name": (params["trigger_name"] ?? "").trim(), + "collection": (params["collection"] ?? "").trim(), + "events": events + } + script_path = (params["script_path"] ?? "").trim() + payload["script_path"] = script_path unless script_path.blank? + webhook_url = (params["webhook_url"] ?? "").trim() + payload["webhook_url"] = webhook_url unless webhook_url.blank? + webhook_secret = (params["webhook_secret"] ?? "").trim() + payload["webhook_secret"] = webhook_secret unless webhook_secret.blank? + filter_expr = (params["filter"] ?? "").trim() + payload["filter"] = filter_expr unless filter_expr.blank? + queue = (params["queue"] ?? "").trim() + payload["queue"] = queue unless queue.blank? + priority = (params["priority"] ?? "").trim() + payload["priority"] = priority.to_int() unless priority.blank? + max_retries = (params["max_retries"] ?? "").trim() + payload["max_retries"] = max_retries.to_int() unless max_retries.blank? + return payload + end + + def _load + result = SolidbClient.get_api(SolidbEndpoints.triggers(@db)) + @triggers = (result["data"] ?? {})["triggers"] ?? [] + if !result["ok"] + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + # Collection names feed the target-collection dropdown in the create + # modal. Internal collections (_triggers, _env, _columnar_*...) are not + # valid trigger targets -- hide them. + collections_result = SolidbClient.get_api(SolidbEndpoints.collections(@db)) + all_collections = (collections_result["data"] ?? {})["collections"] ?? [] + visible = all_collections.filter do |coll| !(coll["name"] ?? "").starts_with("_") end + @collection_names = visible.map do |coll| coll["name"] ?? "" end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Triggers · " + @db + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("triggers/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("triggers/index") + end +end diff --git a/admin/app/controllers/users_controller.sl b/admin/app/controllers/users_controller.sl new file mode 100644 index 00000000..4e67266a --- /dev/null +++ b/admin/app/controllers/users_controller.sl @@ -0,0 +1,88 @@ +# Users - manage SoliDB users and their role assignments. + +class UsersController < Controller + static { + this.layout = "application" + } + + # GET /users + def index + @title = "Users" + this._reset_banners() + this._load() + end + + # POST /users + def create + username = (params["username"] ?? "").trim() + password = params["password"] ?? "" + if username.blank? || password.blank? + return this._respond({ "ok": false, "status": 422, "error": "username and password are required" }, "") + end + payload = { "username": username, "password": password } + initial_role = (params["initial_role"] ?? "").trim() + payload["initial_role"] = initial_role unless initial_role.blank? + result = SolidbClient.post_api(SolidbEndpoints.users(), payload) + return this._respond(result, "user " + username + " created") + end + + # DELETE /users/:username + def delete + username = params["username"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.user(username)) + return this._respond(result, "user " + username + " deleted") + end + + # POST /users/:username/roles + def add_role + username = params["username"] ?? "" + role = (params["role"] ?? "").trim() + if role.blank? + return this._respond({ "ok": false, "status": 422, "error": "role is required" }, "") + end + payload = { "role": role } + database = (params["database"] ?? "").trim() + payload["database"] = database unless database.blank? + result = SolidbClient.post_api(SolidbEndpoints.user_roles(username), payload) + return this._respond(result, "role " + role + " granted to " + username) + end + + # DELETE /users/:username/roles/:role + def remove_role + username = params["username"] ?? "" + role = params["role"] ?? "" + result = SolidbClient.delete_api(SolidbEndpoints.user_role(username, role)) + return this._respond(result, "role " + role + " revoked from " + username) + end + + def _load + users_result = SolidbClient.get_api(SolidbEndpoints.users()) + @users = (users_result["data"] ?? {})["users"] ?? [] + roles_result = SolidbClient.get_api(SolidbEndpoints.roles()) + @role_names = (roles_result["data"] ?? []).map do |role| role["name"] end + if !users_result["ok"] + @flash_error = users_result["error"] ?? "request failed" + @solidb_down = (users_result["status"] ?? -1) == 0 + end + end + + def _reset_banners + @flash_error = "" + @flash_notice = "" + @solidb_down = false + end + + def _respond(result, notice) + @title = "Users" + this._reset_banners() + if result["ok"] + @flash_notice = notice + else + @flash_error = result["error"] ?? "request failed" + @solidb_down = (result["status"] ?? -1) == 0 + end + this._load() + return render("users/index", { "layout": false }) if req["headers"]["hx-request"] == "true" + return render("users/index") + end +end diff --git a/admin/app/helpers/application_helper.sl b/admin/app/helpers/application_helper.sl new file mode 100644 index 00000000..c5ec07ae --- /dev/null +++ b/admin/app/helpers/application_helper.sl @@ -0,0 +1,82 @@ +# Application-wide view helpers + +# Truncate text to a maximum length with ellipsis +def truncate_text(text: String, length: Int, suffix: String) -> String + if text.length() <= length + return text + end + return text.substring(0, length - suffix.length()) + suffix +end + +# Capitalize first letter of a string +def capitalize(text: String) -> String + if text.length() == 0 + return text + end + return text.substring(0, 1).upcase() + text.substring(1, text.length()) +end + +# SEC-012: Reject href values that would let an attacker run JS through +# `javascript:` (or similar) URL schemes. HTML-escaping the URL is *not* +# enough — the browser still parses `javascript:alert(1)` inside an +# `href` attribute. Mirror the allowlist used by the markdown sanitiser. +def _is_safe_link_url(url) + lower = url.downcase() + if lower.starts_with("http://") or lower.starts_with("https://") or lower.starts_with("mailto:") + return true + end + if lower.starts_with("/") or lower.starts_with("#") or lower.starts_with("?") + return true + end + # No allowed scheme prefix; treat as relative *only* if there is no + # scheme separator (`:`) before the first /?#. Anything else is a + # custom scheme like javascript:/data: and must be refused. + cut = len(lower) + s = lower.index_of("/") + if s != -1 and s < cut + cut = s + end + q = lower.index_of("?") + if q != -1 and q < cut + cut = q + end + h = lower.index_of("#") + if h != -1 and h < cut + cut = h + end + return !lower.substring(0, cut).contains(":") +end + +def _safe_link_url(url) + if _is_safe_link_url(url) + return url + end + return "#" +end + +# Generate an HTML link +def link_to(text: String, url: String) -> String + return "" + html_escape(text) + "" +end + +# Generate an HTML link with CSS class +def link_to_class(text: String, url: String, css_class: String) -> String + opening = "" + return opening + html_escape(text) + "" +end + +# Pluralize a word based on count +def pluralize(count: Int, singular: String, plural: String) -> String + if count == 1 + return str(count) + " " + singular + end + return str(count) + " " + plural +end + +# Simple pluralize (adds 's') +def pluralize_simple(count: Int, word: String) -> String + if count == 1 + return str(count) + " " + word + end + return str(count) + " " + word + "s" +end diff --git a/admin/app/helpers/solidb_helper.sl b/admin/app/helpers/solidb_helper.sl new file mode 100644 index 00000000..bcd70028 --- /dev/null +++ b/admin/app/helpers/solidb_helper.sl @@ -0,0 +1,83 @@ +# View formatters for SoliDB admin values (bytes, uptime, JSON blobs). + +def fmt_bytes(num_bytes) + return "0 B" if num_bytes.nil? || num_bytes <= 0 + return str(num_bytes) + " B" if num_bytes < 1024 + kb = num_bytes / 1024.0 + return str((kb * 10.0).round() / 10.0) + " KB" if kb < 1024 + mb = kb / 1024.0 + return str((mb * 10.0).round() / 10.0) + " MB" if mb < 1024 + gb = mb / 1024.0 + return str((gb * 10.0).round() / 10.0) + " GB" +end + +def fmt_uptime(seconds) + return "-" if seconds.nil? || seconds < 0 + return str(seconds) + "s" if seconds < 60 + minutes = seconds / 60 + return str(minutes) + "m" if minutes < 60 + hours = minutes / 60 + return str(hours) + "h " + str(minutes % 60) + "m" if hours < 24 + days = hours / 24 + return str(days) + "d " + str(hours % 24) + "h" +end + +def fmt_percent(value) + return "-" if value.nil? + return str((value * 10.0).round() / 10.0) + "%" +end + +def json_compact(value) + return "" if value.nil? + compact = JSON.stringify(value) rescue "" + return compact ?? "" +end + +def fmt_us(microseconds) + return "-" if microseconds.nil? + return str(microseconds) + " µs" if microseconds < 1000 + milliseconds = microseconds / 1000.0 + return str((milliseconds * 100.0).round() / 100.0) + " ms" if milliseconds < 1000 + seconds = milliseconds / 1000.0 + return str((seconds * 100.0).round() / 100.0) + " s" +end + +# The explain API returns filter expressions as Rust AST debug strings +# ("BinaryOp { left: FieldAccess(Variable(\"c\"), \"age\"), ... }"). +# Rewrite the common nodes back into SDBQL-ish syntax; unknown nodes are +# left as-is, so a partial rewrite still beats the raw dump. +def explain_expr(raw) + return "" if raw.nil? + text = raw + # Leaves. BindVariable must run before Variable (it contains it). + text = text.gsub("BindVariable\\(\"([\\w]+)\"\\)", "@$1") + text = text.gsub("Variable\\(\"([\\w]+)\"\\)", "$1") + text = text.gsub("Literal\\(Bool\\((\\w+)\\)\\)", "$1") + text = text.gsub("Literal\\(Int\\((-?\\d+)\\)\\)", "$1") + text = text.gsub("Literal\\(Float\\((-?[\\d.]+)\\)\\)", "$1") + text = text.gsub("Literal\\(String\\((\"[^\"]*\")\\)\\)", "$1") + text = text.gsub("Literal\\(Null\\)", "null") + # Composite nodes, innermost first - repeat until nothing changes. + prev = "" + while prev != text + prev = text + text = text.gsub("FieldAccess\\(([\\w.@\"\\[\\]]+), \"([\\w]+)\"\\)", "$1.$2") + text = text.gsub("FunctionCall\\(\"([\\w]+)\", \\[([^\\[\\]{}]*)\\]\\)", "$1($2)") + text = text.gsub("UnaryOp \\{ op: Not, operand: ([^{}]*?) \\}", "(NOT $1)") + text = text.gsub("UnaryOp \\{ op: Negate, operand: ([^{}]*?) \\}", "(-$1)") + text = text.gsub("BinaryOp \\{ left: ([^{}]*?), op: ([\\w]+), right: ([^{}]*?) \\}", "($1 $2 $3)") + end + operator_symbols = { + " Equal ": " == ", " NotEqual ": " != ", " LessThan ": " < ", + " LessThanOrEqual ": " <= ", " GreaterThan ": " > ", " GreaterThanOrEqual ": " >= ", + " In ": " IN ", " NotIn ": " NOT IN ", " And ": " AND ", " Or ": " OR ", + " Add ": " + ", " Subtract ": " - ", " Multiply ": " * ", " Divide ": " / ", + " Modulus ": " % ", " Exponent ": " ^ ", " Like ": " LIKE ", " NotLike ": " NOT LIKE ", + " RegEx ": " =~ ", " NotRegEx ": " !~ ", " FuzzyEqual ": " ~= " + } + for operator_name in operator_symbols.keys() + text = text.replace(operator_name, operator_symbols[operator_name]) + end + return text +end + diff --git a/admin/app/middleware/CLAUDE.md b/admin/app/middleware/CLAUDE.md new file mode 100644 index 00000000..7c64aa39 --- /dev/null +++ b/admin/app/middleware/CLAUDE.md @@ -0,0 +1,259 @@ +# Middleware + +Middleware sits between the HTTP server and your controller. Use it for +cross-cutting concerns: auth, logging, request munging, response headers, +rate limits. + +Files live in `app/middleware/*.sl`. The loader scans each `.sl` file at boot, +registers every top-level `def` / `fn` declaration as a middleware function, +and orders them by per-function `# order:` directives. + +## A minimal middleware + +```soli +# app/middleware/auth.sl + +# order: 20 +# scope_only: true + +def authenticate(req) + api_key = req["headers"]["x-api-key"] ?? "" + if api_key == "" + return { + "continue": false, + "response": { + "status": 401, + "headers": { "Content-Type": "application/json" }, + "body": { "error": "Unauthorized" }.to_json + } + } + end + + return { "continue": true, "request": req } +end +``` + +Two things to internalize: + +1. The **return shape** decides whether the request proceeds or is + short-circuited. +2. The **comment directives** above the `def` line determine the function's + order and scoping. + +## Return shape + +Every middleware must return a hash. Two valid shapes: + +| Shape | Effect | +|---------------------------------------------------------|-------------------------------------------------| +| `{ "continue": true, "request": req }` | Proceed to the next middleware / handler. | +| `{ "continue": false, "response": { "status": ..., "body": ..., "headers": {...} } }` | Stop here; return that response. | + +When proceeding, you can pass back a **modified copy** of `req` — that's how +middleware feeds data forward: + +```soli +def attach_request_id(req) + req["request_id"] = uuid() # new field for downstream layers + return { "continue": true, "request": req } +end +``` + +Downstream middleware and the controller see the updated `req`. Don't mutate +`req` in place — return the modified hash via the `"request"` key. + +## File-top directives + +Comments **immediately above** a `def` / `fn` line attach to that function. +Three are supported (see `docs/middleware.md` for the canonical reference): + +| Directive | Default | Meaning | +|----------------------|----------|--------------------------------------------------------------------| +| `# order: N` | `100` | Lower numbers run first. Use `10`-`20` for auth, `90`+ for tail. | +| `# global_only: true`| `false` | Always runs on every request; cannot be excluded by a scope block. | +| `# scope_only: true` | `false` | Only runs when explicitly wrapped in `middleware("name", -> {...})`.| + +Both `#` and `//` are accepted as the comment marker (the loader tries +each prefix). + +There is **no** `# methods:` or `# paths:` directive — restrict by `req["method"]` +or `req["path"]` inside the function body, or use route-level scoping (next +section). + +### Function names != filenames + +The loader registers each function by its **function name**, not the file +name. A file `app/middleware/auth.sl` containing `def authenticate(req) ...` +exposes the middleware as `authenticate`, not `auth`. Use the function name +when scoping in `config/routes.sl`: + +```soli +middleware("authenticate", -> { + get("/admin", "admin#index") +}) +``` + +You can also put multiple middleware functions in one file — each gets its +own directives from the comments immediately above its `def` line. + +```soli +# app/middleware/api.sl + +# order: 10 +def cors(req) + # ... +end + +# order: 50 +# scope_only: true +def require_api_key(req) + # ... +end +``` + +Private functions (names starting with `_`) are skipped — useful for helpers +inside the file. + +## Execution order + +Middleware runs in **ascending `order` value** — `order: 10` runs before +`order: 50` runs before `order: 100`. Default is `100`. Pick small numbers +for things that should see the raw request (auth, rate-limit), large numbers +for things that wrap the response (compression, logging). + +Recommended ranges: + +| Range | Typical use | +|-------------|---------------------------------------------------| +| `1-20` | Logging, request ID assignment, CORS pre-flight. | +| `20-40` | Authentication, session bootstrap. | +| `40-80` | Authorization, feature flags, body parsing. | +| `80-100` | Catch-alls; ad-hoc per-app middleware. | + +Ties between middleware with the same `order` are broken by load order +(filesystem walk order). Be explicit with `order:` to avoid relying on that. + +## Scoping in `config/routes.sl` + +`scope_only` middleware run only inside `middleware("name", -> { ... })` +blocks. Routes outside the block are unaffected. + +```soli +# config/routes.sl + +get("/", "home#index") # no auth +get("/health", "home#health") # no auth + +middleware("authenticate", -> { + get("/admin", "admin#index") + resources("admin/users") + resources("admin/posts") +}) +``` + +A non-`scope_only` middleware (default) always runs — `middleware("name", ...)` +blocks only opt **in** additional `scope_only` middleware; they don't opt +**out** of global ones. + +Combine multiple scoped middleware by nesting: + +```soli +middleware("authenticate", -> { + middleware("require_admin", -> { + resources("admin/users") + }) +}) +``` + +## Common patterns + +### Auth — short-circuit on failure, forward `current_user` on success + +```soli +# order: 20 +# scope_only: true + +def authenticate(req) + user_id = session_get("user_id") + if user_id == nil + return { + "continue": false, + "response": { "status": 302, "headers": { "Location": "/login" }, "body": "" } + } + end + + req["current_user"] = User.find_by("id", user_id) + return { "continue": true, "request": req } +end +``` + +The handler reads `req["current_user"]` without needing to look it up again. + +### Logging — runs on every request + +```soli +# order: 5 +# global_only: true + +def request_log(req) + print("#{req[\"method\"]} #{req[\"path\"]}") + return { "continue": true, "request": req } +end +``` + +### Rate limit — short-circuit with 429 + +```soli +# order: 15 + +def rate_limit(req) + key = req["headers"]["x-api-key"] ?? req["remote_ip"] + if not _allow(key) + return { + "continue": false, + "response": { "status": 429, "body": "Too many requests" } + } + end + return { "continue": true, "request": req } +end + +def _allow(key) # private, not registered as middleware + # ... bucket logic ... +end +``` + +## Testing middleware + +E2E specs in `tests/` exercise middleware as a side-effect of hitting a route: + +```soli +describe("authenticate middleware") do + test("denies unauthenticated requests to /admin") do + response = get("/admin") + assert_eq(res_status(response), 302) + assert_eq(res_headers(response)["Location"], "/login") + end + + test("allows authenticated requests to /admin") do + as_user(1) + response = get("/admin") + assert_eq(res_status(response), 200) + end +end +``` + +There's no built-in way to call a middleware function in isolation — they're +designed to run in the request pipeline. Test them via E2E. + +## Do / Don't + +| Do | Don't | +|-------------------------------------------------------------|------------------------------------------------------------------| +| Use `# order:` to make the ordering explicit | Rely on filesystem load order | +| Return the modified `req` via `"request"` | Mutate `req` in place | +| Use `# scope_only: true` for anything not globally desired | Add per-request `req["path"]` checks to gate a global middleware | +| Use `.to_json` to serialize the response body | Use legacy `json_stringify(...)` — convention is the method form | +| Use `#{...}` interpolation | Use `\(...)` — the lexer rejects it | +| Add `_helper` private functions in the same file | Pull single-file helpers into a separate module | +| Pass forward data via `req["custom_key"]` | Use global variables to communicate between middleware and handler| +| Test via E2E specs | Try to unit-test by calling the function directly | diff --git a/admin/app/middleware/cors.sl b/admin/app/middleware/cors.sl new file mode 100644 index 00000000..b4c895da --- /dev/null +++ b/admin/app/middleware/cors.sl @@ -0,0 +1,24 @@ +# ============================================================================ +# CORS Middleware (Global) +# ============================================================================ +# +# This middleware adds CORS headers to all responses. +# It runs for ALL requests automatically. +# +# Configuration: +# - `# order: N` - Execution order (lower runs first) +# - `# global_only: true` - Runs for all requests, cannot be scoped +# +# ============================================================================ + +# order: 5 +# global_only: true + +def add_cors_headers(req: Any) -> Any + # Add CORS headers to the request context + # These will be included in the response + return { + "continue": true, + "request": req + } +end diff --git a/admin/app/models/CLAUDE.md b/admin/app/models/CLAUDE.md new file mode 100644 index 00000000..7753bb3f --- /dev/null +++ b/admin/app/models/CLAUDE.md @@ -0,0 +1,460 @@ +# Models + +This directory holds the data layer. **One file per model**: `post.sl` defines +`class Post < Model`. Filenames are `snake_case.sl`; class names are +`PascalCase`, singular. + +Models are auto-loaded by `soli serve` — controllers and migrations reference +them by class name without an `import`. Adding `import "../models/*.sl"` +inside a controller trips `style/redundant-model-import`. + +Models own validation, persistence, and business rules. Controllers are thin; +push every "X happens when Y is created" rule into the model layer. + +## Anatomy of a model + +```soli +class Post < Model + # Associations + belongs_to("user") + has_many("comments") + + # Validations + validates("title", { "presence": true, "min_length": 3, "max_length": 200 }) + validates("body", { "presence": true }) + + # Lifecycle callbacks + before_save("normalize_title") + after_create("notify_subscribers") + + # Named scopes + scope("published", fn() { this.where({ "status": "published" }) }) + scope("recent", fn() { this.order("created_at", "desc").limit(10) }) + + # Instance methods (your own logic) + def normalize_title + this.title = this.title.trim() + end + + def notify_subscribers + # ... + end +end +``` + +Models are **untyped** — you don't declare `title: String` at class level. +Fields are inferred from what you assign / persist, and validated by the rules +you register. + +## Inherited CRUD (don't override) + +These come with `< Model`. They use the worker's pre-configured SoliDB +connection. + +| Class method | What it does | +|---------------------------------------|-----------------------------------------------------------------------| +| `Model.all` | All records as instances. | +| `Model.find(id)` | Lookup by id. **Raises `RecordNotFound` on miss → 404 in controllers.**| +| `Model.find_by(field, val)` | First match, or `nil`. | +| `Model.first_by(field, val)` | First match with ordering, or `nil`. | +| `Model.where({...})` / `where("doc.x == @a", {"a": ...})` | Filter (see Querying below). | +| `Model.create({...})` | Insert with validation. Always returns an instance (see `_errors`). | +| `Model.find_or_create_by(field, val, defaults?)` | Look up or insert. | +| `Model.upsert(key, data)` | Insert if absent, else update. | +| `Model.create_many([{...}, ...])` | Batch insert. | +| `Model.count` | Row count. | +| `Model.delete_all` | Wipe the collection. Dangerous — use with care. | +| `Model.with_deleted` / `Model.only_deleted` | Include / restrict to soft-deleted records. | +| `Model.transaction` | Open a transaction (returns a Transaction with `get/create/update/delete/commit/rollback`). | +| `Model.paginate({ "page": 1, "per": 20 })` | Returns `{ "records": [...], "pagination": {...} }`. | + +| Instance method | What it does | +|---------------------------------------|-----------------------------------------------------------------------| +| `instance.save([attrs])` | Insert or update. Returns `true` / `false`. | +| `instance.update({...})` | Apply attrs and save. | +| `instance.delete` | Delete (or soft-delete if the model uses soft-deletes). | +| `instance.restore` | Undo a soft-delete. | +| `instance.reload` | Re-fetch from the DB; refresh all fields. | +| `instance.increment("counter", n=1)` | Atomic `+=`. | +| `instance.decrement("counter", n=1)` | Atomic `-=`. | +| `instance.touch` | Bump `_updated_at`. | +| `instance._errors` | Array of `{"field": ..., "message": ...}` after a failed save/create. | + +If you find yourself shadowing one of these with a `static def all ... end` or +similar, **stop** — you're working against the framework. Add a `scope` or a +class-side helper with a different name instead. + +## Querying + +Three forms, in order of preference: + +### 1. Hash form (safe — use this for user input) + +```soli +User.where({ "role": params["role"], "active": true }) +``` + +Keys are validated against the model's field set; values are bound as +parameters. Safe to pass `params` values straight in. + +### 2. String form with binds (developer-trusted, but parameterized) + +```soli +User.where("doc.age >= @min AND doc.role == @role", { + "min": params["min_age"], + "role": params["role"] +}) +``` + +The condition is your code — **never concatenate user input into the string**. +Bind everything that came from outside via `@name` placeholders. Use this when +the hash form isn't expressive enough (range, OR, function calls). + +### 3. Raw `@sdbql{ ... }` (when the ORM doesn't fit) + +```soli +let min_age = 18 +let users = @sdbql{ + FOR u IN users + FILTER u.age >= #{min_age} + SORT u.name ASC + LIMIT 50 + RETURN u +} +``` + +`#{expr}` inside the block is **bound as a parameter** (not string-interpolated +as text), so it's safe. Reach for this for joins, subqueries, and anything +hand-tuned. Returns raw documents, not model instances. + +### Chaining + +`where`, `order`, `limit`, `offset`, `select`/`fields`, `pluck`, `includes`, +`includes_count`, `join` all return a chainable `QueryBuilder`. Terminate the +chain with one of: + +| Terminator | Returns | +|--------------------|----------------------------------------------------------| +| `.all` | Array of instances. | +| `.first` | First instance, or `nil`. | +| `.count` | Number. | +| `.exists` | Boolean. | +| `.pluck("field")` | Array of values. | +| `.sum/avg/min/max("field")` | Numeric aggregate. | +| `.group_by(field, func, agg_field)` | Array of `{group, result}` hashes. | + +```soli +let recent = Post + .where({ "status": "published" }) + .order("created_at", "desc") + .limit(20) + .all + +let total_views = Post.where({ "user_id": user.id }).sum("views") +``` + +## Validations + +Pass an options hash to `validates`. All keys are optional; combine freely. + +| Option | Effect | +|------------------------|----------------------------------------------------------------------------| +| `"presence": true` | Required; rejects `null`, `""`, missing. | +| `"uniqueness": true` | Best-effort pre-check + relies on a unique DB index for atomicity. | +| `"min_length": N` | String length ≥ N. | +| `"max_length": N` | String length ≤ N. | +| `"format": "regex"` | String matches the pattern. | +| `"numericality": true` | Value is a number. | +| `"min": N` / `"max": N`| Numeric bounds. | +| `"custom": "method"` | Calls `this.method()` for full custom checks; method appends to `_errors`. | + +```soli +class User < Model + validates("email", { "presence": true, "uniqueness": true, "format": "^[^@]+@[^@]+$" }) + validates("age", { "numericality": true, "min": 0, "max": 150 }) + validates("name", { "custom": "validate_name" }) + + def validate_name + if this.name.blank? || this.name.length() < 2 + this._errors.push({ "field": "name", "message": "too short" }) + end + end +end +``` + +### Reading `_errors` + +`_errors` is an **array of hashes**, not a hash keyed by field: + +```soli +@user = User.create(params) +if @user._errors + for err in @user._errors + print("#{err.field}: #{err.message}") + end +end +``` + +On a clean save `_errors` is `nil` (not `[]`). Check `if @user._errors` — +truthiness is correct here. + +## Associations + +```soli +class Post < Model + belongs_to("user") # Post.user_id (FK), post.user (instance) + has_many("comments") # user.comments (QueryBuilder) + has_one("featured_image") # one-to-one + has_and_belongs_to_many("tags") # M2M via join collection +end +``` + +Conventions: + +- `belongs_to("user")` adds the `user_id` FK to **this** collection. Instance + accessor `post.user` lazy-loads on first read. +- `has_many("comments")` adds the FK on the *other* side (comments have + `user_id`). The accessor returns a `QueryBuilder` — chain on it: + `user.posts.where({"status": "published"}).count`. +- `has_one` works like `has_many` but returns a single instance. +- All four accept overrides: + `belongs_to("author", { "class_name": "User", "foreign_key": "author_id" })`. + +### Eager loading + +Avoid N+1 by pre-loading on the query: + +```soli +let posts = Post.where({...}).includes("user", "comments").all +# posts[0].user and posts[0].comments are now materialized in memory +``` + +`includes_count("comments")` adds a `comments_count` integer to each +instance — handy for index pages. + +## Scopes + +A `scope` is a class-side query alias. The body runs with `this` bound to a +fresh `QueryBuilder`, so `this.where(...)` / `this.order(...)` chain off it. + +```soli +class Post < Model + scope("published", fn() { this.where({ "status": "published" }) }) + scope("by_user", fn(user_id) { this.where({ "user_id": user_id }) }) +end + +Post.published.order("created_at", "desc").limit(20).all +Post.by_user(current_user.id).published.count +``` + +Both `Post.published` and `Post.published()` invoke the scope. + +For class-body DSL closures — scopes, validators, callbacks — prefer the +explicit `fn() { this.method(...) }` form over implicit-self alternatives. + +## Lifecycle callbacks + +Eight hooks, each takes a **method-name string** (not a lambda): + +| Hook | When it fires | +|-------------------|------------------------------------------------| +| `before_create` | New record, before insert. | +| `after_create` | New record, after insert succeeded. | +| `before_update` | Existing record, before save. | +| `after_update` | Existing record, after save succeeded. | +| `before_save` | Either insert or update, before persist. | +| `after_save` | Either insert or update, after persist. | +| `before_delete` | Before `instance.delete`. | +| `after_delete` | After delete succeeded. | + +```soli +class Post < Model + before_save("normalize_title") + after_create("notify_subscribers") + + def normalize_title + this.title = this.title.trim() + end + + def notify_subscribers + # send mail, enqueue job, etc. + end +end +``` + +A `before_*` callback that mutates `this._errors` (or returns `false`, +depending on hook) aborts the operation. + +## Other class-body helpers + +- `attr_accessible(field1, field2, ...)` — whitelist fields for mass-assignment. + When set, `Model.create(params)` silently drops any key not on the list. + Pair with controller-side `_permit_params` for defense in depth. +- `uploader("avatar", { ... })` — declare a blob attachment field. See + **Attachments and uploads** below for the full contract. +- `translate("title", "body")` — declare translatable fields (i18n). + +## Attachments and uploads + +Declare a blob attachment with `uploader("field", { ... })` in the class body. +The framework wires the validation, storage (SoliDB blob collection), and +URL/HTTP plumbing for you — controllers don't need to touch the blob store. + +```soli +class Contact < Model + uploader("photo", { + "multiple": false, + "content_types": ["image/jpeg", "image/png", "image/webp"], + "max_size": 2_000_000, # bytes — rejects above this + "collection": "contact_photos" # optional; defaults to "contact_photos" + }) + + uploader("attachments", { # multi-file field + "multiple": true, + "content_types": ["application/pdf", "image/png"], + "max_size": 5_000_000 + }) +end +``` + +| Option | Meaning | +|-----------------|----------------------------------------------------------------------------------| +| `multiple` | `false` (default) → one blob per record. `true` → array of blob ids. | +| `content_types` | Allow-list of MIME types. Anything else is rejected before storage. | +| `max_size` | Hard cap in bytes. Above this → `_errors` populated, no blob stored. | +| `collection` | SoliDB blob collection name. Defaults to `_s` (`contact_photos`). | + +The uploader adds a `_blob_id` column (single) or `_blob_ids` +array (multiple) to the document. You don't read those directly — use the +auto-generated instance methods below. + +### Auto-generated instance methods + +`uploader("photo", ...)` adds three methods on every instance: + +| Method | What it does | +|-------------------------------|------------------------------------------------------------------------------| +| `contact.attach_photo(file)` | Validate + store the file, update the `_blob_id(s)` column. | +| `contact.detach_photo(id?)` | Delete the blob and clear the column. `id` is required when `multiple: true`. | +| `contact.photo_url(opts?)` | Return the public URL for the stored blob (or `nil` if none stored). | + +`file` is the hash returned by `find_uploaded_file(req, "photo")` in a +controller (see **Controllers — Handling file uploads**). On a failed attach, +`contact._errors` is populated and `attach_photo` returns `false` — the same +error-rendering flow used by `Model.create` validation failures. + +```soli +def create + @contact = Contact.create(this._permit_params(params)) + if @contact._errors + return render("contacts/new") + end + + photo = find_uploaded_file(params, "photo") + if !photo.nil? && !@contact.attach_photo(photo) + return render("contacts/new") # attach failed → _errors set + end + + redirect(contact_path(@contact)) +end +``` + +### Wiring the upload routes + +Add `uploads("contacts", "photo")` to `config/routes.sl`. That single call +registers a GET / POST / DELETE family (plus `:blob_id`-scoped variants for +multi-file fields) backed by the framework's built-in `AttachmentsController`: + +```soli +# config/routes.sl +resources("contacts") +uploads("contacts", "photo") # for the photo field +uploads("contacts", "attachments") # for the multi-file field +``` + +| Route | Purpose | +|------------------------------------------------------|----------------------------------| +| `GET /contacts/:id/photo` | Stream the blob (with transforms).| +| `POST /contacts/:id/photo` | Upload a file (multipart). | +| `DELETE /contacts/:id/photo` | Detach (single) or the named blob.| +| `GET /contacts/:id/attachments/:blob_id` | Stream one entry from a multi field. | +| `DELETE /contacts/:id/attachments/:blob_id` | Remove that one entry. | + +The default routes target the framework's `AttachmentsController` — override +by defining your own `class AttachmentsController < Controller` if you need +auth checks, signed URLs, etc. Soli's loader processes app controllers after +the framework prelude, so a same-named class shadows the default cleanly. + +### Cleanup on delete + +`detach_all_uploads(record)` is available for `before_delete` hooks if you +need to wipe attached blobs alongside the record. Without it, a deleted +record leaves orphan blobs in the collection. + +```soli +class Contact < Model + uploader("photo", { ... }) + before_delete("cleanup_uploads") + + def cleanup_uploads + detach_all_uploads(this) + end +end +``` + +## Inspecting AQL queries (`--dev`) + +`dev_queries()` returns the AQL stack issued for the current request when the +server runs with `--dev`. Each entry is `{ "query": String, "bind_vars": +Hash | null, "duration_ms": Float }`. Useful for building a debug bar or +spotting N+1s. + +```erb +<% if dev_queries().length() > 0 %> +
    + <% for q in dev_queries() %> + 
    <%= q.query %> (<%= q.duration_ms %>ms)
    + <% end %> +
    +<% end %> +``` + +In production, `dev_queries()` returns `[]` (so the `length() > 0` guard +collapses to nothing) with zero overhead. + +## Do / Don't + +| Do | Don't | +|---------------------------------------------------------------|--------------------------------------------------------------------| +| Put validation rules on the model | Validate in the controller | +| Push business rules into model methods | Spread "what happens when X is created" across controllers | +| Use `where({...})` (hash form) for user input | Concatenate strings: `where("role = " + params["role"])` | +| Use `#{expr}` bound interpolation in `@sdbql{...}` | Use `\(expr)` — that's a docs typo; the lexer rejects it | +| Use `find_by` / `first_by` when nil-on-miss is correct | Wrap `find` in try/catch to convert raise → nil | +| Chain `.where.order.limit.all` for readability | Build SDBQL strings in the controller | +| Use `includes(...)` to dodge N+1 | Call `post.user` inside a `for post in posts` loop without eager load | +| Use callbacks for cross-cutting concerns (timestamps, slugs) | Use callbacks for anything you'd want to disable in a test | +| Declare `attr_accessible` on the model | Trust the controller to filter every caller | + +## Spec location + +Model specs live in `tests/_model_spec.sl`. Example: + +```soli +describe("Post") do + test("rejects empty title") do + @post = Post.new({ "title": "", "body": "x" }) + @post.save + assert(@post._errors) + assert_eq(@post._errors[0].field, "title") + end + + test("normalize_title trims whitespace before save") do + @post = Post.create({ "title": " hello ", "body": "x" }) + assert_eq(@post.title, "hello") + end +end +``` + +Hit the real DB in model specs — that's where the validation and constraint +behavior actually lives. Don't mock the database. diff --git a/admin/app/services/admin_context.sl b/admin/app/services/admin_context.sl new file mode 100644 index 00000000..d0588d0e --- /dev/null +++ b/admin/app/services/admin_context.sl @@ -0,0 +1,14 @@ +# app/services/admin_context.sl +# +# Small shared lookups used by several controllers (e.g. the topbar database +# picker on db-scoped pages). + +class AdminContext + # Sorted database names, [] when SoliDB is unreachable. + static def database_names() + result = SolidbClient.get_api(SolidbEndpoints.databases()) + return [] unless result["ok"] + names = (result["data"] ?? {})["databases"] ?? [] + return names.sort() + end +end diff --git a/admin/app/services/solidb_client.sl b/admin/app/services/solidb_client.sl new file mode 100644 index 00000000..adb02d75 --- /dev/null +++ b/admin/app/services/solidb_client.sl @@ -0,0 +1,172 @@ +# app/services/solidb_client.sl +# +# Single gateway to the SoliDB HTTP API. Controllers never call HTTP.request +# directly -- they go through SolidbClient.get_api/post_api/put_api/delete_api, +# which return a uniform { "ok", "status", "data", "error" } hash. +# +# Auth: POST /auth/login with the SOLIDB_USERNAME/SOLIDB_PASSWORD env creds +# yields a JWT (~24h exp). The token is cached process-wide in a static field +# (same pattern as bonfire's SolidbLive). On any 401 the cache is cleared and +# the request retried once with a fresh login, so expiry is self-healing. +# +# NOTE: HTTP.request()'s SSRF guard blocks loopback hosts unless +# SOLI_DEV_ALLOW_SSRF=1 is set (dev only -- see .env). + +class SolidbClient + # Process-wide JWT cache. One shared admin token is correct here: every + # request acts as the single SOLIDB_USERNAME identity, nothing per-user. + static cached_token: String = "" + + static def host() + return getenv("SOLIDB_HOST") ?? "http://localhost:6745" + end + + static def username() + return getenv("SOLIDB_USERNAME") ?? "admin" + end + + static def password() + return getenv("SOLIDB_PASSWORD") ?? "admin" + end + + # Browser-reachable WebSocket base for the changefeed page. Prefer an + # explicit SOLIDB_PUBLIC_WS_URL (prod, behind a proxy); otherwise derive + # from SOLIDB_HOST by swapping the scheme to ws(s). + static def public_ws_url() + explicit = getenv("SOLIDB_PUBLIC_WS_URL") ?? "" + return explicit unless explicit.blank? + return SolidbClient.derive_ws_url(SolidbClient.host()) + end + + # Scheme-swap derivation, split out so every host shape is unit-testable. + static def derive_ws_url(http_url) + if http_url.starts_with("https://") + return "wss://" + http_url.substring(8, http_url.length()) + elsif http_url.starts_with("http://") + return "ws://" + http_url.substring(7, http_url.length()) + end + return http_url + end + + # --- pure helpers (unit-tested without network) --------------------------- + + static def auth_headers(token) + return { + "Authorization": "Bearer " + token, + "Content-Type": "application/json", + "Accept": "application/json" + } + end + + static def is_unauthorized(resp) + return false if resp.nil? + return (resp["status"] ?? 0) == 401 + end + + # Raw HTTP response -> uniform result hash. nil means the request never got + # out (connection refused / SSRF-blocked); status 0 marks that case so the + # UI can show the "SoliDB unreachable" banner instead of a per-action error. + static def interpret(resp) + if resp.nil? + return { "ok": false, "status": 0, "data": nil, "error": "SoliDB unreachable" } + end + status = resp["status"] ?? 0 + data = JSON.parse(resp["body"] ?? "") rescue nil + if status >= 200 && status < 300 + return { "ok": true, "status": status, "data": data, "error": nil } + end + return { "ok": false, "status": status, "data": data, "error": SolidbClient.error_message(status, data) } + end + + # Best-effort human message from a SoliDB error body. + static def error_message(status, data) + detail = "" + detail = data["error"] ?? (data["message"] ?? "") unless data.nil? + return "HTTP " + str(status) if detail.blank? + return "HTTP " + str(status) + " - " + detail + end + + # --- auth ------------------------------------------------------------------ + + # Mint a JWT via POST /auth/login and cache it. Returns "" on failure (a + # failed mint never poisons a previously-good cache entry -- it returns + # before the assignment). + static def login() + headers = { "Content-Type": "application/json", "Accept": "application/json" } + body = { "username": SolidbClient.username(), "password": SolidbClient.password() } + resp = HTTP.request("POST", SolidbClient.host() + "/auth/login", headers, body) rescue nil + result = SolidbClient.interpret(resp) + return "" unless result["ok"] + token = (result["data"] ?? {})["token"] ?? "" + return "" if token.blank? + SolidbClient.cached_token = token + return token + end + + static def token() + return SolidbClient.cached_token unless SolidbClient.cached_token.blank? + return SolidbClient.login() + end + + static def clear_token() + SolidbClient.cached_token = "" + end + + # --- generic request with one re-auth retry on 401 ------------------------- + + static def api(method, path, body = nil) + auth = SolidbClient.token() + if auth.blank? + return { "ok": false, "status": 0, "data": nil, + "error": "SoliDB authentication failed (check SOLIDB_HOST / SOLIDB_USERNAME / SOLIDB_PASSWORD)" } + end + resp = SolidbClient.send_request(method, path, body, auth) + # A nil response can be a stale pooled keep-alive connection dying on + # send ("error sending request") - the request never reached SoliDB, + # so one immediate retry on a fresh connection is safe and self-heals. + resp = SolidbClient.send_request(method, path, body, auth) if resp.nil? + if SolidbClient.is_unauthorized(resp) + SolidbClient.clear_token() + auth = SolidbClient.login() + return SolidbClient.interpret(resp) if auth.blank? + resp = SolidbClient.send_request(method, path, body, auth) + end + return SolidbClient.interpret(resp) + end + + static def send_request(method, path, body, auth) + headers = SolidbClient.auth_headers(auth) + full_url = SolidbClient.host() + path + if body.nil? + resp = HTTP.request(method, full_url, headers) rescue nil + return resp + end + resp = HTTP.request(method, full_url, headers, body) rescue nil + return resp + end + + static def get_api(path) + return SolidbClient.api("GET", path) + end + + static def post_api(path, body = nil) + return SolidbClient.api("POST", path, body) + end + + static def put_api(path, body = nil) + return SolidbClient.api("PUT", path, body) + end + + static def delete_api(path) + return SolidbClient.api("DELETE", path) + end + + # --- live queries ----------------------------------------------------------- + + # Short-lived token for the browser's changefeed WebSocket. + static def livequery_token() + result = SolidbClient.get_api("/_api/livequery/token") + return "" unless result["ok"] + return (result["data"] ?? {})["token"] ?? "" + end +end diff --git a/admin/app/services/solidb_endpoints.sl b/admin/app/services/solidb_endpoints.sl new file mode 100644 index 00000000..94eb4ee4 --- /dev/null +++ b/admin/app/services/solidb_endpoints.sl @@ -0,0 +1,192 @@ +# app/services/solidb_endpoints.sl +# +# Pure SoliDB API path builders. Controllers compose SolidbClient calls with +# these instead of hand-concatenating URL strings, so the paths live in one +# place and every builder is unit-testable. + +class SolidbEndpoints + # --- server / cluster --- + static def health() + return "/_api/health" + end + + static def cluster_info() + return "/_api/cluster/info" + end + + static def livequery_token() + return "/_api/livequery/token" + end + + # --- databases --- + static def databases() + return "/_api/databases" + end + + static def database_create() + return "/_api/database" + end + + static def database(name) + return "/_api/database/" + name + end + + # --- users / roles / api keys --- + static def users() + return "/_api/auth/users" + end + + static def user(username) + return "/_api/auth/users/" + username + end + + static def user_roles(username) + return "/_api/auth/users/" + username + "/roles" + end + + static def user_role(username, role) + return "/_api/auth/users/" + username + "/roles/" + role + end + + static def roles() + return "/_api/auth/roles" + end + + static def role(name) + return "/_api/auth/roles/" + name + end + + static def api_keys() + return "/_api/auth/api-keys" + end + + static def api_key(key_id) + return "/_api/auth/api-keys/" + key_id + end + + # --- collections --- + static def collections(db) + return "/_api/database/" + db + "/collection" + end + + static def collection(db, name) + return "/_api/database/" + db + "/collection/" + name + end + + static def collection_stats(db, name) + return SolidbEndpoints.collection(db, name) + "/stats" + end + + static def collection_truncate(db, name) + return SolidbEndpoints.collection(db, name) + "/truncate" + end + + # --- indexes (standard + fulltext share one family; geo / ttl have + # their own create+list APIs, but DELETE /index/{coll}/{name} drops any) --- + static def collection_indexes(db, name) + return "/_api/database/" + db + "/index/" + name + end + + static def collection_index(db, name, index_name) + return SolidbEndpoints.collection_indexes(db, name) + "/" + index_name + end + + static def collection_indexes_rebuild(db, name) + return SolidbEndpoints.collection_indexes(db, name) + "/rebuild" + end + + static def geo_indexes(db, name) + return "/_api/database/" + db + "/geo/" + name + end + + static def ttl_indexes(db, name) + return "/_api/database/" + db + "/ttl/" + name + end + + # --- columnar collections (separate API family) --- + static def columnar(db) + return "/_api/database/" + db + "/columnar" + end + + static def columnar_collection(db, name) + return "/_api/database/" + db + "/columnar/" + name + end + + # --- documents --- + static def documents(db, collection_name) + return "/_api/database/" + db + "/document/" + collection_name + end + + static def document(db, collection_name, key) + return "/_api/database/" + db + "/document/" + collection_name + "/" + key + end + + # --- queries --- + static def cursor(db) + return "/_api/database/" + db + "/cursor" + end + + static def explain(db) + return "/_api/database/" + db + "/explain" + end + + # --- lua scripts --- + static def scripts(db) + return "/_api/database/" + db + "/scripts" + end + + static def script(db, script_id) + return "/_api/database/" + db + "/scripts/" + script_id + end + + # --- queues / jobs / cron --- + static def queues(db) + return "/_api/database/" + db + "/queues" + end + + static def queue_jobs(db, queue_name) + return "/_api/database/" + db + "/queues/" + queue_name + "/jobs" + end + + static def queue_enqueue(db, queue_name) + return "/_api/database/" + db + "/queues/" + queue_name + "/enqueue" + end + + static def queue_job(db, job_id) + return "/_api/database/" + db + "/queues/jobs/" + job_id + end + + static def queue_job_run_now(db, job_id) + return "/_api/database/" + db + "/queues/jobs/" + job_id + "/run-now" + end + + static def cron_jobs(db) + return "/_api/database/" + db + "/cron" + end + + static def cron_job(db, cron_id) + return "/_api/database/" + db + "/cron/" + cron_id + end + + # --- triggers --- + static def triggers(db) + return "/_api/database/" + db + "/triggers" + end + + static def trigger(db, trigger_id) + return "/_api/database/" + db + "/triggers/" + trigger_id + end + + static def trigger_toggle(db, trigger_id) + return SolidbEndpoints.trigger(db, trigger_id) + "/toggle" + end + + # --- env vars --- + static def env_vars(db) + return "/_api/database/" + db + "/env" + end + + static def env_var(db, key) + return "/_api/database/" + db + "/env/" + key + end +end diff --git a/admin/app/views/CLAUDE.md b/admin/app/views/CLAUDE.md new file mode 100644 index 00000000..38da48cb --- /dev/null +++ b/admin/app/views/CLAUDE.md @@ -0,0 +1,409 @@ +# Views + +This directory holds ERB-style templates with the `.html.slv` extension. + +**Convention**: a controller action renders `app/views//.html.slv` +by default. `PostsController#show` → `app/views/posts/show.html.slv`. +Partials live next to their parent view and start with an underscore: +`_post.html.slv`. + +## Template tags + +| Tag | Behavior | +|----------------------|----------------------------------------------------------------| +| `<%= expr %>` | Evaluate and **HTML-escape**. Safe default for user input. | +| `<%- expr %>` | Evaluate, output **raw** (no escaping). Trusted HTML only. | +| `<% stmt %>` | Evaluate; no output. Used for control flow. | +| `<%# comment %>` | Comment — content is silently dropped, even across lines. | + +`<%== expr %>` was removed (SEC-023). If you need to render pre-escaped HTML +(e.g. a Markdown render), use `<%- html %>` *or* `<%= html_unescape(html) %>`, +not `<%==`. + +XSS is the default risk. Default to `<%= %>` for anything that touches user +content; reach for `<%- %>` only when you can prove the value is trusted. + +## Locals — three ways to pass data into a template + +1. **Implicit via `@`-variables on the controller** (recommended; covers most + cases). Anything you assign to `@field` inside an action becomes available + as both `@field` and bare `field` in the template — no need to pass it in + `render(...)`'s data hash. Underscore-prefixed fields (`@_internal`) are + private and not exposed. + + ```soli + # in the controller + def show + @post = Post.find(params["id"]) + @title = "Read \"#{@post.title}\"" + end + ``` + + ```erb +

    <%= @title %>

    +
    <%= @post.body %>
    + ``` + +2. **Explicit data hash on `render`**. Use this when you want to render a + different template, or pass a value under a name that doesn't match a + controller field. + + ```soli + render("posts/show_summary", { "summary_text": build_summary(@post) }) + ``` + +3. **`locals[...]` for collisions**. When a local name shadows a builtin or + helper, read it through the `locals` hash instead of bare. `locals` is + always defined (even when no data was passed). + + ```erb + <%# `partial` is a builtin; locals["partial"] disambiguates %> + <%= locals["partial"] %> + ``` + +## Iteration + +```erb +<% for post in @posts %> +
    +

    <%= h(post.title) %>

    + <%= post.body %> +
    +<% end %> + +<% for post, i in @posts %> +

    #<%= i %>: <%= h(post.title) %>

    +<% end %> + +<% if @posts.length() == 0 %> +

    No posts yet.

    +<% end %> +``` + +`<% xs.each do |x| %> ... <% end %>` does **NOT** work inside ERB. The +template engine only recognises `for x in xs` / `for x, i in xs` as the loop +syntax. Use `each` in regular `.sl` code, `for` in templates. + +## Escape helpers + +Pick the helper that matches the *context* where the value lands. `<%= %>` +calls `h()` automatically, which is right for HTML body text but wrong for +attributes / JS / URLs. + +| Helper | Use when the value goes into… | +|--------------------------|----------------------------------------------------------------------| +| `h(value)` | HTML body text (default — auto-applied by `<%= %>`). | +| `attr(value)` | An HTML attribute. Escapes `"`, `'`, `<`, `>`, `&`. | +| `j(value)` | A ` + +">search + +<%# user-authored rich text: sanitize, then emit as raw output %> +<%- sanitize_html(post.body_html) %> +``` + +## Partials + +Render a partial with `partial("dir/name", { "local_key": value })`. The +template file **must** be named `_name.html.slv` — the leading underscore is +mandatory. `render_partial(...)` is an alias for the same builtin. + +```erb +<% for post in @posts %> + <%- partial("posts/post", { "post": post }) %> +<% end %> +``` + +`partial` returns HTML, so render it with `<%-` (raw output). `<%=` would +HTML-escape the partial's own markup and show tags as text. + +> **`<%-` skips escaping — sanitize untrusted HTML first.** `<%-` is safe for +> trusted, framework-produced HTML (like a `partial`). For any HTML that +> originated from a user (rich-text fields, imported content, Markdown +> rendered to HTML), pass it through `sanitize_html(value)` first — it keeps +> safe tags (links, basic formatting) and strips scripts/event handlers/ +> `javascript:` URLs. Use `strip_html(value)` instead when you want plain text +> with no tags at all. Never put raw user HTML straight into `<%-`. + +`app/views/posts/_post.html.slv`: + +```erb +
    +

    <%= h(post.title) %>

    + <%= h(post.body) %> +
    +``` + +Inside the partial, `post` is read as a bare local. Use `locals["post"]` if +the name collides with anything. + +## View helpers + +Always available inside templates: + +| Helper | What it does | +|--------------------------------------------|--------------------------------------------------------------------| +| `partial("dir/name", { ... })` | Render a partial. | +| `public_path("css/app.css")` | Cache-busted asset URL (fingerprinted). | +| `upload_url(record, "field", opts?)` | URL for an uploader-managed blob (with optional transforms — see **Rendering uploads**). | +| `t("posts.title")` | i18n lookup; respects current locale. | +| `time_ago(timestamp)` | "3 minutes ago"-style relative time. | +| `current_path()` | Current request path. | +| `current_method()` | Current HTTP method (`"GET"`, `"POST"`, ...). | +| `current_path?(path)` | Boolean — is the request on this path? | +| `range(start, stop)` | `[start, ..., stop-1]` for `for i in range(0, 5)`. | +| `sanitize_html(html)` / `strip_html(html)` | See "Escape helpers". | +| `dev_queries()` | AQL stack for the current request (`--dev` only — `[]` otherwise). | + +**Named route helpers** (`posts_path()`, `post_path(post)`, `new_post_path()`, +`edit_post_path(post)`, plus `*_url` variants) come from `resources(...)` in +`config/routes.sl`. Use them in templates — never hand-build URLs. + +```erb +All posts +Read +Edit +``` + +There is **no `link_to` helper** in Soli — write the `` tag yourself with +`<%= attr(...) %>` around URLs that contain user data. + +## Rendering uploads and image transforms + +For attachment fields declared with `uploader(...)` on a model (see +`app/models/CLAUDE.md` → **Attachments and uploads**), the auto-generated +`_url` instance method — or the standalone `upload_url(record, "field")` +helper — returns a URL served by the framework's `AttachmentsController`. +The URL hits SoliDB, decodes the blob, and streams it back. + +```erb +<% if !@contact.photo_url().nil? %> + <%= attr(@contact.name) %> +<% end %> +``` + +For a `multiple: true` field, the per-record getter returns `nil` — iterate +the stored blob ids and ask for each URL: + +```erb +<% for blob_id in (@document.attachments_blob_ids ?? []) %> +
  • + Download +
    • +<% end %> +``` + +### Image transforms via query string + +When the stored blob is an image (`Content-Type: image/*`), the URL accepts a +small set of query parameters and the controller pipes the blob through +Soli's `Image` builtin before responding. Browsers cache each (URL, +query-string) combination independently — same params → cache hit, no +re-transformation cost. Failed transforms fall back to streaming the original +bytes. + +Pass options as a hash to `upload_url` / `_url`: + +```erb +"> + +"> + +"> +``` + +| Key | Effect | +|----------------------|--------------------------------------------------------------------------| +| `w`, `h` | Resize to those dimensions (may distort if both set without `fit`). | +| `thumb` | Square-fit thumbnail, max edge = value. | +| `square` | Sugar for `w=N&h=N&fit=cover` — square crop to N×N. | +| `fit` | `"cover"` (fill + center-crop) or `"contain"` (fit inside). Needs `w`+`h`.| +| `crop` | `"x,y,w,h"` — pick a source region first; runs before sizing. | +| `flipx`, `flipy` | Horizontal / vertical flip. | +| `rot` | `90`, `180`, or `270` — quarter-turn rotation. | +| `blur` | Gaussian blur sigma (float). | +| `bright` | Brightness delta (signed int). | +| `contrast` | Contrast factor (float; 1.0 = unchanged). | +| `hue` | Hue rotation in degrees. | +| `gray` | Truthy → grayscale. | +| `invert` | Truthy → invert colors. | +| `fmt` | Re-encode: `"webp"`, `"png"`, `"jpeg"`. | +| `q` | Output quality (int, encoder-specific). | + +Pipeline order is fixed: `crop` → `flip`/`rot` → resize (`thumb` / `fit` / +`w+h` / `square`) → effects (`blur`, `bright`, `contrast`, `hue`, `invert`, +`gray`) → encode (`fmt`, `q`). Width/height are clamped server-side to 1000 px +so a hand-crafted URL can't drive a multi-GB allocation. + +For ad-hoc image processing outside the uploader pipeline (e.g. a job that +generates a derivative and stores it under another field), use the `Image` +builtin directly: + +```soli +img = Image.from_buffer(file["data"]) +thumb_b64 = img.thumbnail(200).format("webp").quality(80).to_buffer() +``` + +`Image` is mostly used in models/jobs, not in views — keep templates thin. + +## App-level helpers + +Anything you put in `app/helpers/*.sl` is auto-loaded and available inside +every template. No `import` needed. Define a helper as a free-standing +function: + +```soli +# app/helpers/markdown_helpers.sl + +def render_markdown(text) + # ... call your markdown engine ... +end +``` + +Then in a view: + +```erb +<%- render_markdown(@post.body) %> +``` + +Group helpers thematically — `application_helper.sl`, `markdown_helpers.sl`, +`form_helpers.sl` — rather than one mega-file. + +## Client interactivity — HTMX + Alpine.js + +Every scaffolded app ships with **HTMX** and **Alpine.js** preloaded in +`app/views/layouts/application.html.slv`: + +```erb + + +``` + +This is Soli's default client stack. Reach for it before introducing React / +Vue / similar — most "make this page interactive" tasks fit it cleanly, +without a build step. + +**HTMX** for server-driven partials. Swap a piece of the DOM with the HTML +response of a Soli action — no JSON, no client-side templating: + +```erb + + +<%= post.likes %> +``` + +Pair with a controller that returns the partial fragment directly: + +```soli +def like + @post = Post.find(params["id"]) + @post.increment("likes") + render("posts/_like_count") # tiny partial, no layout +end +``` + +For HTMX requests, skip the layout by detecting the `HX-Request` header in +the controller (e.g. `if req["headers"]["hx-request"] == "true"`) and pass +`{ "layout": false }` to `render`. + +**Alpine.js** for purely-local UI state (open/closed, expanded/collapsed, +in-input validation flash). Keep it scoped — no app-level Alpine stores. + +```erb +
    + +
    …content…
    +
    +``` + +Rule of thumb: **server first**. If a piece of state lives on the server +(user data, validations, search results), use HTMX. If it lives only in the +DOM (modals, dropdowns, tabs), use Alpine. Mix freely — they don't conflict. + +See `docs/client-interactivity.md` (bundled in every Soli app) for the +deeper end of either library. + +## Layouts + +Every render runs inside a layout unless explicitly opted out. + +- **Default**: `app/views/layouts/application.html.slv`. +- **Per-controller**: set `this.layout = "X"` in the controller's `static { }` + block to use `app/views/layouts/X.html.slv` instead. +- **Per-render**: pass `"layout"` in the data hash: + `render("posts/show", { "layout": "minimal" })`. +- **No layout**: `render("posts/show", { "layout": false })`. + +The layout calls `<%= yield %>` (or the equivalent `<%= content %>` local) +where the view's content gets inserted. + +```erb + + + + + <%= @title ?? "MyApp" %> + "> + + + <%= yield %> + + +``` + +## File layout + +``` +app/views/ +├── layouts/ +│ └── application.html.slv # wrap-around for all renders by default +├── posts/ +│ ├── index.html.slv +│ ├── show.html.slv +│ ├── new.html.slv +│ ├── edit.html.slv +│ └── _post.html.slv # partial — underscore prefix is mandatory +└── shared/ + └── _nav.html.slv # cross-cutting partials live here +``` + +`render("posts/show")` → `app/views/posts/show.html.slv`. +`partial("posts/post", ...)` → `app/views/posts/_post.html.slv`. + +## Style + +- **Indent at 2 spaces** in `.slv` files — matches the docs site convention, + keeps diffs and partials consistent. +- One template per action; pull cross-cutting markup into a partial. +- Keep logic out of templates. If a `<% %>` block grows past a few lines, + move it to a helper or a controller `@field`. +- Always close your tags. `<% if %>` needs `<% end %>`; `<% for %>` needs + `<% end %>`. + +## Do / Don't + +| Do | Don't | +|----------------------------------------------------------|--------------------------------------------------------------------| +| Use `<%= %>` by default | Use `<%- %>` for values that touched user input | +| Use `attr(...)` / `j(...)` / `url(...)` for non-body contexts | Trust `h()` to be safe inside attributes — it's HTML-body-only | +| Set `@field` in the controller and read it in the view | Re-pass `@field` in the `render(...)` data hash | +| Use named route helpers — `post_path(post)` | Hand-build `"/posts/" + str(post.id)` | +| Use `for x in xs` for loops | Try `<% xs.each do |x| %>` — that doesn't parse inside ERB | +| Use `#{expr}` for interpolation in strings inside `.sl` blocks | Use `\(expr)` — the lexer rejects that | +| Keep templates thin; push logic into helpers | Embed business rules in `<% %>` blocks | +| Put cross-cutting markup in `_partials.html.slv` | Copy-paste header/nav across every action's view | diff --git a/admin/app/views/api_keys/index.html.slv b/admin/app/views/api_keys/index.html.slv new file mode 100644 index 00000000..e2fb7add --- /dev/null +++ b/admin/app/views/api_keys/index.html.slv @@ -0,0 +1,91 @@ +<%- partial("shared/banners", { "solidb_down": @solidb_down, "flash_error": @flash_error, "flash_notice": @flash_notice }) %> + +<% if !@created_key.blank? %> +
    +

    raw key — shown once, copy it now

    +
    + <%= @created_key %> + +
    +
    +<% end %> + +
    +
    +
    +

    Access

    +

    API Keys (<%= @keys.length() %>)

    +

    + token credentials for programmatic access without a username/password — + the raw key is shown once at creation, store it somewhere safe +

    +
    + +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    + +
    + + + + + + + + + + + + <% for key in @keys %> + + + + + + + + <% end %> + +
    namerolesscoped databasescreatedactions
    <%= key["name"] ?? "" %> +
    + <% for role_name in (key["roles"] ?? []) %> + <%= role_name %> + <% end %> +
    +     		+ <% scoped = key["scoped_databases"] ?? [] %> + <%= scoped.length() > 0 ? scoped.join(", ") : "all" %> + <%= (key["created_at"] ?? "").substring(0, 10) %> + +
    + <% if @keys.length() == 0 %> +

    no api keys

    + <% end %> +
    +
    diff --git a/admin/app/views/collections/_indexes.html.slv b/admin/app/views/collections/_indexes.html.slv new file mode 100644 index 00000000..7dcbf139 --- /dev/null +++ b/admin/app/views/collections/_indexes.html.slv @@ -0,0 +1,178 @@ +<%# x-effect locks body scroll while the modal is open so wheel events over + the fixed overlay don't scroll the page behind it. %> +
    +
    +

    indexes / <%= collection_name %> + (<%= indexes.length() %>)

    +
    + + +
    +
    + + <% if !indexes_error.blank? %> +

    <%= indexes_error %>

    + <% end %> + <% if !indexes_notice.blank? %> +

    <%= indexes_notice %>

    + <% end %> + + <% if indexes.length() > 0 %> + + + + + + + + + + + + + <% for idx in indexes %> + <% idx_name = idx["name"] ?? "" %> + <%# Literal class strings so the Tailwind scanner compiles them: + border-amber-800 text-amber-400 border-violet-800 text-violet-400 + border-sky-800 text-sky-400 border-rose-800 text-rose-400 + border-zinc-700 text-zinc-400 %> + <% idx_badges = { + "hash": "border-sky-800 text-sky-400", + "fulltext": "border-violet-800 text-violet-400", + "geo": "border-amber-800 text-amber-400", + "ttl": "border-rose-800 text-rose-400", + "persistent": "border-zinc-700 text-zinc-400" + } %> + + + + + + + + + <% end %> + +
    nametypefieldsuniqueinfoactions
    <%= idx_name %>"><%= idx["type"] %><%= (idx["fields"] ?? []).join(", ") %><%= idx["unique"] ? "yes" : "—" %><%= idx["detail"] %> + +
    + <% else %> +

    no indexes on <%= collection_name %>

    + <% end %> + + <%# Teleported to : the panel lives inside a table row, and an + ancestor there acts as containing block for position:fixed, cropping + the modal to the panel area. htmx.process() is required because htmx + never initializes