From ea2eeb64dab8311fb1c57491da278923685aae20 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Wed, 28 Oct 2026 12:20:41 +0530 Subject: [PATCH 01/16] fix(engine): reclaim dead tuples, fail-stop on partial apply, auto-checkpoint, harden WAL/replication - free old tuple on overwrite/delete; compact and reuse pages before growing file - idempotent per-key WAL replay over already-flushed pages - halt writes after a post-WAL apply failure until restart - auto-checkpoint when WAL exceeds -wal-checkpoint-mb (default 64) - sync data file and WAL dir around checkpoint rotation - leader replays archived WAL generation for follower catch-up - constant-time gRPC token compare; 0600 data/WAL files - README claims, CI macOS Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 2 +- README.md | 6 +- cmd/server/dev.go | 106 ++++++++++++------------ cmd/server/main.go | 6 ++ docker-compose.yml | 120 +++++++++++++-------------- docs/clients.md | 2 +- examples/python/client.py | 2 +- go.mod | 26 +++--- internal/api/engine.go | 9 +- internal/api/engine_batch.go | 59 ++----------- internal/api/engine_checkpoint.go | 30 +++++++ internal/api/engine_delete.go | 25 ++---- internal/api/engine_ops.go | 37 ++------- internal/api/engine_space.go | 132 ++++++++++++++++++++++++++++++ internal/api/engine_space_test.go | 125 ++++++++++++++++++++++++++++ internal/api/grpc_auth.go | 3 +- internal/api/recovery.go | 40 +-------- internal/replication/leader.go | 24 +++--- internal/storage/disk_manager.go | 2 +- internal/storage/page.go | 2 +- internal/wal/wal.go | 13 ++- studio/next-env.d.ts | 12 +-- 22 files changed, 485 insertions(+), 298 deletions(-) create mode 100644 internal/api/engine_space.go create mode 100644 internal/api/engine_space_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1309d1a..fba4ec9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Checkout Code diff --git a/README.md b/README.md index 56dc024..2a431de 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@

-

High-Performance, Crash-Resilient Distributed Database — Built from Scratch in Pure Go

+

Crash-Safe Embedded OLTP Database with Leader-Follower Replication — Built from Scratch in Pure Go

Go Version License Zero Allocs - Tests + CI Studio

@@ -27,7 +27,7 @@ It ships as a **single static binary** that includes a full gRPC service, WAL-ba | Concern | How BeastDB addresses it | |---|---| -| **Predictable latency** | B+ Tree with `O(log N)` point reads at 180 ns, 0 allocs — no compaction spikes | +| **Predictable latency** | B+ Tree with `O(log N)` cached point reads at 180 ns, 0 allocs — no compaction spikes | | **Crash safety** | 23-byte ARIES WAL frames with CRC32 torn-write detection; full replay on boot | | **Operational simplicity** | Single binary, `go run ./cmd/server -dev` for local dev, `docker compose up` for prod | | **Developer ergonomics** | Embedded Studio web UI, `beastctl` CLI, partition-aware key builder, fixture seeding | diff --git a/cmd/server/dev.go b/cmd/server/dev.go index bd70040..909ec44 100644 --- a/cmd/server/dev.go +++ b/cmd/server/dev.go @@ -1,53 +1,53 @@ -package main - -import ( - "fmt" - "log" - "os" -) - -// setupDevMode initializes an ephemeral directory and returns cleanup func. -func setupDevMode(dataDir *string, webAddr *string, adminPassword *string, port int) func() { - tmpDir, err := os.MkdirTemp("", "beastdb-dev-*") - if err != nil { - log.Fatalf("Failed to create temporary data directory: %v", err) - } - *dataDir = tmpDir - - if *adminPassword == "admin" { - *adminPassword = "admin" - } - if *webAddr == "" { - *webAddr = "127.0.0.1:8088" - } - - printDevBanner(port, *webAddr, *adminPassword, tmpDir) - - return func() { - log.Printf("[Dev Emulator] Cleaning up temporary storage: %s", tmpDir) - _ = os.RemoveAll(tmpDir) - } -} - -func printDevBanner(grpcPort int, webAddr, password, tmpDir string) { - fmt.Println() - fmt.Println("┌─────────────────────────────────────────────────────────────┐") - fmt.Println("│ ⚡ BeastDB Dev Emulator (Ephemeral In-Memory Mode) │") - fmt.Println("├─────────────────────────────────────────────────────────────┤") - fmt.Printf("│ • gRPC Service: 127.0.0.1:%-30d│\n", grpcPort) - if webAddr != "" { - fmt.Printf("│ • Studio Console: http://%-32s│\n", webAddr) - fmt.Printf("│ • Credentials: admin / %-31s│\n", password) - } - fmt.Printf("│ • Ephemeral Dir: %-40s│\n", truncatePath(tmpDir, 40)) - fmt.Println("│ • Auto-cleanup: All data is purged on process termination│") - fmt.Println("└─────────────────────────────────────────────────────────────┘") - fmt.Println() -} - -func truncatePath(s string, max int) string { - if len(s) <= max { - return s - } - return "..." + s[len(s)-(max-3):] -} +package main + +import ( + "fmt" + "log" + "os" +) + +// setupDevMode initializes an ephemeral directory and returns cleanup func. +func setupDevMode(dataDir *string, webAddr *string, adminPassword *string, port int) func() { + tmpDir, err := os.MkdirTemp("", "beastdb-dev-*") + if err != nil { + log.Fatalf("Failed to create temporary data directory: %v", err) + } + *dataDir = tmpDir + + if *adminPassword == "admin" { + *adminPassword = "admin" + } + if *webAddr == "" { + *webAddr = "127.0.0.1:8088" + } + + printDevBanner(port, *webAddr, *adminPassword, tmpDir) + + return func() { + log.Printf("[Dev Emulator] Cleaning up temporary storage: %s", tmpDir) + _ = os.RemoveAll(tmpDir) + } +} + +func printDevBanner(grpcPort int, webAddr, password, tmpDir string) { + fmt.Println() + fmt.Println("┌─────────────────────────────────────────────────────────────┐") + fmt.Println("│ ⚡ BeastDB Dev Emulator (Ephemeral In-Memory Mode) │") + fmt.Println("├─────────────────────────────────────────────────────────────┤") + fmt.Printf("│ • gRPC Service: 127.0.0.1:%-30d│\n", grpcPort) + if webAddr != "" { + fmt.Printf("│ • Studio Console: http://%-32s│\n", webAddr) + fmt.Printf("│ • Credentials: admin / %-31s│\n", password) + } + fmt.Printf("│ • Ephemeral Dir: %-40s│\n", truncatePath(tmpDir, 40)) + fmt.Println("│ • Auto-cleanup: All data is purged on process termination│") + fmt.Println("└─────────────────────────────────────────────────────────────┘") + fmt.Println() +} + +func truncatePath(s string, max int) string { + if len(s) <= max { + return s + } + return "..." + s[len(s)-(max-3):] +} diff --git a/cmd/server/main.go b/cmd/server/main.go index bc6968d..14d0c59 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -38,6 +38,7 @@ func main() { tlsCA := flag.String("tls-ca", getEnv("BEASTDB_TLS_CA", ""), "Path to TLS CA certificate file (PEM)") insecureAuth := flag.Bool("insecure-auth", false, "Permit default credentials on public interfaces (unsafe, for testing only)") devMode := flag.Bool("dev", false, "Run in ephemeral emulator mode with temporary storage auto-purged on exit") + walCheckpointMB := flag.Int("wal-checkpoint-mb", getEnvInt("BEASTDB_WAL_CHECKPOINT_MB", 64), "Checkpoint and rotate the WAL once it exceeds this many MiB") flag.Parse() grpcBind := fmt.Sprintf("%s:%d", *bindAddr, *port) @@ -67,6 +68,10 @@ func main() { log.Printf("🔒 Node running in follower mode: engine write path set to read-only.") } + ckptCtx, stopCkpt := context.WithCancel(context.Background()) + defer stopCkpt() + engine.StartAutoCheckpoint(ckptCtx, 10*time.Second, int64(*walCheckpointMB)<<20) + tokenStore := initTokenStore(engine) var grpcOpts []grpc.ServerOption if *tlsCert != "" && *tlsKey != "" { @@ -160,6 +165,7 @@ func main() { log.Printf("Received signal %s. Initiating graceful shutdown...", sig) cancel() + stopCkpt() grpcServer.GracefulStop() if err := engine.Close(); err != nil { log.Printf("Error flushing engine to disk: %v", err) diff --git a/docker-compose.yml b/docker-compose.yml index fe640af..a2e6b2b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,60 +1,60 @@ -services: - beastdb-primary: - build: - context: . - dockerfile: Dockerfile - container_name: beastdb-primary - command: ["-role", "leader", "-port", "50051", "-data-dir", "/data", "-pool-size", "256", "-web-addr", "0.0.0.0:8088"] - environment: - - BEASTDB_ADMIN_PASSWORD=${BEASTDB_ADMIN_PASSWORD:?BEASTDB_ADMIN_PASSWORD environment variable must be set for deployment} - - BEASTDB_SESSION_SECRET=${BEASTDB_SESSION_SECRET:-} - - BEASTDB_API_TOKEN=${BEASTDB_API_TOKEN:-} - ports: - - "50051:50051" - - "8088:8088" - volumes: - - primary_data:/data - networks: - - beastnet - restart: unless-stopped - - beastdb-replica-1: - build: - context: . - dockerfile: Dockerfile - container_name: beastdb-replica-1 - command: ["-role", "follower", "-port", "50051", "-leader-addr", "beastdb-primary:50051", "-replica-id", "replica-1", "-data-dir", "/data", "-pool-size", "256"] - ports: - - "50052:50051" - volumes: - - replica1_data:/data - networks: - - beastnet - depends_on: - - beastdb-primary - restart: unless-stopped - - beastdb-replica-2: - build: - context: . - dockerfile: Dockerfile - container_name: beastdb-replica-2 - command: ["-role", "follower", "-port", "50051", "-leader-addr", "beastdb-primary:50051", "-replica-id", "replica-2", "-data-dir", "/data", "-pool-size", "256"] - ports: - - "50053:50051" - volumes: - - replica2_data:/data - networks: - - beastnet - depends_on: - - beastdb-primary - restart: unless-stopped - -volumes: - primary_data: - replica1_data: - replica2_data: - -networks: - beastnet: - driver: bridge +services: + beastdb-primary: + build: + context: . + dockerfile: Dockerfile + container_name: beastdb-primary + command: ["-role", "leader", "-port", "50051", "-data-dir", "/data", "-pool-size", "256", "-web-addr", "0.0.0.0:8088"] + environment: + - BEASTDB_ADMIN_PASSWORD=${BEASTDB_ADMIN_PASSWORD:?BEASTDB_ADMIN_PASSWORD environment variable must be set for deployment} + - BEASTDB_SESSION_SECRET=${BEASTDB_SESSION_SECRET:-} + - BEASTDB_API_TOKEN=${BEASTDB_API_TOKEN:-} + ports: + - "50051:50051" + - "8088:8088" + volumes: + - primary_data:/data + networks: + - beastnet + restart: unless-stopped + + beastdb-replica-1: + build: + context: . + dockerfile: Dockerfile + container_name: beastdb-replica-1 + command: ["-role", "follower", "-port", "50051", "-leader-addr", "beastdb-primary:50051", "-replica-id", "replica-1", "-data-dir", "/data", "-pool-size", "256"] + ports: + - "50052:50051" + volumes: + - replica1_data:/data + networks: + - beastnet + depends_on: + - beastdb-primary + restart: unless-stopped + + beastdb-replica-2: + build: + context: . + dockerfile: Dockerfile + container_name: beastdb-replica-2 + command: ["-role", "follower", "-port", "50051", "-leader-addr", "beastdb-primary:50051", "-replica-id", "replica-2", "-data-dir", "/data", "-pool-size", "256"] + ports: + - "50053:50051" + volumes: + - replica2_data:/data + networks: + - beastnet + depends_on: + - beastdb-primary + restart: unless-stopped + +volumes: + primary_data: + replica1_data: + replica2_data: + +networks: + beastnet: + driver: bridge diff --git a/docs/clients.md b/docs/clients.md index 19238d9..a31e7ae 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -152,4 +152,4 @@ func main() { | **Node / TS** | @grpc/grpc-js + dynamic/static proto | None (Go not needed) | | **Rust** | onic + prost | None (Go not needed) | | **Java** | grpc-java + protobuf-java | None (Go not needed) | -| **C# / .NET** | Grpc.Net.Client | None (Go not needed) | +| **C# / .NET** | Grpc.Net.Client | None (Go not needed) | diff --git a/examples/python/client.py b/examples/python/client.py index 6e4add5..7aad6b9 100644 --- a/examples/python/client.py +++ b/examples/python/client.py @@ -45,4 +45,4 @@ def run(target: str = localhost:50051): if __name__ == __main__: addr = sys.argv[1] if len(sys.argv) > 1 else localhost:50051 - run(addr) + run(addr) diff --git a/go.mod b/go.mod index 8f33b59..6149eb7 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ -module github.com/ChromaBeast/beastdb - -go 1.26.5 - -require ( - golang.org/x/crypto v0.57.0 // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.42.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.83.2 // indirect - google.golang.org/protobuf v1.36.12 // indirect -) +module github.com/ChromaBeast/beastdb + +go 1.26.5 + +require ( + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/internal/api/engine.go b/internal/api/engine.go index bd9f356..85cb95f 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -21,6 +21,8 @@ type Engine struct { observer CommitObserver secIndex *index.SecondaryIndex readOnly bool + failed error // set by halt; blocks writes until restart + reclaim map[uint64]struct{} // data pages holding tombstoned tuples mu sync.RWMutex } @@ -132,6 +134,9 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { } e.mu.Lock() defer e.mu.Unlock() + if e.failed != nil { + return e.failed + } for _, bOp := range batchOps { if len(bOp.Key) < 8 { continue @@ -139,12 +144,12 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { k := binary.LittleEndian.Uint64(bOp.Key) if bOp.Type == wal.OpPut { if err := e.applyPutInMemory(k, bOp.Value); err != nil { - return err + return e.halt(err) } e.indexValueLocked(k, bOp.Value) } else if bOp.Type == wal.OpDelete { if err := e.applyDeleteInMemory(k); err != nil { - return err + return e.halt(err) } e.unindexValueLocked(k) } diff --git a/internal/api/engine_batch.go b/internal/api/engine_batch.go index 16b9b6e..1b19571 100644 --- a/internal/api/engine_batch.go +++ b/internal/api/engine_batch.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/ChromaBeast/beastdb/internal/index" - "github.com/ChromaBeast/beastdb/internal/storage" "github.com/ChromaBeast/beastdb/internal/wal" ) @@ -47,6 +46,9 @@ func (e *Engine) BatchWrite(ops []BatchOperation) error { e.mu.Lock() defer e.mu.Unlock() + if e.failed != nil { + return e.failed + } // 1. Serialize all operations into a single atomic WAL batch payload walOps := make([]wal.BatchOp, len(ops)) @@ -76,19 +78,19 @@ func (e *Engine) BatchWrite(ops []BatchOperation) error { // 2. Commit the entire batch atomically to the WAL lsn, err := e.wal.Write(wal.OpBatch, nil, payload) if err != nil { - return err + return e.halt(err) } // 3. Apply each mutation to the buffer pool, B+ Tree, and secondary index for _, op := range ops { if op.Type == BatchOpPut { if err := e.applyPutInMemory(op.Key, op.Value); err != nil { - return err + return e.halt(err) } e.indexValueLocked(op.Key, op.Value) } else { if err := e.applyDeleteInMemory(op.Key); err != nil && err != index.ErrKeyNotFound { - return err + return e.halt(err) } e.unindexValueLocked(op.Key) } @@ -101,52 +103,3 @@ func (e *Engine) BatchWrite(ops []BatchOperation) error { return nil } - -func (e *Engine) applyPutInMemory(key uint64, value []byte) error { - page, err := e.bpm.FetchPage(e.activeDataPage) - if err != nil { - return err - } - - slotID, err := page.InsertTuple(value) - if err == storage.ErrPageFull { - _ = e.bpm.UnpinPage(e.activeDataPage, false) - newPage, newID, err := e.bpm.NewPage() - if err != nil { - return err - } - if err := e.updateMetaActiveData(newID); err != nil { - _ = e.bpm.UnpinPage(newID, false) - return err - } - e.activeDataPage = newID - slotID, err = newPage.InsertTuple(value) - if err != nil { - _ = e.bpm.UnpinPage(newID, false) - return err - } - page = newPage - } - _ = e.bpm.UnpinPage(e.activeDataPage, true) - - rid := storage.RID{PageID: e.activeDataPage, SlotID: slotID} - return e.tree.Insert(key, rid) -} - -func (e *Engine) applyDeleteInMemory(key uint64) error { - rid, err := e.tree.Find(key) - if err == index.ErrKeyNotFound { - return nil - } - if err != nil { - return err - } - - page, err := e.bpm.FetchPage(rid.PageID) - if err == nil { - _ = page.DeleteTuple(rid.SlotID) - _ = e.bpm.UnpinPage(rid.PageID, true) - } - - return e.tree.Delete(key) -} diff --git a/internal/api/engine_checkpoint.go b/internal/api/engine_checkpoint.go index 580346f..91036ba 100644 --- a/internal/api/engine_checkpoint.go +++ b/internal/api/engine_checkpoint.go @@ -1,7 +1,10 @@ package api import ( + "context" "fmt" + "log" + "time" "github.com/ChromaBeast/beastdb/internal/storage" ) @@ -46,5 +49,32 @@ func (e *Engine) Checkpoint() error { if err := e.bpm.FlushAll(); err != nil { return fmt.Errorf("checkpoint flush dirty pages: %w", err) } + // Dirty pages (and any file extension) must be durable before the WAL that + // covers them is archived. + if err := e.disk.Sync(); err != nil { + return fmt.Errorf("checkpoint sync data file: %w", err) + } return e.wal.Checkpoint() } + +// StartAutoCheckpoint checkpoints whenever the WAL exceeds maxWALBytes, polling every +// interval, until ctx is cancelled. It bounds WAL size and crash-recovery time. +func (e *Engine) StartAutoCheckpoint(ctx context.Context, interval time.Duration, maxWALBytes int64) { + go func() { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if e.wal.FileSize() < maxWALBytes { + continue + } + if err := e.Checkpoint(); err != nil { + log.Printf("auto-checkpoint failed: %v", err) + } + } + } + }() +} diff --git a/internal/api/engine_delete.go b/internal/api/engine_delete.go index dd300dc..93c08aa 100644 --- a/internal/api/engine_delete.go +++ b/internal/api/engine_delete.go @@ -3,7 +3,6 @@ package api import ( "encoding/binary" - "github.com/ChromaBeast/beastdb/internal/index" "github.com/ChromaBeast/beastdb/internal/wal" ) @@ -18,30 +17,18 @@ func (e *Engine) Delete(key uint64) error { } func (e *Engine) deleteLocked(key uint64) error { + if e.failed != nil { + return e.failed + } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) lsn, err := e.wal.Write(wal.OpDelete, keyBytes[:], nil) if err != nil { - return err - } - - rid, err := e.tree.Find(key) - if err == index.ErrKeyNotFound { - e.notifyCommit(lsn, wal.OpDelete, keyBytes[:], nil) - return nil - } - if err != nil { - return err - } - - page, err := e.bpm.FetchPage(rid.PageID) - if err == nil { - _ = page.DeleteTuple(rid.SlotID) - _ = e.bpm.UnpinPage(rid.PageID, true) + return e.halt(err) } - if err := e.tree.Delete(key); err != nil { - return err + if err := e.applyDeleteInMemory(key); err != nil { + return e.halt(err) } e.unindexValueLocked(key) diff --git a/internal/api/engine_ops.go b/internal/api/engine_ops.go index 062cf49..49c9e96 100644 --- a/internal/api/engine_ops.go +++ b/internal/api/engine_ops.go @@ -54,42 +54,17 @@ func (e *Engine) PutIfAbsent(key uint64, value []byte) error { } func (e *Engine) putLocked(key uint64, value []byte) error { + if e.failed != nil { + return e.failed + } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) lsn, err := e.wal.Write(wal.OpPut, keyBytes[:], value) if err != nil { - return err - } - - page, err := e.bpm.FetchPage(e.activeDataPage) - if err != nil { - return err + return e.halt(err) } - - slotID, err := page.InsertTuple(value) - if err == storage.ErrPageFull { - _ = e.bpm.UnpinPage(e.activeDataPage, false) - newPage, newID, err := e.bpm.NewPage() - if err != nil { - return err - } - if err := e.updateMetaActiveData(newID); err != nil { - _ = e.bpm.UnpinPage(newID, false) - return err - } - e.activeDataPage = newID - slotID, err = newPage.InsertTuple(value) - if err != nil { - _ = e.bpm.UnpinPage(newID, false) - return err - } - page = newPage - } - _ = e.bpm.UnpinPage(e.activeDataPage, true) - - rid := storage.RID{PageID: e.activeDataPage, SlotID: slotID} - if err := e.tree.Insert(key, rid); err != nil { - return err + if err := e.applyPutInMemory(key, value); err != nil { + return e.halt(err) } e.indexValueLocked(key, value) diff --git a/internal/api/engine_space.go b/internal/api/engine_space.go new file mode 100644 index 0000000..e0d7bd1 --- /dev/null +++ b/internal/api/engine_space.go @@ -0,0 +1,132 @@ +package api + +import ( + "fmt" + + "github.com/ChromaBeast/beastdb/internal/index" + "github.com/ChromaBeast/beastdb/internal/storage" +) + +// halt records an unrecoverable mid-apply failure. The WAL already holds the +// mutation, so memory and disk may disagree until the process restarts and +// replays it; all further writes are refused until then. +func (e *Engine) halt(err error) error { + if e.failed == nil { + e.failed = fmt.Errorf("engine halted, restart to replay WAL: %w", err) + } + return e.failed +} + +// insertTuple stores value in a data page, reusing dead space before growing the file. +// Caller must hold e.mu exclusively. +func (e *Engine) insertTuple(value []byte) (storage.RID, error) { + page, err := e.bpm.FetchPage(e.activeDataPage) + if err != nil { + return storage.RID{}, err + } + slotID, err := page.InsertTuple(value) + if err == storage.ErrPageFull { + // Tombstones on the active page may be holding reclaimable bytes. + page.Compact() + slotID, err = page.InsertTuple(value) + } + if err == nil { + _ = e.bpm.UnpinPage(e.activeDataPage, true) + return storage.RID{PageID: e.activeDataPage, SlotID: slotID}, nil + } + _ = e.bpm.UnpinPage(e.activeDataPage, true) + if err != storage.ErrPageFull { + return storage.RID{}, err + } + + newPage, newID, err := e.nextDataPage(len(value)) + if err != nil { + return storage.RID{}, err + } + if err := e.updateMetaActiveData(newID); err != nil { + _ = e.bpm.UnpinPage(newID, false) + return storage.RID{}, err + } + e.activeDataPage = newID + slotID, err = newPage.InsertTuple(value) + _ = e.bpm.UnpinPage(newID, err == nil) + if err != nil { + return storage.RID{}, err + } + return storage.RID{PageID: newID, SlotID: slotID}, nil +} + +// nextDataPage returns a pinned page with room for size bytes: a compacted page +// with dead tuples if one fits, otherwise a freshly allocated page. +func (e *Engine) nextDataPage(size int) (*storage.SlottedPage, uint64, error) { + for id := range e.reclaim { + delete(e.reclaim, id) + if id == e.activeDataPage { + continue + } + p, err := e.bpm.FetchPage(id) + if err != nil { + continue + } + p.Compact() + if p.FreeSpace() >= size+storage.SlotSize { + _ = e.bpm.UnpinPage(id, true) + p, err = e.bpm.FetchPage(id) + if err == nil { + return p, id, nil + } + continue + } + _ = e.bpm.UnpinPage(id, true) + } + return e.bpm.NewPage() +} + +// freeTuple tombstones the tuple at rid and queues its page for space reuse. +func (e *Engine) freeTuple(rid storage.RID) { + page, err := e.bpm.FetchPage(rid.PageID) + if err != nil { + return + } + _ = page.DeleteTuple(rid.SlotID) + _ = e.bpm.UnpinPage(rid.PageID, true) + if rid.PageID != e.activeDataPage { + if e.reclaim == nil { + e.reclaim = make(map[uint64]struct{}) + } + e.reclaim[rid.PageID] = struct{}{} + } +} + +// applyPutInMemory writes value for key, freeing any tuple the key previously pointed at. +// It is idempotent per key, which is what makes WAL replay after a partial flush safe. +func (e *Engine) applyPutInMemory(key uint64, value []byte) error { + oldRID, err := e.tree.Find(key) + hadOld := err == nil + if err != nil && err != index.ErrKeyNotFound { + return err + } + rid, err := e.insertTuple(value) + if err != nil { + return err + } + if err := e.tree.Insert(key, rid); err != nil { + return err + } + if hadOld { + e.freeTuple(oldRID) + } + return nil +} + +func (e *Engine) applyDeleteInMemory(key uint64) error { + rid, err := e.tree.Find(key) + if err == index.ErrKeyNotFound { + return nil + } + if err != nil { + return err + } + e.freeTuple(rid) + return e.tree.Delete(key) +} diff --git a/internal/api/engine_space_test.go b/internal/api/engine_space_test.go new file mode 100644 index 0000000..37e7583 --- /dev/null +++ b/internal/api/engine_space_test.go @@ -0,0 +1,125 @@ +package api + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestOverwriteAndDeleteReuseSpace(t *testing.T) { + dir := t.TempDir() + e, err := NewEngine(filepath.Join(dir, "d.bin"), filepath.Join(dir, "d.wal"), 32) + if err != nil { + t.Fatal(err) + } + defer e.Close() + + val := make([]byte, 500) + for round := 0; round < 3; round++ { + for k := uint64(1); k <= 100; k++ { + if err := e.Put(k, val); err != nil { + t.Fatal(err) + } + } + } + pagesAfter3 := e.disk.NumPages() + for round := 0; round < 20; round++ { + for k := uint64(1); k <= 100; k++ { + if err := e.Put(k, val); err != nil { + t.Fatal(err) + } + } + } + if grew := e.disk.NumPages() - pagesAfter3; grew > 4 { + t.Fatalf("file grew by %d pages across overwrites; dead tuples not reclaimed", grew) + } + for k := uint64(1); k <= 100; k++ { + if v, ok, err := e.Get(k); err != nil || !ok || len(v) != 500 { + t.Fatalf("key %d unreadable after reuse: ok=%v err=%v", k, ok, err) + } + } +} + +func TestHaltBlocksWritesAfterApplyFailure(t *testing.T) { + dir := t.TempDir() + e, err := NewEngine(filepath.Join(dir, "h.bin"), filepath.Join(dir, "h.wal"), 32) + if err != nil { + t.Fatal(err) + } + defer e.Close() + if err := e.Put(1, []byte("a")); err != nil { + t.Fatal(err) + } + e.mu.Lock() + e.halt(fmt.Errorf("injected")) + e.mu.Unlock() + if err := e.Put(2, []byte("b")); err == nil { + t.Fatal("write succeeded on halted engine") + } + if err := e.BatchWrite([]BatchOperation{{Type: BatchOpPut, Key: 3, Value: []byte("c")}}); err == nil { + t.Fatal("batch succeeded on halted engine") + } + if _, ok, _ := e.Get(1); !ok { + t.Fatal("reads must keep working while halted") + } +} + +// Replaying a WAL tail whose effects were already flushed must not duplicate or orphan data. +func TestReplayOfAlreadyFlushedTailIsIdempotent(t *testing.T) { + dir := t.TempDir() + db, wl := filepath.Join(dir, "r.bin"), filepath.Join(dir, "r.wal") + e, err := NewEngine(db, wl, 32) + if err != nil { + t.Fatal(err) + } + for k := uint64(1); k <= 50; k++ { + _ = e.Put(k, []byte(fmt.Sprintf("v%d", k))) + } + _ = e.Delete(7) + // Flush pages but pretend the checkpoint LSN was never recorded: crash after flush. + if err := e.bpm.FlushAll(); err != nil { + t.Fatal(err) + } + walCopy, _ := os.ReadFile(wl) + _ = e.Close() + _ = os.WriteFile(wl, walCopy, 0600) + + // Reset the checkpoint LSN to 0 so the whole WAL replays over flushed pages. + e2, err := NewEngine(db, wl, 32) + if err != nil { + t.Fatal(err) + } + defer e2.Close() + for k := uint64(1); k <= 50; k++ { + v, ok, err := e2.Get(k) + if k == 7 { + if ok { + t.Fatal("deleted key resurrected") + } + continue + } + if err != nil || !ok || string(v) != fmt.Sprintf("v%d", k) { + t.Fatalf("key %d: %q ok=%v err=%v", k, v, ok, err) + } + } +} + +func TestAutoCheckpointRotatesWAL(t *testing.T) { + dir := t.TempDir() + e, err := NewEngine(filepath.Join(dir, "c.bin"), filepath.Join(dir, "c.wal"), 32) + if err != nil { + t.Fatal(err) + } + defer e.Close() + for k := uint64(1); k <= 200; k++ { + _ = e.Put(k, make([]byte, 100)) + } + before := e.wal.FileSize() + if err := e.Checkpoint(); err != nil { + t.Fatal(err) + } + if after := e.wal.FileSize(); after >= before { + t.Fatalf("WAL not rotated: %d -> %d", before, after) + } +} diff --git a/internal/api/grpc_auth.go b/internal/api/grpc_auth.go index efaa0f5..3f0a115 100644 --- a/internal/api/grpc_auth.go +++ b/internal/api/grpc_auth.go @@ -2,6 +2,7 @@ package api import ( "context" + "crypto/subtle" "strings" "google.golang.org/grpc" @@ -70,7 +71,7 @@ func (a *AuthInterceptor) authorize(ctx context.Context) error { return status.Errorf(codes.Unauthenticated, "authorization must be Bearer token") } token := strings.TrimSpace(header[7:]) - if a.staticToken != "" && token == a.staticToken { + if a.staticToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(a.staticToken)) == 1 { return nil } if a.validator != nil && a.validator.Validate(token) { diff --git a/internal/api/recovery.go b/internal/api/recovery.go index f1e5c44..cd8e249 100644 --- a/internal/api/recovery.go +++ b/internal/api/recovery.go @@ -3,7 +3,6 @@ package api import ( "encoding/binary" - "github.com/ChromaBeast/beastdb/internal/index" "github.com/ChromaBeast/beastdb/internal/storage" "github.com/ChromaBeast/beastdb/internal/wal" ) @@ -74,49 +73,14 @@ func (e *Engine) applyRecordLocked(rec *wal.Record) error { switch rec.Type { case wal.OpPut: - page, err := e.bpm.FetchPage(e.activeDataPage) - if err != nil { - return err - } - - slotID, err := page.InsertTuple(rec.Value) - if err == storage.ErrPageFull { - _ = e.bpm.UnpinPage(e.activeDataPage, false) - newPage, newID, err := e.bpm.NewPage() - if err != nil { - return err - } - e.activeDataPage = newID - slotID, err = newPage.InsertTuple(rec.Value) - if err != nil { - _ = e.bpm.UnpinPage(newID, false) - return err - } - page = newPage - } - _ = e.bpm.UnpinPage(e.activeDataPage, true) - - rid := storage.RID{PageID: e.activeDataPage, SlotID: slotID} - if err := e.tree.Insert(key, rid); err != nil { + if err := e.applyPutInMemory(key, rec.Value); err != nil { return err } e.indexValueLocked(key, rec.Value) return nil case wal.OpDelete: - rid, err := e.tree.Find(key) - if err == index.ErrKeyNotFound { - return nil - } - if err != nil { - return err - } - page, err := e.bpm.FetchPage(rid.PageID) - if err == nil { - _ = page.DeleteTuple(rid.SlotID) - _ = e.bpm.UnpinPage(rid.PageID, true) - } - if err := e.tree.Delete(key); err != nil { + if err := e.applyDeleteInMemory(key); err != nil { return err } e.unindexValueLocked(key) diff --git a/internal/replication/leader.go b/internal/replication/leader.go index 676f915..d0b5ba4 100644 --- a/internal/replication/leader.go +++ b/internal/replication/leader.go @@ -57,9 +57,15 @@ func (s *LeaderServer) StreamWAL(req *beastv1.StreamWALRequest, stream grpc.Serv var maxReplayedLSN uint64 - // 2. Catch-up Phase: Replay historical records from local WAL file - _, err := wal.Replay(s.walPath, func(rec *wal.Record) error { - if rec.LSN >= req.FromLsn { + // 2. Catch-up Phase: replay the archived WAL generation (kept by checkpoint + // rotation) and then the active WAL, so a follower that is behind by less than + // one checkpoint still catches up. Anything older surfaces as a gap on the + // follower, which fails loudly rather than diverging silently. + for _, path := range []string{s.walPath + ".bak", s.walPath} { + _, err := wal.Replay(path, func(rec *wal.Record) error { + if rec.LSN < req.FromLsn || rec.LSN <= maxReplayedLSN { + return nil + } msg := &beastv1.WALRecordMessage{ Lsn: rec.LSN, OpType: uint32(rec.Type), @@ -69,14 +75,12 @@ func (s *LeaderServer) StreamWAL(req *beastv1.StreamWALRequest, stream grpc.Serv if err := stream.Send(msg); err != nil { return err } - if rec.LSN > maxReplayedLSN { - maxReplayedLSN = rec.LSN - } + maxReplayedLSN = rec.LSN + return nil + }) + if err != nil { + return status.Errorf(codes.Internal, "wal replay error: %v", err) } - return nil - }) - if err != nil { - return status.Errorf(codes.Internal, "wal replay error: %v", err) } // 3. Real-time Phase: Drain live committed records, skipping any already replayed diff --git a/internal/storage/disk_manager.go b/internal/storage/disk_manager.go index 46b2f1e..229563f 100644 --- a/internal/storage/disk_manager.go +++ b/internal/storage/disk_manager.go @@ -21,7 +21,7 @@ type DiskManager struct { // OpenDiskManager opens or creates a database file for 4KB aligned page I/O. func OpenDiskManager(path string) (*DiskManager, error) { - file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0600) if err != nil { return nil, err } diff --git a/internal/storage/page.go b/internal/storage/page.go index 219e20d..a42953f 100644 --- a/internal/storage/page.go +++ b/internal/storage/page.go @@ -120,7 +120,7 @@ func (p *SlottedPage) Compact() { for i := uint16(0); i < h.SlotCount; i++ { slot := ReadSlot(p.data, i) - if slot.Offset > 0 && slot.Length > 0 { + if slot.Offset > 0 { newUpper -= slot.Length copy(temp[newUpper:newUpper+slot.Length], p.data[slot.Offset:slot.Offset+slot.Length]) WriteSlot(temp, i, Slot{Offset: newUpper, Length: slot.Length}) diff --git a/internal/wal/wal.go b/internal/wal/wal.go index d659d9a..d637fb6 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -3,6 +3,7 @@ package wal import ( "io" "os" + "path/filepath" "sync" ) @@ -17,7 +18,7 @@ type WAL struct { // OpenWAL opens an existing WAL file or creates a new one at the specified path. func OpenWAL(path string, syncOnWrite bool) (*WAL, error) { - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) if err != nil { return nil, err } @@ -139,15 +140,20 @@ func (w *WAL) Checkpoint() error { } // Open a fresh log file for subsequent writes. - newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) + newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) if err != nil { return err } w.file = newFile + + // Persist the rename and the new file's directory entry. + if dir, err := os.Open(filepath.Dir(w.path)); err == nil { + _ = dir.Sync() + _ = dir.Close() + } return nil } - func findHighestLSN(file *os.File) (uint64, error) { if _, err := file.Seek(0, io.SeekStart); err != nil { return 0, err @@ -183,4 +189,3 @@ func (w *WAL) FileSize() int64 { } return 0 } - diff --git a/studio/next-env.d.ts b/studio/next-env.d.ts index 830fb59..2a4d166 100644 --- a/studio/next-env.d.ts +++ b/studio/next-env.d.ts @@ -1,6 +1,6 @@ -/// -/// -/// - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. +/// +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. From c774fb3ede5180348166c52a08a2480e2de0f02c Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Wed, 28 Oct 2026 15:57:09 +0530 Subject: [PATCH 02/16] feat(engine): conditional writes and scan limit, plus post-V1 roadmap plan Add ABSENT/PRESENT/VALUE_EQUALS preconditions to Put, Delete and BatchWrite, evaluated atomically under the engine lock (gRPC ABORTED on failure). Add ScanRequest.limit for paged scans. Only effects are logged to the WAL, so recovery and replication are unchanged. Document semantics and add the roadmap plan. Co-Authored-By: Claude Sonnet 5.5 --- ROADMAP.md | 4 + api/proto/beastdb.pb.go | 342 ++++++++++++++++++++++++----- api/proto/beastdb.proto | 28 +++ api/proto/beastdb_grpc.pb.go | 2 +- docs/roadmap-plan.md | 113 ++++++++++ docs/v1-semantics.md | 8 + internal/api/conditional_test.go | 247 +++++++++++++++++++++ internal/api/engine_batch.go | 13 ++ internal/api/engine_conditional.go | 106 +++++++++ internal/api/grpc_server.go | 55 ++++- 10 files changed, 852 insertions(+), 66 deletions(-) create mode 100644 docs/roadmap-plan.md create mode 100644 internal/api/conditional_test.go create mode 100644 internal/api/engine_conditional.go diff --git a/ROADMAP.md b/ROADMAP.md index 6d00cdc..0f4f488 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -120,6 +120,10 @@ Order is dependency-based, not a calendar promise. Initial planning range for on 7. [x] Operational probes: public `/healthz` and `/readyz` endpoints with [operator guide](docs/runbooks/operator-guide.md) and [disaster recovery runbook](docs/runbooks/disaster-recovery.md). 8. [x] Studio pagination in Table/Gallery views, whitespace-safe status filtering, and reproducible system benchmarks ([docs/benchmarks.md](docs/benchmarks.md)). +## Beyond V1 gates + +See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite) and limit-based scan pagination. + ## References - [PostgreSQL WAL archiving and recovery](https://www.postgresql.org/docs/17/continuous-archiving.html) — backup/restore design reference, not a BeastDB feature claim. diff --git a/api/proto/beastdb.pb.go b/api/proto/beastdb.pb.go index 687ae5d..9152404 100644 --- a/api/proto/beastdb.pb.go +++ b/api/proto/beastdb.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.12 -// protoc v7.36.0 +// protoc (unknown) // source: beastdb.proto package beastv1 @@ -70,6 +70,59 @@ func (BatchOpType) EnumDescriptor() ([]byte, []int) { return file_beastdb_proto_rawDescGZIP(), []int{0} } +// ConditionKind selects the check a conditional write applies to a key. +type ConditionKind int32 + +const ( + ConditionKind_CONDITION_KIND_NONE ConditionKind = 0 // no check + ConditionKind_CONDITION_KIND_ABSENT ConditionKind = 1 // key must not exist + ConditionKind_CONDITION_KIND_PRESENT ConditionKind = 2 // key must exist + ConditionKind_CONDITION_KIND_VALUE_EQUALS ConditionKind = 3 // key must exist with exactly `value` +) + +// Enum value maps for ConditionKind. +var ( + ConditionKind_name = map[int32]string{ + 0: "CONDITION_KIND_NONE", + 1: "CONDITION_KIND_ABSENT", + 2: "CONDITION_KIND_PRESENT", + 3: "CONDITION_KIND_VALUE_EQUALS", + } + ConditionKind_value = map[string]int32{ + "CONDITION_KIND_NONE": 0, + "CONDITION_KIND_ABSENT": 1, + "CONDITION_KIND_PRESENT": 2, + "CONDITION_KIND_VALUE_EQUALS": 3, + } +) + +func (x ConditionKind) Enum() *ConditionKind { + p := new(ConditionKind) + *p = x + return p +} + +func (x ConditionKind) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ConditionKind) Descriptor() protoreflect.EnumDescriptor { + return file_beastdb_proto_enumTypes[1].Descriptor() +} + +func (ConditionKind) Type() protoreflect.EnumType { + return &file_beastdb_proto_enumTypes[1] +} + +func (x ConditionKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ConditionKind.Descriptor instead. +func (ConditionKind) EnumDescriptor() ([]byte, []int) { + return file_beastdb_proto_rawDescGZIP(), []int{1} +} + type GetRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` @@ -175,9 +228,11 @@ func (x *GetResponse) GetFound() bool { } type PutRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Optional precondition on the key's current state, checked atomically with the write. + Condition *Condition `protobuf:"bytes,3,opt,name=condition,proto3" json:"condition,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -226,6 +281,13 @@ func (x *PutRequest) GetValue() []byte { return nil } +func (x *PutRequest) GetCondition() *Condition { + if x != nil { + return x.Condition + } + return nil +} + type PutResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` @@ -271,8 +333,10 @@ func (x *PutResponse) GetSuccess() bool { } type DeleteRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` + // Optional precondition on the key's current state, checked atomically with the delete. + Condition *Condition `protobuf:"bytes,2,opt,name=condition,proto3" json:"condition,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -314,6 +378,13 @@ func (x *DeleteRequest) GetKey() uint64 { return 0 } +func (x *DeleteRequest) GetCondition() *Condition { + if x != nil { + return x.Condition + } + return nil +} + type DeleteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` @@ -359,9 +430,12 @@ func (x *DeleteResponse) GetSuccess() bool { } type ScanRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - StartKey uint64 `protobuf:"varint,1,opt,name=start_key,json=startKey,proto3" json:"start_key,omitempty"` - EndKey uint64 `protobuf:"varint,2,opt,name=end_key,json=endKey,proto3" json:"end_key,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + StartKey uint64 `protobuf:"varint,1,opt,name=start_key,json=startKey,proto3" json:"start_key,omitempty"` + EndKey uint64 `protobuf:"varint,2,opt,name=end_key,json=endKey,proto3" json:"end_key,omitempty"` + // Maximum records to return; 0 means no limit. To resume a limited scan, + // pass last returned key + 1 as start_key. + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -410,6 +484,13 @@ func (x *ScanRequest) GetEndKey() uint64 { return 0 } +func (x *ScanRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + type KeyValueResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` @@ -739,8 +820,11 @@ func (x *BatchOperation) GetValue() []byte { } type BatchWriteRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Operations []*BatchOperation `protobuf:"bytes,1,rep,name=operations,proto3" json:"operations,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Operations []*BatchOperation `protobuf:"bytes,1,rep,name=operations,proto3" json:"operations,omitempty"` + // Preconditions evaluated before any operation applies; if any fails the + // whole batch is rejected with ABORTED and nothing is written. + Conditions []*KeyCondition `protobuf:"bytes,2,rep,name=conditions,proto3" json:"conditions,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -782,6 +866,13 @@ func (x *BatchWriteRequest) GetOperations() []*BatchOperation { return nil } +func (x *BatchWriteRequest) GetConditions() []*KeyCondition { + if x != nil { + return x.Conditions + } + return nil +} + type BatchWriteResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` @@ -834,6 +925,110 @@ func (x *BatchWriteResponse) GetAppliedCount() uint32 { return 0 } +type Condition struct { + state protoimpl.MessageState `protogen:"open.v1"` + Kind ConditionKind `protobuf:"varint,1,opt,name=kind,proto3,enum=beastdb.v1.ConditionKind" json:"kind,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` // used by CONDITION_KIND_VALUE_EQUALS + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Condition) Reset() { + *x = Condition{} + mi := &file_beastdb_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Condition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Condition) ProtoMessage() {} + +func (x *Condition) ProtoReflect() protoreflect.Message { + mi := &file_beastdb_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Condition.ProtoReflect.Descriptor instead. +func (*Condition) Descriptor() ([]byte, []int) { + return file_beastdb_proto_rawDescGZIP(), []int{15} +} + +func (x *Condition) GetKind() ConditionKind { + if x != nil { + return x.Kind + } + return ConditionKind_CONDITION_KIND_NONE +} + +func (x *Condition) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +type KeyCondition struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key uint64 `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"` + Condition *Condition `protobuf:"bytes,2,opt,name=condition,proto3" json:"condition,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *KeyCondition) Reset() { + *x = KeyCondition{} + mi := &file_beastdb_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *KeyCondition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KeyCondition) ProtoMessage() {} + +func (x *KeyCondition) ProtoReflect() protoreflect.Message { + mi := &file_beastdb_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KeyCondition.ProtoReflect.Descriptor instead. +func (*KeyCondition) Descriptor() ([]byte, []int) { + return file_beastdb_proto_rawDescGZIP(), []int{16} +} + +func (x *KeyCondition) GetKey() uint64 { + if x != nil { + return x.Key + } + return 0 +} + +func (x *KeyCondition) GetCondition() *Condition { + if x != nil { + return x.Condition + } + return nil +} + var File_beastdb_proto protoreflect.FileDescriptor const file_beastdb_proto_rawDesc = "" + @@ -846,20 +1041,23 @@ const file_beastdb_proto_rawDesc = "" + "\vGetResponse\x12\x10\n" + "\x03key\x18\x01 \x01(\x04R\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\x12\x14\n" + - "\x05found\x18\x03 \x01(\bR\x05found\"4\n" + + "\x05found\x18\x03 \x01(\bR\x05found\"i\n" + "\n" + "PutRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\x04R\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\fR\x05value\"'\n" + + "\x05value\x18\x02 \x01(\fR\x05value\x123\n" + + "\tcondition\x18\x03 \x01(\v2\x15.beastdb.v1.ConditionR\tcondition\"'\n" + "\vPutResponse\x12\x18\n" + - "\asuccess\x18\x01 \x01(\bR\asuccess\"!\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\"V\n" + "\rDeleteRequest\x12\x10\n" + - "\x03key\x18\x01 \x01(\x04R\x03key\"*\n" + + "\x03key\x18\x01 \x01(\x04R\x03key\x123\n" + + "\tcondition\x18\x02 \x01(\v2\x15.beastdb.v1.ConditionR\tcondition\"*\n" + "\x0eDeleteResponse\x12\x18\n" + - "\asuccess\x18\x01 \x01(\bR\asuccess\"C\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\"Y\n" + "\vScanRequest\x12\x1b\n" + "\tstart_key\x18\x01 \x01(\x04R\bstartKey\x12\x17\n" + - "\aend_key\x18\x02 \x01(\x04R\x06endKey\":\n" + + "\aend_key\x18\x02 \x01(\x04R\x06endKey\x12\x14\n" + + "\x05limit\x18\x03 \x01(\rR\x05limit\":\n" + "\x10KeyValueResponse\x12\x10\n" + "\x03key\x18\x01 \x01(\x04R\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\"L\n" + @@ -881,18 +1079,32 @@ const file_beastdb_proto_rawDesc = "" + "\x0eBatchOperation\x120\n" + "\aop_type\x18\x01 \x01(\x0e2\x17.beastdb.v1.BatchOpTypeR\x06opType\x12\x10\n" + "\x03key\x18\x02 \x01(\x04R\x03key\x12\x14\n" + - "\x05value\x18\x03 \x01(\fR\x05value\"O\n" + + "\x05value\x18\x03 \x01(\fR\x05value\"\x89\x01\n" + "\x11BatchWriteRequest\x12:\n" + "\n" + "operations\x18\x01 \x03(\v2\x1a.beastdb.v1.BatchOperationR\n" + - "operations\"S\n" + + "operations\x128\n" + + "\n" + + "conditions\x18\x02 \x03(\v2\x18.beastdb.v1.KeyConditionR\n" + + "conditions\"S\n" + "\x12BatchWriteResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + - "\rapplied_count\x18\x02 \x01(\rR\fappliedCount*]\n" + + "\rapplied_count\x18\x02 \x01(\rR\fappliedCount\"P\n" + + "\tCondition\x12-\n" + + "\x04kind\x18\x01 \x01(\x0e2\x19.beastdb.v1.ConditionKindR\x04kind\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value\"U\n" + + "\fKeyCondition\x12\x10\n" + + "\x03key\x18\x01 \x01(\x04R\x03key\x123\n" + + "\tcondition\x18\x02 \x01(\v2\x15.beastdb.v1.ConditionR\tcondition*]\n" + "\vBatchOpType\x12\x1d\n" + "\x19BATCH_OP_TYPE_UNSPECIFIED\x10\x00\x12\x15\n" + "\x11BATCH_OP_TYPE_PUT\x10\x01\x12\x18\n" + - "\x14BATCH_OP_TYPE_DELETE\x10\x022\xcf\x02\n" + + "\x14BATCH_OP_TYPE_DELETE\x10\x02*\x80\x01\n" + + "\rConditionKind\x12\x17\n" + + "\x13CONDITION_KIND_NONE\x10\x00\x12\x19\n" + + "\x15CONDITION_KIND_ABSENT\x10\x01\x12\x1a\n" + + "\x16CONDITION_KIND_PRESENT\x10\x02\x12\x1f\n" + + "\x1bCONDITION_KIND_VALUE_EQUALS\x10\x032\xcf\x02\n" + "\x0eBeastDBService\x126\n" + "\x03Get\x12\x16.beastdb.v1.GetRequest\x1a\x17.beastdb.v1.GetResponse\x126\n" + "\x03Put\x12\x16.beastdb.v1.PutRequest\x1a\x17.beastdb.v1.PutResponse\x12?\n" + @@ -916,48 +1128,56 @@ func file_beastdb_proto_rawDescGZIP() []byte { return file_beastdb_proto_rawDescData } -var file_beastdb_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_beastdb_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_beastdb_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_beastdb_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_beastdb_proto_goTypes = []any{ (BatchOpType)(0), // 0: beastdb.v1.BatchOpType - (*GetRequest)(nil), // 1: beastdb.v1.GetRequest - (*GetResponse)(nil), // 2: beastdb.v1.GetResponse - (*PutRequest)(nil), // 3: beastdb.v1.PutRequest - (*PutResponse)(nil), // 4: beastdb.v1.PutResponse - (*DeleteRequest)(nil), // 5: beastdb.v1.DeleteRequest - (*DeleteResponse)(nil), // 6: beastdb.v1.DeleteResponse - (*ScanRequest)(nil), // 7: beastdb.v1.ScanRequest - (*KeyValueResponse)(nil), // 8: beastdb.v1.KeyValueResponse - (*StreamWALRequest)(nil), // 9: beastdb.v1.StreamWALRequest - (*WALRecordMessage)(nil), // 10: beastdb.v1.WALRecordMessage - (*ReplicationAck)(nil), // 11: beastdb.v1.ReplicationAck - (*AckResponse)(nil), // 12: beastdb.v1.AckResponse - (*BatchOperation)(nil), // 13: beastdb.v1.BatchOperation - (*BatchWriteRequest)(nil), // 14: beastdb.v1.BatchWriteRequest - (*BatchWriteResponse)(nil), // 15: beastdb.v1.BatchWriteResponse + (ConditionKind)(0), // 1: beastdb.v1.ConditionKind + (*GetRequest)(nil), // 2: beastdb.v1.GetRequest + (*GetResponse)(nil), // 3: beastdb.v1.GetResponse + (*PutRequest)(nil), // 4: beastdb.v1.PutRequest + (*PutResponse)(nil), // 5: beastdb.v1.PutResponse + (*DeleteRequest)(nil), // 6: beastdb.v1.DeleteRequest + (*DeleteResponse)(nil), // 7: beastdb.v1.DeleteResponse + (*ScanRequest)(nil), // 8: beastdb.v1.ScanRequest + (*KeyValueResponse)(nil), // 9: beastdb.v1.KeyValueResponse + (*StreamWALRequest)(nil), // 10: beastdb.v1.StreamWALRequest + (*WALRecordMessage)(nil), // 11: beastdb.v1.WALRecordMessage + (*ReplicationAck)(nil), // 12: beastdb.v1.ReplicationAck + (*AckResponse)(nil), // 13: beastdb.v1.AckResponse + (*BatchOperation)(nil), // 14: beastdb.v1.BatchOperation + (*BatchWriteRequest)(nil), // 15: beastdb.v1.BatchWriteRequest + (*BatchWriteResponse)(nil), // 16: beastdb.v1.BatchWriteResponse + (*Condition)(nil), // 17: beastdb.v1.Condition + (*KeyCondition)(nil), // 18: beastdb.v1.KeyCondition } var file_beastdb_proto_depIdxs = []int32{ - 0, // 0: beastdb.v1.BatchOperation.op_type:type_name -> beastdb.v1.BatchOpType - 13, // 1: beastdb.v1.BatchWriteRequest.operations:type_name -> beastdb.v1.BatchOperation - 1, // 2: beastdb.v1.BeastDBService.Get:input_type -> beastdb.v1.GetRequest - 3, // 3: beastdb.v1.BeastDBService.Put:input_type -> beastdb.v1.PutRequest - 5, // 4: beastdb.v1.BeastDBService.Delete:input_type -> beastdb.v1.DeleteRequest - 7, // 5: beastdb.v1.BeastDBService.Scan:input_type -> beastdb.v1.ScanRequest - 14, // 6: beastdb.v1.BeastDBService.BatchWrite:input_type -> beastdb.v1.BatchWriteRequest - 9, // 7: beastdb.v1.ReplicationService.StreamWAL:input_type -> beastdb.v1.StreamWALRequest - 11, // 8: beastdb.v1.ReplicationService.Ack:input_type -> beastdb.v1.ReplicationAck - 2, // 9: beastdb.v1.BeastDBService.Get:output_type -> beastdb.v1.GetResponse - 4, // 10: beastdb.v1.BeastDBService.Put:output_type -> beastdb.v1.PutResponse - 6, // 11: beastdb.v1.BeastDBService.Delete:output_type -> beastdb.v1.DeleteResponse - 8, // 12: beastdb.v1.BeastDBService.Scan:output_type -> beastdb.v1.KeyValueResponse - 15, // 13: beastdb.v1.BeastDBService.BatchWrite:output_type -> beastdb.v1.BatchWriteResponse - 10, // 14: beastdb.v1.ReplicationService.StreamWAL:output_type -> beastdb.v1.WALRecordMessage - 12, // 15: beastdb.v1.ReplicationService.Ack:output_type -> beastdb.v1.AckResponse - 9, // [9:16] is the sub-list for method output_type - 2, // [2:9] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 17, // 0: beastdb.v1.PutRequest.condition:type_name -> beastdb.v1.Condition + 17, // 1: beastdb.v1.DeleteRequest.condition:type_name -> beastdb.v1.Condition + 0, // 2: beastdb.v1.BatchOperation.op_type:type_name -> beastdb.v1.BatchOpType + 14, // 3: beastdb.v1.BatchWriteRequest.operations:type_name -> beastdb.v1.BatchOperation + 18, // 4: beastdb.v1.BatchWriteRequest.conditions:type_name -> beastdb.v1.KeyCondition + 1, // 5: beastdb.v1.Condition.kind:type_name -> beastdb.v1.ConditionKind + 17, // 6: beastdb.v1.KeyCondition.condition:type_name -> beastdb.v1.Condition + 2, // 7: beastdb.v1.BeastDBService.Get:input_type -> beastdb.v1.GetRequest + 4, // 8: beastdb.v1.BeastDBService.Put:input_type -> beastdb.v1.PutRequest + 6, // 9: beastdb.v1.BeastDBService.Delete:input_type -> beastdb.v1.DeleteRequest + 8, // 10: beastdb.v1.BeastDBService.Scan:input_type -> beastdb.v1.ScanRequest + 15, // 11: beastdb.v1.BeastDBService.BatchWrite:input_type -> beastdb.v1.BatchWriteRequest + 10, // 12: beastdb.v1.ReplicationService.StreamWAL:input_type -> beastdb.v1.StreamWALRequest + 12, // 13: beastdb.v1.ReplicationService.Ack:input_type -> beastdb.v1.ReplicationAck + 3, // 14: beastdb.v1.BeastDBService.Get:output_type -> beastdb.v1.GetResponse + 5, // 15: beastdb.v1.BeastDBService.Put:output_type -> beastdb.v1.PutResponse + 7, // 16: beastdb.v1.BeastDBService.Delete:output_type -> beastdb.v1.DeleteResponse + 9, // 17: beastdb.v1.BeastDBService.Scan:output_type -> beastdb.v1.KeyValueResponse + 16, // 18: beastdb.v1.BeastDBService.BatchWrite:output_type -> beastdb.v1.BatchWriteResponse + 11, // 19: beastdb.v1.ReplicationService.StreamWAL:output_type -> beastdb.v1.WALRecordMessage + 13, // 20: beastdb.v1.ReplicationService.Ack:output_type -> beastdb.v1.AckResponse + 14, // [14:21] is the sub-list for method output_type + 7, // [7:14] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_beastdb_proto_init() } @@ -970,8 +1190,8 @@ func file_beastdb_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_beastdb_proto_rawDesc), len(file_beastdb_proto_rawDesc)), - NumEnums: 1, - NumMessages: 15, + NumEnums: 2, + NumMessages: 17, NumExtensions: 0, NumServices: 2, }, diff --git a/api/proto/beastdb.proto b/api/proto/beastdb.proto index aae4a0c..ea65016 100644 --- a/api/proto/beastdb.proto +++ b/api/proto/beastdb.proto @@ -32,6 +32,8 @@ message GetResponse { message PutRequest { uint64 key = 1; bytes value = 2; + // Optional precondition on the key's current state, checked atomically with the write. + Condition condition = 3; } message PutResponse { @@ -40,6 +42,8 @@ message PutResponse { message DeleteRequest { uint64 key = 1; + // Optional precondition on the key's current state, checked atomically with the delete. + Condition condition = 2; } message DeleteResponse { @@ -49,6 +53,9 @@ message DeleteResponse { message ScanRequest { uint64 start_key = 1; uint64 end_key = 2; + // Maximum records to return; 0 means no limit. To resume a limited scan, + // pass last returned key + 1 as start_key. + uint32 limit = 3; } message KeyValueResponse { @@ -91,9 +98,30 @@ message BatchOperation { message BatchWriteRequest { repeated BatchOperation operations = 1; + // Preconditions evaluated before any operation applies; if any fails the + // whole batch is rejected with ABORTED and nothing is written. + repeated KeyCondition conditions = 2; } message BatchWriteResponse { bool success = 1; uint32 applied_count = 2; } + +// ConditionKind selects the check a conditional write applies to a key. +enum ConditionKind { + CONDITION_KIND_NONE = 0; // no check + CONDITION_KIND_ABSENT = 1; // key must not exist + CONDITION_KIND_PRESENT = 2; // key must exist + CONDITION_KIND_VALUE_EQUALS = 3; // key must exist with exactly `value` +} + +message Condition { + ConditionKind kind = 1; + bytes value = 2; // used by CONDITION_KIND_VALUE_EQUALS +} + +message KeyCondition { + uint64 key = 1; + Condition condition = 2; +} diff --git a/api/proto/beastdb_grpc.pb.go b/api/proto/beastdb_grpc.pb.go index a6d7a97..6680805 100644 --- a/api/proto/beastdb_grpc.pb.go +++ b/api/proto/beastdb_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.2 -// - protoc v7.36.0 +// - protoc (unknown) // source: beastdb.proto package beastv1 diff --git a/docs/roadmap-plan.md b/docs/roadmap-plan.md new file mode 100644 index 0000000..97c0901 --- /dev/null +++ b/docs/roadmap-plan.md @@ -0,0 +1,113 @@ +# BeastDB roadmap plan: from V1 gates to a general-purpose database + +**Status:** proposed, 28 October 2026. Builds on [ROADMAP.md](../ROADMAP.md) (V1 gates M0–M6, first iteration done). Nothing below is shipped until it has tests and evidence on the release commit. + +**Principle:** BeastDB is domain-agnostic. Every item below is a general primitive or operational capability. Application schemas, partition names and key layouts belong to client projects (`partitions.json`, `-partition-config`), never to the engine. + +## Where BeastDB stands + +Done: B+ tree storage, WAL with CRC and recovery, atomic batch writes, checkpoints, leader–follower WAL streaming, token auth, TLS/mTLS, health and readiness probes, backup/restore CLI with a measured drill, Studio, CI, benchmarks. + +Gaps that real clients keep hitting, all generic: + +| Gap | Why it matters | +|---|---| +| No conditional or compare-and-swap writes | Read-modify-write races (counters, token rotation, uniqueness, "create if absent") cannot be made safe across more than one writer process. Clients fall back to process-local locks, which limits them to one writer. | +| Batches cannot carry preconditions | Multi-key invariants (move value A→B, swap an index entry) can't be enforced atomically. | +| 64-bit keys only, caller hashes identifiers | Hashing a string ID to fewer bits risks silent collisions. Clients need either wider keys or a safe pattern. | +| Scans are unbounded streams without resume tokens | Large ranges risk memory and lock pressure; clients cap results ad hoc. | +| No expiry (TTL) | Sessions, caches and job records must be swept by hand. | +| No change feed | Clients poll to learn about writes; replication already has a WAL stream that could be exposed. | +| Single role model | One shared token per service, no read-only or scoped tokens. | +| Backups are manual | No schedule, retention or off-host copy. | +| No official client libraries | Each application re-implements dialing, retries, auth metadata and key helpers. | +| Failover is manual | Documented, but operator-driven. | + +## Phases + +Order is dependency-based. Each phase ends with a runnable demo, tests and docs. + +### Phase A — Safe concurrent writes (highest value) + +1. **Conditional write primitive.** Add `expected` preconditions to `Put`/`Delete`: *absent*, *present*, or *value equals / version equals*. Return gRPC `ABORTED` (the idiomatic code for retryable concurrency conflicts). Implemented in the engine under the existing write lock, logged as normal WAL records, replicated unchanged. +2. **Conditional `BatchWrite`.** A batch carries a list of preconditions evaluated before any operation applies; all pass or nothing is written. +3. **Per-record version (optional).** A monotonically increasing version returned by `Get`/`Put`, usable as a precondition (optimistic concurrency without comparing whole values). +4. **Tests.** Model-based test against a reference map with concurrent writers; crash and replay tests prove preconditions are not re-evaluated on recovery or replication; race detector clean. +5. **API compatibility.** New protobuf fields only (additive); old clients keep working. Document in [v1-semantics.md](v1-semantics.md). + +**Status: implemented (conditional Put/Delete/BatchWrite; optional per-record versions deferred: `VALUE_EQUALS` covers the same need without an on-disk format change).** + +**Exit:** two independent processes can increment a counter, rotate a token or claim a unique name with no lost updates, shown by a stress test. + +### Phase B — Scan and key ergonomics + +1. **Paginated scans:** `limit` on `ScanRequest` and resume by last key + 1 are **implemented**. A server-enforced maximum page size and reverse/prefix scans remain. +2. **Reverse and prefix scans** over the uint64 range, with documented bounds semantics. +3. **Optional wide-key mode design.** Options to evaluate and decide with a written ADR: (a) keep uint64 and ship a documented collision-safe pattern plus a client helper, or (b) add a variable-length key space. Decision gated on benchmarks and on-disk-format migration cost. +4. **Key-layout helper library** (client side, optional convention, not required by the engine). + +**Exit:** a client can page through a 10M-key range with bounded memory on both sides. + +### Phase C — Lifecycle features + +1. **TTL.** Per-record expiry timestamp checked on read and removed by a background sweeper; expiry is deterministic and replicated. +2. **Change feed.** Expose the committed WAL (or a filtered key range) as a server stream with resume position, reusing replication's contiguous-LSN guarantees. Documented delivery semantics: at-least-once, ordered per feed. +3. **Bulk import** with idempotent upserts and progress reporting (extends `beastctl import`). + +**Exit:** an expiring record disappears on schedule on leader and follower; a consumer resumes a feed after restart without gaps. + +### Phase D — Security and multi-tenancy primitives + +1. **Scoped tokens.** Roles (read-only, read-write, admin) and optional key-range scopes per token, enforced in gRPC, HTTP, Studio and CLI. +2. **Token lifecycle:** create, rotate, revoke, audit log of administrative actions; denied-action tests. +3. **Rate limits** on login and expensive scans. +4. **Verified TLS defaults** in the reference Compose file and certificate-rotation runbook. + +**Exit:** a service holding a read-only, range-scoped token provably cannot write or read outside its range. + +### Phase E — Operations + +1. **Metrics endpoint** (Prometheus-compatible) covering request/error/latency, WAL size and sync latency, dirty/pinned pages, checkpoint age, free disk, leader/follower LSNs and lag. +2. **Scheduled backups** with retention, checksum verification and optional off-host copy; restore drill runs in CI. +3. **Promotion tooling:** `beastctl promote`/`fence` with LSN checks, measured RPO/RTO. Automatic failover only after consensus research (out of V1). +4. **Integrity check command** that validates pages, WAL and index/data consistency, failing closed with diagnostics. + +**Exit:** an operator is alerted to a lagging replica and completes a rehearsed promotion within the stated RTO. + +### Phase F — Developer experience + +1. **Official clients:** Go, TypeScript and Python packages with connection management, retry/backoff, auth, TLS, conditional-write helpers, pagination iterators and an in-memory test double that matches server semantics. +2. **`beastctl`:** stable exit codes, JSON output, timeouts, safe restore prompts. +3. **Studio:** finish pagination and filters everywhere, honest loaded-record counts, keyboard and screen-reader support. +4. **One end-to-end tutorial** from install to upgrade, tested in CI. + +**Exit:** a new developer completes the tutorial without reading source. + +### Phase G — Release validation (V1 → V1.x) + +Mixed read/write/scan benchmarks at several data sizes, soak, disk-full, partition and crash-loop campaigns, upgrade/rollback tests, signed artifacts with SBOM, evidence index per gate. Features are frozen before the tag. + +## How this improves BeastDB + +| Change | Improvement | +|---|---| +| Conditional writes and batches | Removes the biggest correctness limit: multiple writer processes become safe. Counters, uniqueness, token rotation and index maintenance stop depending on application-side locks. | +| Pagination and scan limits | Bounds memory and lock time, so a wide scan can no longer degrade other clients. | +| Key ergonomics decision | Makes silent key collisions a documented, tested concern instead of each client's private risk. | +| TTL | Sessions, caches and job records clean themselves; storage no longer grows from forgotten records. | +| Change feed | Clients react to writes without polling, which supports notifications, cache invalidation and downstream indexing. | +| Scoped tokens and audit | Least-privilege per service; a leaked read-only token can't modify data. | +| Metrics, scheduled backups, integrity check | Failures are detected and recoverable before users notice; recovery time and data-loss window are measured. | +| Official clients | One tested implementation of dialing, retries and semantics replaces per-application copies, cutting integration bugs. | +| Evidence-gated release | Claims in the README match measured behavior, so users can trust the guarantees. | + +## Non-goals for V1/V1.x + +SQL, joins, multi-key interactive transactions, automatic failover or consensus, multi-leader writes, hosted service. Conditional batches cover single-node atomic multi-key updates only. + +## Suggested sequencing and risk + +1. Phase A first: it unblocks safe multi-writer deployments and is small relative to its value. +2. Phases B and E can proceed in parallel after A (separate files and subsystems); C and D follow. +3. Phase F runs alongside, updating clients as each primitive lands. +4. Highest risks: on-disk format changes (versions, TTL, wide keys) need migration and rollback tests; WAL record changes need replication compatibility tests. Mitigation: ADR per format change, feature flags, mixed-version tests before release. diff --git a/docs/v1-semantics.md b/docs/v1-semantics.md index 3950468..5924c0c 100644 --- a/docs/v1-semantics.md +++ b/docs/v1-semantics.md @@ -30,6 +30,13 @@ * **Atomicity:** All-or-nothing. On crash recovery, uncommitted or partial batches are rolled back; fully synced batches are replayed completely. * **Limits:** Recommended batch limit of 1,000 operations or 2 MB aggregate payload. +### Conditional writes: `Put`/`Delete` with `Condition`, `BatchWrite` with `conditions` +* **Conditions:** `ABSENT` (key must not exist), `PRESENT` (key must exist), `VALUE_EQUALS` (key must exist with exactly the given bytes). Omitted or `NONE` means unconditional. +* **Atomicity:** The check and the write happen under the same exclusive engine lock, so no other writer can interleave. For batches, every condition is evaluated before any operation applies: all hold and the whole batch commits, or none is written. +* **Failure:** gRPC status `ABORTED` ("precondition failed"); the caller should re-read and retry. Read-only replicas still return `FAILED_PRECONDITION`. +* **Durability and replication:** Only the resulting mutation is logged (a normal `OpPut`/`OpDelete`/`OpBatch` frame). Preconditions are never re-evaluated on recovery or on followers. +* **Scope:** Single node, single leader. A compare-and-swap loop (`Get`, then `Put` with `VALUE_EQUALS`) is safe across any number of client processes writing to the same leader. + ### `Get(key uint64) ([]byte, bool, error)` * **Consistency:** Read-your-own-writes on the leader. * **Return Values:** Returns `(value, true, nil)` if found, or `(nil, false, nil)` if missing. Returns a non-nil error only upon hardware/storage failure. @@ -37,6 +44,7 @@ ### `Scan(startKey, endKey uint64) (*Cursor, error)` * **Ordering:** Strictly ascending numerical order along the B+ Tree leaf chain. * **Bounds:** Inclusive `[startKey, endKey]`. +* **Pagination (gRPC):** `ScanRequest.limit` caps the records returned (0 = unlimited). To resume, pass last returned key + 1 as `start_key`. * **Resource Safety:** Cursors hold buffer pool page pins during traversal and **must** be closed via `cursor.Close()`. --- diff --git a/internal/api/conditional_test.go b/internal/api/conditional_test.go new file mode 100644 index 0000000..7f957ce --- /dev/null +++ b/internal/api/conditional_test.go @@ -0,0 +1,247 @@ +package api + +import ( + "context" + "errors" + "io" + "path/filepath" + "strconv" + "sync" + "testing" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newCondEngine(t *testing.T) (*Engine, string, string) { + t.Helper() + dir := t.TempDir() + db, wl := filepath.Join(dir, "c.bin"), filepath.Join(dir, "c.wal") + e, err := NewEngine(db, wl, 50) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.Close() }) + return e, db, wl +} + +func TestPutIfConditions(t *testing.T) { + e, _, _ := newCondEngine(t) + + if err := e.PutIf(1, []byte("a"), Condition{Kind: CondPresent}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("present on missing key: got %v", err) + } + if err := e.PutIf(1, []byte("a"), Condition{Kind: CondAbsent}); err != nil { + t.Fatalf("absent on missing key: %v", err) + } + if err := e.PutIf(1, []byte("b"), Condition{Kind: CondAbsent}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("absent on existing key: got %v", err) + } + if err := e.PutIf(1, []byte("b"), Condition{Kind: CondValueEquals, Value: []byte("zzz")}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("value mismatch: got %v", err) + } + if err := e.PutIf(1, []byte("b"), Condition{Kind: CondValueEquals, Value: []byte("a")}); err != nil { + t.Fatalf("value match: %v", err) + } + if v, _, _ := e.Get(1); string(v) != "b" { + t.Fatalf("got %q want b", v) + } + if err := e.PutIf(2, []byte("x"), Condition{Kind: CondValueEquals, Value: []byte("a")}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("value-equals on missing key: got %v", err) + } + if _, found, _ := e.Get(2); found { + t.Fatal("failed conditional put must not write") + } +} + +func TestDeleteIf(t *testing.T) { + e, _, _ := newCondEngine(t) + _ = e.Put(1, []byte("a")) + if err := e.DeleteIf(1, Condition{Kind: CondValueEquals, Value: []byte("no")}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("got %v", err) + } + if _, found, _ := e.Get(1); !found { + t.Fatal("record deleted despite failed condition") + } + if err := e.DeleteIf(1, Condition{Kind: CondValueEquals, Value: []byte("a")}); err != nil { + t.Fatal(err) + } + if err := e.DeleteIf(1, Condition{Kind: CondPresent}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("delete of missing key with present condition: got %v", err) + } +} + +func TestBatchWriteIfIsAllOrNothing(t *testing.T) { + e, _, _ := newCondEngine(t) + _ = e.Put(1, []byte("old")) + + ops := []BatchOperation{ + {Type: BatchOpPut, Key: 2, Value: []byte("new")}, + {Type: BatchOpDelete, Key: 1}, + } + err := e.BatchWriteIf(ops, []KeyCondition{ + {Key: 1, Cond: Condition{Kind: CondValueEquals, Value: []byte("old")}}, + {Key: 2, Cond: Condition{Kind: CondPresent}}, // fails + }) + if !errors.Is(err, ErrConditionFailed) { + t.Fatalf("got %v", err) + } + if _, found, _ := e.Get(2); found { + t.Fatal("partial batch applied") + } + if _, found, _ := e.Get(1); !found { + t.Fatal("delete applied despite failed condition") + } + + err = e.BatchWriteIf(ops, []KeyCondition{ + {Key: 1, Cond: Condition{Kind: CondValueEquals, Value: []byte("old")}}, + {Key: 2, Cond: Condition{Kind: CondAbsent}}, + }) + if err != nil { + t.Fatal(err) + } + if _, found, _ := e.Get(1); found { + t.Fatal("key 1 should be deleted") + } + if v, _, _ := e.Get(2); string(v) != "new" { + t.Fatalf("key 2 = %q", v) + } +} + +// Many writers increment one counter with compare-and-swap; no update may be lost. +func TestConcurrentCompareAndSwapCounter(t *testing.T) { + e, _, _ := newCondEngine(t) + _ = e.Put(7, []byte("0")) + + const workers, perWorker = 8, 25 + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < perWorker; i++ { + for { + cur, _, err := e.Get(7) + if err != nil { + t.Error(err) + return + } + n, _ := strconv.Atoi(string(cur)) + err = e.PutIf(7, []byte(strconv.Itoa(n+1)), Condition{Kind: CondValueEquals, Value: cur}) + if err == nil { + break + } + if !errors.Is(err, ErrConditionFailed) { + t.Error(err) + return + } + } + } + }() + } + wg.Wait() + v, _, _ := e.Get(7) + if string(v) != strconv.Itoa(workers*perWorker) { + t.Fatalf("lost updates: counter = %s, want %d", v, workers*perWorker) + } +} + +// Conditional writes log only their effect; reopening must reproduce the state. +func TestConditionalWritesSurviveRestart(t *testing.T) { + dir := t.TempDir() + db, wl := filepath.Join(dir, "r.bin"), filepath.Join(dir, "r.wal") + e, err := NewEngine(db, wl, 50) + if err != nil { + t.Fatal(err) + } + _ = e.PutIf(1, []byte("a"), Condition{Kind: CondAbsent}) + _ = e.PutIf(1, []byte("b"), Condition{Kind: CondValueEquals, Value: []byte("a")}) + _ = e.PutIf(1, []byte("c"), Condition{Kind: CondValueEquals, Value: []byte("a")}) // fails + _ = e.BatchWriteIf([]BatchOperation{{Type: BatchOpPut, Key: 2, Value: []byte("z")}}, + []KeyCondition{{Key: 1, Cond: Condition{Kind: CondValueEquals, Value: []byte("b")}}}) + _ = e.Close() + + e2, err := NewEngine(db, wl, 50) + if err != nil { + t.Fatal(err) + } + defer e2.Close() + if v, _, _ := e2.Get(1); string(v) != "b" { + t.Fatalf("key 1 = %q after restart", v) + } + if v, _, _ := e2.Get(2); string(v) != "z" { + t.Fatalf("key 2 = %q after restart", v) + } +} + +func TestGRPCConditionalAndScanLimit(t *testing.T) { + client, cleanup := setupTestGRPCServer(t) + defer cleanup() + ctx := context.Background() + + absent := &beastv1.Condition{Kind: beastv1.ConditionKind_CONDITION_KIND_ABSENT} + if _, err := client.Put(ctx, &beastv1.PutRequest{Key: 5, Value: []byte("v1"), Condition: absent}); err != nil { + t.Fatal(err) + } + _, err := client.Put(ctx, &beastv1.PutRequest{Key: 5, Value: []byte("v2"), Condition: absent}) + if status.Code(err) != codes.Aborted { + t.Fatalf("want Aborted, got %v", err) + } + eq := &beastv1.Condition{Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: []byte("v1")} + if _, err := client.Delete(ctx, &beastv1.DeleteRequest{Key: 5, Condition: &beastv1.Condition{ + Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: []byte("nope")}}); status.Code(err) != codes.Aborted { + t.Fatalf("want Aborted, got %v", err) + } + if _, err := client.Put(ctx, &beastv1.PutRequest{Key: 5, Value: []byte("v2"), Condition: eq}); err != nil { + t.Fatal(err) + } + + _, err = client.BatchWrite(ctx, &beastv1.BatchWriteRequest{ + Operations: []*beastv1.BatchOperation{{OpType: beastv1.BatchOpType_BATCH_OP_TYPE_PUT, Key: 6, Value: []byte("x")}}, + Conditions: []*beastv1.KeyCondition{{Key: 5, Condition: eq}}, // value is now v2 + }) + if status.Code(err) != codes.Aborted { + t.Fatalf("want Aborted, got %v", err) + } + if r, _ := client.Get(ctx, &beastv1.GetRequest{Key: 6}); r.Found { + t.Fatal("batch applied despite failed condition") + } + + // Scan limit and resume. + for k := uint64(100); k < 110; k++ { + if _, err := client.Put(ctx, &beastv1.PutRequest{Key: k, Value: []byte("s")}); err != nil { + t.Fatal(err) + } + } + var got []uint64 + start := uint64(100) + for { + stream, err := client.Scan(ctx, &beastv1.ScanRequest{StartKey: start, EndKey: 109, Limit: 4}) + if err != nil { + t.Fatal(err) + } + n := 0 + for { + kv, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + got = append(got, kv.Key) + start = kv.Key + 1 + n++ + } + if n == 0 { + break + } + if n > 4 { + t.Fatalf("page returned %d > limit", n) + } + } + if len(got) != 10 { + t.Fatalf("paged scan returned %v", got) + } +} diff --git a/internal/api/engine_batch.go b/internal/api/engine_batch.go index 1b19571..a04bce4 100644 --- a/internal/api/engine_batch.go +++ b/internal/api/engine_batch.go @@ -32,6 +32,14 @@ type BatchOperation struct { // All operations are logged in a single WAL record with a single CRC32 checksum, ensuring // that all operations commit together or none do. func (e *Engine) BatchWrite(ops []BatchOperation) error { + return e.BatchWriteIf(ops, nil) +} + +// BatchWriteIf is BatchWrite gated by preconditions: every condition is evaluated +// under the same exclusive lock before any operation applies, so either all +// conditions hold and the whole batch commits, or ErrConditionFailed is returned +// and nothing is written. +func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error { if e.IsReadOnly() { return ErrReadOnlyReplica } @@ -49,6 +57,11 @@ func (e *Engine) BatchWrite(ops []BatchOperation) error { if e.failed != nil { return e.failed } + for _, c := range conds { + if err := e.checkConditionLocked(c.Key, c.Cond); err != nil { + return err + } + } // 1. Serialize all operations into a single atomic WAL batch payload walOps := make([]wal.BatchOp, len(ops)) diff --git a/internal/api/engine_conditional.go b/internal/api/engine_conditional.go new file mode 100644 index 0000000..12c432c --- /dev/null +++ b/internal/api/engine_conditional.go @@ -0,0 +1,106 @@ +package api + +import ( + "bytes" + "errors" + + "github.com/ChromaBeast/beastdb/internal/index" +) + +// ErrConditionFailed indicates a write precondition did not hold; nothing was written. +var ErrConditionFailed = errors.New("write precondition failed") + +// ConditionKind selects the check applied to a key before a conditional write. +type ConditionKind byte + +const ( + // CondNone applies no check. + CondNone ConditionKind = iota + // CondAbsent requires the key to not exist. + CondAbsent + // CondPresent requires the key to exist. + CondPresent + // CondValueEquals requires the key to exist with exactly the given value. + CondValueEquals +) + +// Condition is a precondition on the current state of a key. +type Condition struct { + Kind ConditionKind + Value []byte +} + +// KeyCondition binds a Condition to a key, for use with BatchWriteIf. +type KeyCondition struct { + Key uint64 + Cond Condition +} + +// checkConditionLocked evaluates cond against the current record. Caller holds e.mu. +func (e *Engine) checkConditionLocked(key uint64, cond Condition) error { + if cond.Kind == CondNone { + return nil + } + rid, err := e.tree.Find(key) + if err == index.ErrKeyNotFound { + if cond.Kind == CondAbsent { + return nil + } + return ErrConditionFailed + } + if err != nil { + return err + } + switch cond.Kind { + case CondAbsent: + return ErrConditionFailed + case CondPresent: + return nil + case CondValueEquals: + cur, err := e.ReadTuple(rid) + if err != nil { + return err + } + if !bytes.Equal(cur, cond.Value) { + return ErrConditionFailed + } + return nil + default: + return errors.New("engine: unknown condition kind") + } +} + +// PutIf writes value only if cond holds for key, atomically with the check. +func (e *Engine) PutIf(key uint64, value []byte, cond Condition) error { + if e.IsReadOnly() { + return ErrReadOnlyReplica + } + if len(value) > MaxRecordSize { + return ErrRecordTooLarge + } + e.mu.Lock() + defer e.mu.Unlock() + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.putLocked(key, value) +} + +// DeleteIf removes key only if cond holds, atomically with the check. +func (e *Engine) DeleteIf(key uint64, cond Condition) error { + if e.IsReadOnly() { + return ErrReadOnlyReplica + } + e.mu.Lock() + defer e.mu.Unlock() + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.deleteLocked(key) +} diff --git a/internal/api/grpc_server.go b/internal/api/grpc_server.go index 98f577c..9c1a9f1 100644 --- a/internal/api/grpc_server.go +++ b/internal/api/grpc_server.go @@ -69,7 +69,14 @@ func (s *GRPCServer) Put(ctx context.Context, req *beastv1.PutRequest) (*beastv1 if len(req.Value) > MaxRecordSize { return nil, status.Errorf(codes.InvalidArgument, "record value exceeds max 4000 bytes: %d bytes provided", len(req.Value)) } - if err := s.engine.Put(req.Key, req.Value); err != nil { + cond, err := conditionFromProto(req.GetCondition()) + if err != nil { + return nil, err + } + if err := s.engine.PutIf(req.Key, req.Value, cond); err != nil { + if errors.Is(err, ErrConditionFailed) { + return nil, status.Errorf(codes.Aborted, "precondition failed") + } if errors.Is(err, ErrReadOnlyReplica) { return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } @@ -86,7 +93,14 @@ func (s *GRPCServer) Delete(ctx context.Context, req *beastv1.DeleteRequest) (*b if s.engine.IsReadOnly() { return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } - err := s.engine.Delete(req.Key) + cond, err := conditionFromProto(req.GetCondition()) + if err != nil { + return nil, err + } + err = s.engine.DeleteIf(req.Key, cond) + if errors.Is(err, ErrConditionFailed) { + return nil, status.Errorf(codes.Aborted, "precondition failed") + } if errors.Is(err, ErrReadOnlyReplica) { return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } @@ -104,7 +118,8 @@ func (s *GRPCServer) Scan(req *beastv1.ScanRequest, stream grpc.ServerStreamingS } defer cursor.Close() - for { + var sent uint32 + for req.Limit == 0 || sent < req.Limit { if stream.Context().Err() != nil { return stream.Context().Err() } @@ -129,6 +144,7 @@ func (s *GRPCServer) Scan(req *beastv1.ScanRequest, stream grpc.ServerStreamingS if err := stream.Send(resp); err != nil { return err } + sent++ } return nil } @@ -162,7 +178,19 @@ func (s *GRPCServer) BatchWrite(ctx context.Context, req *beastv1.BatchWriteRequ return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } - if err := s.engine.BatchWrite(ops); err != nil { + conds := make([]KeyCondition, 0, len(req.Conditions)) + for _, kc := range req.Conditions { + cond, err := conditionFromProto(kc.GetCondition()) + if err != nil { + return nil, err + } + conds = append(conds, KeyCondition{Key: kc.GetKey(), Cond: cond}) + } + + if err := s.engine.BatchWriteIf(ops, conds); err != nil { + if errors.Is(err, ErrConditionFailed) { + return nil, status.Errorf(codes.Aborted, "precondition failed") + } if errors.Is(err, ErrReadOnlyReplica) { return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } @@ -177,3 +205,22 @@ func (s *GRPCServer) BatchWrite(ctx context.Context, req *beastv1.BatchWriteRequ AppliedCount: uint32(len(ops)), }, nil } + +// conditionFromProto converts a wire Condition (nil means no check) to an engine Condition. +func conditionFromProto(c *beastv1.Condition) (Condition, error) { + if c == nil { + return Condition{}, nil + } + switch c.Kind { + case beastv1.ConditionKind_CONDITION_KIND_NONE: + return Condition{}, nil + case beastv1.ConditionKind_CONDITION_KIND_ABSENT: + return Condition{Kind: CondAbsent}, nil + case beastv1.ConditionKind_CONDITION_KIND_PRESENT: + return Condition{Kind: CondPresent}, nil + case beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS: + return Condition{Kind: CondValueEquals, Value: c.Value}, nil + default: + return Condition{}, status.Errorf(codes.InvalidArgument, "unsupported condition kind: %v", c.Kind) + } +} From 069adff0a48523a4087b65ea025ad001fb9562fb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 16:37:09 +0000 Subject: [PATCH 03/16] feat(engine): values up to 2 MiB, page-slot reuse, and WAL frame cap Values were capped at 4,000 bytes, one slotted-page tuple. Larger values (up to MaxValueSize, 2 MiB) are now stored as a CRC-checked chain of chunk tuples headed by a manifest; the B+ tree flags the manifest RID with bit 15 of the slot ID. Small values take the existing path unchanged, existing files need no migration, and the WAL, recovery and replication still log whole values. Batches may carry up to 8 MiB of values. See docs/adr/0001-large-values.md. - Reads verify chunk magic, chain length and CRC (ErrCorruptValue); overwrite and delete free a chain only after it validates. - gRPC: MessageSizeServerOptions / MessageSizeDialOption raise the 4 MiB default; the server, followers and beastctl use them. Write-error status mapping is shared in grpc_status.go. - WAL: frames over 32 MiB are refused on write, and a torn header claiming a huge length now reads as a torn tail instead of allocating up to 4 GiB. - Storage: InsertTuple reuses tombstoned slots. Previously a recycled page's slot directory grew until it could not host a full tuple, and pages too full for one request were dropped from the reuse list; 300 overwrites of a 512 KB value grew the file to 2,120 pages, now 266, and 600 small overwrites added 264 pages, now 3. Tests: boundary sizes, overwrite/delete space reuse, crash and clean restart, CAS and batches on large values, scans, corruption detection, a mixed-size oracle across restarts, gRPC 6 MiB batches, and catch-up plus live replication of 2 MiB values. Full suite passes with -race. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017mWMfRGkKdRgKc5mtSsWEG --- ROADMAP.md | 2 +- cmd/beastctl/client.go | 2 + cmd/server/main.go | 4 +- docs/adr/0001-large-values.md | 39 +++++ docs/roadmap-plan.md | 9 +- docs/v1-semantics.md | 5 +- internal/api/engine.go | 2 +- internal/api/engine_batch.go | 6 +- internal/api/engine_conditional.go | 2 +- internal/api/engine_large.go | 196 +++++++++++++++++++++++++ internal/api/engine_large_test.go | 185 +++++++++++++++++++++++ internal/api/engine_ops.go | 36 ++--- internal/api/engine_space.go | 62 ++++++-- internal/api/grpc_server.go | 44 +----- internal/api/grpc_server_test.go | 51 +++++++ internal/api/grpc_status.go | 37 +++++ internal/api/limits_readonly_test.go | 14 +- internal/replication/large_rep_test.go | 84 +++++++++++ internal/storage/page.go | 36 +++-- internal/storage/page_test.go | 17 +++ internal/wal/record.go | 9 ++ internal/wal/wal.go | 4 + internal/wal/wal_test.go | 10 ++ 23 files changed, 750 insertions(+), 106 deletions(-) create mode 100644 docs/adr/0001-large-values.md create mode 100644 internal/api/engine_large.go create mode 100644 internal/api/engine_large_test.go create mode 100644 internal/api/grpc_status.go create mode 100644 internal/replication/large_rep_test.go diff --git a/ROADMAP.md b/ROADMAP.md index 0f4f488..15ddabb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -122,7 +122,7 @@ Order is dependency-based, not a calendar promise. Initial planning range for on ## Beyond V1 gates -See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite) and limit-based scan pagination. +See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite), limit-based scan pagination, and values up to 2 MiB with 8 MiB batches ([ADR 0001](docs/adr/0001-large-values.md)). ## References diff --git a/cmd/beastctl/client.go b/cmd/beastctl/client.go index 35e68e5..207f0fc 100644 --- a/cmd/beastctl/client.go +++ b/cmd/beastctl/client.go @@ -8,6 +8,7 @@ import ( "time" beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "github.com/ChromaBeast/beastdb/internal/api" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -49,6 +50,7 @@ func DialWithToken(addr, token string) (*Client, error) { opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock(), + api.MessageSizeDialOption(), } if token != "" { opts = append(opts, grpc.WithPerRPCCredentials(tokenAuth{token: token})) diff --git a/cmd/server/main.go b/cmd/server/main.go index 14d0c59..624c994 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -73,7 +73,7 @@ func main() { engine.StartAutoCheckpoint(ckptCtx, 10*time.Second, int64(*walCheckpointMB)<<20) tokenStore := initTokenStore(engine) - var grpcOpts []grpc.ServerOption + grpcOpts := api.MessageSizeServerOptions() if *tlsCert != "" && *tlsKey != "" { creds, err := loadServerTLS(*tlsCert, *tlsKey, *tlsCA) if err != nil { @@ -103,7 +103,7 @@ func main() { if err != nil { log.Fatalf("Failed to initialize follower client TLS: %v", err) } - conn, err := grpc.NewClient(*leaderAddr, grpc.WithTransportCredentials(clientCreds)) + conn, err := grpc.NewClient(*leaderAddr, grpc.WithTransportCredentials(clientCreds), api.MessageSizeDialOption()) if err != nil { log.Fatalf("Failed to dial leader node: %v", err) } diff --git a/docs/adr/0001-large-values.md b/docs/adr/0001-large-values.md new file mode 100644 index 0000000..658fbbb --- /dev/null +++ b/docs/adr/0001-large-values.md @@ -0,0 +1,39 @@ +# ADR 0001: Values larger than one page + +**Status:** accepted, 28 October 2026. + +## Context + +Every value lived in one slotted-page tuple, so values were capped at 4,000 bytes. Clients +storing documents, reports, serialized aggregates or index lists had to invent their own +chunking, each with its own atomicity and corruption bugs. + +## Decision + +- Values up to `MaxRecordSize` (4,000 bytes) are stored inline exactly as before. +- Values up to `MaxValueSize` (2 MiB) are stored as a chain of chunk tuples headed by a + 24-byte manifest tuple (magic, version, length, CRC32, chunk count, first chunk RID). Each + chunk carries a magic byte and its successor's RID; page 0 (the meta page) ends the chain. +- The B+ tree points at the manifest with bit 15 of the RID's slot ID set. A 4 KB page holds + at most ~1,000 slots, so inline RIDs never use that bit. +- The WAL still logs the whole value in one frame (`OpPut`) or one batch frame, so recovery + and replication are unchanged: replay re-chains the value. A batch's values are capped at + `MaxBatchBytes` (8 MiB) and a WAL frame at 32 MiB. +- Reads verify chunk magic, chain length and CRC and return `ErrCorruptValue` on mismatch. + Overwrite and delete free the chain only after it validates, so a damaged chain is leaked + rather than allowed to tombstone tuples owned by other keys. + +## Consequences + +- No migration: existing files contain no flagged RIDs. A file containing large values, + opened by an older binary, fails reads of those keys with `ErrSlotOutOfBounds` instead of + returning manifest bytes as data. Downgrade after writing large values is not supported. +- gRPC's default 4 MiB message limit is raised with `api.MessageSizeServerOptions` and + `api.MessageSizeDialOption`; the server, followers and `beastctl` use them, and clients + that send large batches must too. +- A large write holds the engine's write lock while it writes ~260 chunk tuples per MiB. +- Writing this exposed two allocator defects, fixed alongside: inserts never reused + tombstoned slot entries, so the slot directory of a recycled page grew until it could no + longer host a full tuple; and pages too full for one request were dropped from the reuse + list for good. 300 overwrites of a 512 KB value now keep the file at two copies' worth of + pages instead of growing it eightfold. diff --git a/docs/roadmap-plan.md b/docs/roadmap-plan.md index 97c0901..0b9f92f 100644 --- a/docs/roadmap-plan.md +++ b/docs/roadmap-plan.md @@ -41,10 +41,11 @@ Order is dependency-based. Each phase ends with a runnable demo, tests and docs. ### Phase B — Scan and key ergonomics -1. **Paginated scans:** `limit` on `ScanRequest` and resume by last key + 1 are **implemented**. A server-enforced maximum page size and reverse/prefix scans remain. -2. **Reverse and prefix scans** over the uint64 range, with documented bounds semantics. -3. **Optional wide-key mode design.** Options to evaluate and decide with a written ADR: (a) keep uint64 and ship a documented collision-safe pattern plus a client helper, or (b) add a variable-length key space. Decision gated on benchmarks and on-disk-format migration cost. -4. **Key-layout helper library** (client side, optional convention, not required by the engine). +1. **Large values:** values up to 2 MiB, chained across pages with CRC checks, and 8 MiB batches are **implemented** ([ADR 0001](adr/0001-large-values.md)). +2. **Paginated scans:** `limit` on `ScanRequest` and resume by last key + 1 are **implemented**. A server-enforced maximum page size and reverse/prefix scans remain. +3. **Reverse and prefix scans** over the uint64 range, with documented bounds semantics. +4. **Optional wide-key mode design.** Options to evaluate and decide with a written ADR: (a) keep uint64 and ship a documented collision-safe pattern plus a client helper, or (b) add a variable-length key space. Decision gated on benchmarks and on-disk-format migration cost. +5. **Key-layout helper library** (client side, optional convention, not required by the engine). **Exit:** a client can page through a 10M-key range with bounded memory on both sides. diff --git a/docs/v1-semantics.md b/docs/v1-semantics.md index 5924c0c..366356c 100644 --- a/docs/v1-semantics.md +++ b/docs/v1-semantics.md @@ -8,7 +8,8 @@ - **Key Format:** Unsigned 64-bit integer (`uint64`). - **Value Format:** Opaque byte slice (`[]byte`). -- **Maximum Record Size:** 4,000 bytes per record (bounded by 4KB slotted page capacity). Payloads exceeding this limit return `ErrRecordTooLarge`. +- **Maximum Value Size:** 2 MiB (`MaxValueSize`). Values up to 4,000 bytes (`MaxRecordSize`) are stored inline in one slotted-page tuple; larger ones are chained across pages and checked with a CRC32 on read ([ADR 0001](adr/0001-large-values.md)). Larger payloads return `ErrRecordTooLarge`. +- **Batch Size:** the values in one `BatchWrite` may total at most 8 MiB (`MaxBatchBytes`, `ErrBatchTooLarge`). gRPC clients sending large values must raise the 4 MiB default message limit (`api.MessageSizeDialOption`). - **Key Partitioning Convention:** Applications may optionally pack higher-order bits for table/collection prefixing (e.g., bits 56–63 for partition ID), but the engine treats keys as scalar `uint64` values. --- @@ -28,7 +29,7 @@ ### `BatchWrite(puts []PutOp, deletes []DeleteOp)` * **Durability Guarantee:** The entire batch is serialized into a single `OpBatch` WAL frame and synced in a single I/O operation. * **Atomicity:** All-or-nothing. On crash recovery, uncommitted or partial batches are rolled back; fully synced batches are replayed completely. -* **Limits:** Recommended batch limit of 1,000 operations or 2 MB aggregate payload. +* **Limits:** Values in one batch may total at most 8 MiB; each value at most 2 MiB. ### Conditional writes: `Put`/`Delete` with `Condition`, `BatchWrite` with `conditions` * **Conditions:** `ABSENT` (key must not exist), `PRESENT` (key must exist), `VALUE_EQUALS` (key must exist with exactly the given bytes). Omitted or `NONE` means unconditional. diff --git a/internal/api/engine.go b/internal/api/engine.go index 85cb95f..19c13dd 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -22,7 +22,7 @@ type Engine struct { secIndex *index.SecondaryIndex readOnly bool failed error // set by halt; blocks writes until restart - reclaim map[uint64]struct{} // data pages holding tombstoned tuples + reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree mu sync.RWMutex } diff --git a/internal/api/engine_batch.go b/internal/api/engine_batch.go index a04bce4..1fff44e 100644 --- a/internal/api/engine_batch.go +++ b/internal/api/engine_batch.go @@ -46,10 +46,8 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error if len(ops) == 0 { return ErrEmptyBatchWrite } - for _, op := range ops { - if op.Type == BatchOpPut && len(op.Value) > MaxRecordSize { - return ErrRecordTooLarge - } + if err := checkValueSizes(ops); err != nil { + return err } e.mu.Lock() diff --git a/internal/api/engine_conditional.go b/internal/api/engine_conditional.go index 12c432c..51e0445 100644 --- a/internal/api/engine_conditional.go +++ b/internal/api/engine_conditional.go @@ -75,7 +75,7 @@ func (e *Engine) PutIf(key uint64, value []byte, cond Condition) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - if len(value) > MaxRecordSize { + if len(value) > MaxValueSize { return ErrRecordTooLarge } e.mu.Lock() diff --git a/internal/api/engine_large.go b/internal/api/engine_large.go new file mode 100644 index 0000000..ff26145 --- /dev/null +++ b/internal/api/engine_large.go @@ -0,0 +1,196 @@ +package api + +import ( + "encoding/binary" + "errors" + "hash/crc32" + + "github.com/ChromaBeast/beastdb/internal/storage" +) + +// Values up to MaxRecordSize live inline in one slotted-page tuple. Larger +// values, up to MaxValueSize, are stored as a chain of chunk tuples headed by a +// small manifest tuple; the B+ tree points at the manifest with largeSlotFlag +// set in the RID's slot ID. A 4 KB page holds at most ~1,000 slots, so bit 15 +// is never used by an inline RID, and older binaries reading a flagged RID fail +// with ErrSlotOutOfBounds instead of returning the manifest as data. +const ( + // MaxValueSize is the largest value accepted by Put, PutIf and BatchWrite. + MaxValueSize = 2 << 20 + // MaxBatchBytes bounds the summed value bytes of one BatchWrite. + MaxBatchBytes = 8 << 20 + + largeSlotFlag = 0x8000 + chunkMagic = 0xBC + manifestMagic = 0xBD + manifestV1 = 1 + chunkHeader = 11 // magic(1) + next page(8) + next slot(2) + manifestSize = 24 // magic, version, length(4), crc32(4), chunks(4), first page(8), first slot(2) + chunkPayload = MaxRecordSize - chunkHeader + maxChunks = (MaxValueSize + chunkPayload - 1) / chunkPayload +) + +// ErrCorruptValue reports a large-value chain that fails structural or CRC checks. +var ErrCorruptValue = errors.New("engine: stored value chain is corrupt") + +func isLargeRID(rid storage.RID) bool { return rid.SlotID&largeSlotFlag != 0 } + +func manifestOf(rid storage.RID) storage.RID { + rid.SlotID &^= largeSlotFlag + return rid +} + +// insertValue stores value inline when it fits in one tuple, else as a chunk chain. +// Caller must hold e.mu exclusively. +func (e *Engine) insertValue(value []byte) (storage.RID, error) { + if len(value) <= MaxRecordSize { + return e.insertTuple(value) + } + n := (len(value) + chunkPayload - 1) / chunkPayload + written := make([]storage.RID, 0, n+1) + undo := func() { + for _, r := range written { + e.freeTuple(r) + } + } + buf := make([]byte, MaxRecordSize) + var next storage.RID // zero page ID ends the chain; page 0 is the meta page + // Write the tail first so every chunk can name its successor. + for i := n - 1; i >= 0; i-- { + start := i * chunkPayload + end := min(start+chunkPayload, len(value)) + t := buf[:chunkHeader+end-start] + t[0] = chunkMagic + binary.LittleEndian.PutUint64(t[1:9], next.PageID) + binary.LittleEndian.PutUint16(t[9:11], next.SlotID) + copy(t[chunkHeader:], value[start:end]) + rid, err := e.insertTuple(t) + if err != nil { + undo() + return storage.RID{}, err + } + written = append(written, rid) + next = rid + } + m := buf[:manifestSize] + m[0], m[1] = manifestMagic, manifestV1 + binary.LittleEndian.PutUint32(m[2:6], uint32(len(value))) + binary.LittleEndian.PutUint32(m[6:10], crc32.ChecksumIEEE(value)) + binary.LittleEndian.PutUint32(m[10:14], uint32(n)) + binary.LittleEndian.PutUint64(m[14:22], next.PageID) + binary.LittleEndian.PutUint16(m[22:24], next.SlotID) + rid, err := e.insertTuple(m) + if err != nil { + undo() + return storage.RID{}, err + } + rid.SlotID |= largeSlotFlag + return rid, nil +} + +type manifest struct { + length, crc, chunks uint32 + first storage.RID +} + +func (e *Engine) readManifest(rid storage.RID) (manifest, error) { + m, err := e.readTuple(manifestOf(rid)) + if err != nil { + return manifest{}, err + } + if len(m) != manifestSize || m[0] != manifestMagic || m[1] != manifestV1 { + return manifest{}, ErrCorruptValue + } + out := manifest{ + length: binary.LittleEndian.Uint32(m[2:6]), + crc: binary.LittleEndian.Uint32(m[6:10]), + chunks: binary.LittleEndian.Uint32(m[10:14]), + first: storage.RID{PageID: binary.LittleEndian.Uint64(m[14:22]), SlotID: binary.LittleEndian.Uint16(m[22:24])}, + } + if out.length > MaxValueSize || out.chunks == 0 || out.chunks > maxChunks { + return manifest{}, ErrCorruptValue + } + return out, nil +} + +// walkChain visits each chunk of a large value in order and verifies the chain +// ends exactly where the manifest says. visit receives the chunk's payload. +func (e *Engine) walkChain(m manifest, visit func(rid storage.RID, payload []byte)) error { + next := m.first + for i := uint32(0); i < m.chunks; i++ { + if next.PageID == 0 || isLargeRID(next) { + return ErrCorruptValue + } + t, err := e.readTuple(next) + if err != nil { + return err + } + if len(t) < chunkHeader || t[0] != chunkMagic { + return ErrCorruptValue + } + visit(next, t[chunkHeader:]) + next = storage.RID{PageID: binary.LittleEndian.Uint64(t[1:9]), SlotID: binary.LittleEndian.Uint16(t[9:11])} + } + if next.PageID != 0 { + return ErrCorruptValue + } + return nil +} + +// readValue returns a copy of the value at rid, reassembling and checking large values. +func (e *Engine) readValue(rid storage.RID) ([]byte, error) { + if !isLargeRID(rid) { + return e.readTuple(rid) + } + m, err := e.readManifest(rid) + if err != nil { + return nil, err + } + out := make([]byte, 0, m.length) + if err := e.walkChain(m, func(_ storage.RID, p []byte) { out = append(out, p...) }); err != nil { + return nil, err + } + if uint32(len(out)) != m.length || crc32.ChecksumIEEE(out) != m.crc { + return nil, ErrCorruptValue + } + return out, nil +} + +// freeValue releases the tuples behind rid. A chain that fails validation is +// left in place rather than risk tombstoning tuples that belong to other keys. +func (e *Engine) freeValue(rid storage.RID) { + if !isLargeRID(rid) { + e.freeTuple(rid) + return + } + m, err := e.readManifest(rid) + if err != nil { + return + } + var chunks []storage.RID + if e.walkChain(m, func(r storage.RID, _ []byte) { chunks = append(chunks, r) }) != nil { + return + } + for _, r := range chunks { + e.freeTuple(r) + } + e.freeTuple(manifestOf(rid)) +} + +// checkValueSizes enforces per-value and per-batch limits before anything is logged. +func checkValueSizes(ops []BatchOperation) error { + total := 0 + for _, op := range ops { + if op.Type != BatchOpPut { + continue + } + if len(op.Value) > MaxValueSize { + return ErrRecordTooLarge + } + total += len(op.Value) + } + if total > MaxBatchBytes { + return ErrBatchTooLarge + } + return nil +} diff --git a/internal/api/engine_large_test.go b/internal/api/engine_large_test.go new file mode 100644 index 0000000..ed37cf6 --- /dev/null +++ b/internal/api/engine_large_test.go @@ -0,0 +1,185 @@ +package api + +import ( + "bytes" + "errors" + "math/rand" + "path/filepath" + "testing" +) + +func openLarge(t *testing.T, dir string) *Engine { + t.Helper() + e, err := NewEngine(filepath.Join(dir, "large.bin"), filepath.Join(dir, "large.wal"), 64) + if err != nil { + t.Fatalf("open engine: %v", err) + } + return e +} + +func pattern(n int, seed byte) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(i*31) ^ seed + } + return b +} + +func mustGet(t *testing.T, e *Engine, key uint64, want []byte) { + t.Helper() + got, found, err := e.Get(key) + if err != nil || !found { + t.Fatalf("get %d: found=%v err=%v", key, found, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("get %d: %d bytes back, want %d", key, len(got), len(want)) + } +} + +func TestLargeValueRoundTripAtBoundaries(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + sizes := []int{0, 1, MaxRecordSize, MaxRecordSize + 1, chunkPayload * 3, chunkPayload*3 + 1, 1 << 20, MaxValueSize} + for i, n := range sizes { + if err := e.Put(uint64(i+1), pattern(n, byte(i))); err != nil { + t.Fatalf("put %d bytes: %v", n, err) + } + } + for i, n := range sizes { + mustGet(t, e, uint64(i+1), pattern(n, byte(i))) + } + if err := e.Put(99, make([]byte, MaxValueSize+1)); !errors.Is(err, ErrRecordTooLarge) { + t.Fatalf("expected ErrRecordTooLarge, got %v", err) + } +} + +func TestLargeValueOverwriteAndDeleteReuseSpace(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + val := pattern(512<<10, 7) + for i := 0; i < 5; i++ { + if err := e.Put(1, val); err != nil { + t.Fatal(err) + } + } + afterWarmup := e.disk.NumPages() + for i := 0; i < 40; i++ { + _ = e.Put(1, pattern(512<<10, byte(i))) + } + if grown := e.disk.NumPages() - afterWarmup; grown > afterWarmup/2 { + t.Fatalf("overwrites leaked pages: %d pages after warm-up, %d more after 40 overwrites", afterWarmup, grown) + } + if err := e.Put(1, []byte("small again")); err != nil { + t.Fatal(err) + } + mustGet(t, e, 1, []byte("small again")) + if err := e.Delete(1); err != nil { + t.Fatal(err) + } + if _, found, _ := e.Get(1); found { + t.Fatal("deleted large value still readable") + } +} + +func TestLargeValueSurvivesCrashAndCleanRestart(t *testing.T) { + dir := t.TempDir() + e := openLarge(t, dir) + _ = e.Put(1, pattern(300<<10, 1)) + _ = e.Put(2, pattern(10, 2)) + _ = e.BatchWrite([]BatchOperation{{Type: BatchOpPut, Key: 3, Value: pattern(90<<10, 3)}, {Type: BatchOpDelete, Key: 2}}) + // Abrupt stop: WAL durable, pages never flushed. + _ = e.wal.Sync() + _ = e.wal.Close() + _ = e.disk.Close() + + e = openLarge(t, dir) + mustGet(t, e, 1, pattern(300<<10, 1)) + mustGet(t, e, 3, pattern(90<<10, 3)) + if _, found, _ := e.Get(2); found { + t.Fatal("batch delete lost in recovery") + } + if err := e.Close(); err != nil { + t.Fatal(err) + } + e = openLarge(t, dir) + defer e.Close() + mustGet(t, e, 1, pattern(300<<10, 1)) +} + +func TestLargeValueConditionsBatchesAndScans(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + v1, v2 := pattern(50<<10, 1), pattern(60<<10, 2) + if err := e.PutIf(1, v1, Condition{Kind: CondAbsent}); err != nil { + t.Fatal(err) + } + if err := e.PutIf(1, v2, Condition{Kind: CondValueEquals, Value: v2}); !errors.Is(err, ErrConditionFailed) { + t.Fatalf("stale compare-and-swap must fail, got %v", err) + } + if err := e.PutIf(1, v2, Condition{Kind: CondValueEquals, Value: v1}); err != nil { + t.Fatalf("compare-and-swap on a large value: %v", err) + } + big := []BatchOperation{{Type: BatchOpPut, Key: 2, Value: make([]byte, MaxValueSize)}, {Type: BatchOpPut, Key: 3, Value: make([]byte, MaxValueSize)}, + {Type: BatchOpPut, Key: 4, Value: make([]byte, MaxValueSize)}, {Type: BatchOpPut, Key: 5, Value: make([]byte, MaxValueSize)}, + {Type: BatchOpPut, Key: 6, Value: []byte("x")}} + if err := e.BatchWrite(big); !errors.Is(err, ErrBatchTooLarge) { + t.Fatalf("expected ErrBatchTooLarge, got %v", err) + } + cur, err := e.Scan(0, 10) + if err != nil { + t.Fatal(err) + } + _, rid, ok, err := cur.Next() + if err != nil || !ok { + t.Fatalf("scan found nothing: %v", err) + } + got, err := e.ReadTuple(rid) + _ = cur.Close() + if err != nil || !bytes.Equal(got, v2) { + t.Fatalf("scan must return the whole large value: %d bytes, %v", len(got), err) + } +} + +func TestCorruptChunkIsDetected(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + _ = e.Put(1, pattern(20<<10, 9)) + rid, _ := e.tree.Find(1) + m, _ := e.readManifest(rid) + page, _ := e.bpm.FetchPage(m.first.PageID) + tuple, _ := page.GetTuple(m.first.SlotID) + tuple[chunkHeader+5] ^= 0xFF + _ = e.bpm.UnpinPage(m.first.PageID, true) + if _, _, err := e.Get(1); !errors.Is(err, ErrCorruptValue) { + t.Fatalf("expected ErrCorruptValue, got %v", err) + } +} + +func TestMixedSizeOracleAcrossRestarts(t *testing.T) { + dir := t.TempDir() + e := openLarge(t, dir) + oracle := map[uint64][]byte{} + rng := rand.New(rand.NewSource(42)) + for round := 0; round < 4; round++ { + for op := 0; op < 80; op++ { + key := uint64(rng.Intn(30) + 1) + if rng.Intn(4) == 0 { + _ = e.Delete(key) + delete(oracle, key) + continue + } + sizes := []int{rng.Intn(200), rng.Intn(MaxRecordSize), MaxRecordSize + rng.Intn(20000), rng.Intn(200 << 10)} + val := pattern(sizes[rng.Intn(len(sizes))], byte(op)) + if err := e.Put(key, val); err != nil { + t.Fatalf("put: %v", err) + } + oracle[key] = val + } + _ = e.Close() + e = openLarge(t, dir) + for k, v := range oracle { + mustGet(t, e, k, v) + } + } + _ = e.Close() +} diff --git a/internal/api/engine_ops.go b/internal/api/engine_ops.go index 49c9e96..c637262 100644 --- a/internal/api/engine_ops.go +++ b/internal/api/engine_ops.go @@ -9,13 +9,17 @@ import ( "github.com/ChromaBeast/beastdb/internal/wal" ) +// MaxRecordSize is the largest value stored inline in a single slotted-page +// tuple; larger values up to MaxValueSize are chained (see engine_large.go). const MaxRecordSize = 4000 var ( // ErrRecordExists indicates a key is already present during PutIfAbsent. ErrRecordExists = errors.New("record already exists") - // ErrRecordTooLarge indicates the payload exceeds the 4,000 byte limit. - ErrRecordTooLarge = errors.New("record value exceeds maximum supported size of 4000 bytes") + // ErrRecordTooLarge indicates a value exceeds MaxValueSize (2 MiB). + ErrRecordTooLarge = errors.New("record value exceeds maximum supported size of 2 MiB") + // ErrBatchTooLarge indicates a batch's values sum to more than MaxBatchBytes (8 MiB). + ErrBatchTooLarge = errors.New("batch values exceed maximum supported total of 8 MiB") // ErrReadOnlyReplica indicates mutation attempts on a read-only replica node. ErrReadOnlyReplica = errors.New("writes are not permitted on read-only replica") ) @@ -25,7 +29,7 @@ func (e *Engine) Put(key uint64, value []byte) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - if len(value) > MaxRecordSize { + if len(value) > MaxValueSize { return ErrRecordTooLarge } e.mu.Lock() @@ -38,7 +42,7 @@ func (e *Engine) PutIfAbsent(key uint64, value []byte) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - if len(value) > MaxRecordSize { + if len(value) > MaxValueSize { return ErrRecordTooLarge } e.mu.Lock() @@ -85,23 +89,13 @@ func (e *Engine) Get(key uint64) ([]byte, bool, error) { return nil, false, err } - page, err := e.bpm.FetchPage(rid.PageID) - if err != nil { - return nil, false, err - } - - tuple, err := page.GetTuple(rid.SlotID) - _ = e.bpm.UnpinPage(rid.PageID, false) + val, err := e.readValue(rid) if err != nil { return nil, false, err } - - val := make([]byte, len(tuple)) - copy(val, tuple) return val, true, nil } - // Scan returns a streaming cursor iterator over the requested key range. // The engine's read lock is held for the cursor's entire lifetime and released // in cursor.Close() — this prevents concurrent writes from splitting a leaf @@ -116,19 +110,25 @@ func (e *Engine) Scan(startKey, endKey uint64) (*index.Cursor, error) { return cursor, nil } -// ReadTuple retrieves tuple bytes directly for a given RID. +// ReadTuple returns a copy of the value a B+ tree RID refers to, reassembling +// chained large values. func (e *Engine) ReadTuple(rid storage.RID) ([]byte, error) { + return e.readValue(rid) +} + +// readTuple copies the raw bytes of one slotted-page tuple. +func (e *Engine) readTuple(rid storage.RID) ([]byte, error) { page, err := e.bpm.FetchPage(rid.PageID) if err != nil { return nil, err } tuple, err := page.GetTuple(rid.SlotID) - _ = e.bpm.UnpinPage(rid.PageID, false) if err != nil { + _ = e.bpm.UnpinPage(rid.PageID, false) return nil, err } - val := make([]byte, len(tuple)) copy(val, tuple) + _ = e.bpm.UnpinPage(rid.PageID, false) return val, nil } diff --git a/internal/api/engine_space.go b/internal/api/engine_space.go index e0d7bd1..b2ca3f7 100644 --- a/internal/api/engine_space.go +++ b/internal/api/engine_space.go @@ -34,10 +34,12 @@ func (e *Engine) insertTuple(value []byte) (storage.RID, error) { _ = e.bpm.UnpinPage(e.activeDataPage, true) return storage.RID{PageID: e.activeDataPage, SlotID: slotID}, nil } + leftover := page.FreeSpace() _ = e.bpm.UnpinPage(e.activeDataPage, true) if err != storage.ErrPageFull { return storage.RID{}, err } + oldActive := e.activeDataPage newPage, newID, err := e.nextDataPage(len(value)) if err != nil { @@ -48,6 +50,9 @@ func (e *Engine) insertTuple(value []byte) (storage.RID, error) { return storage.RID{}, err } e.activeDataPage = newID + if leftover >= minTrackedFree { + e.trackFree(oldActive, leftover) + } slotID, err = newPage.InsertTuple(value) _ = e.bpm.UnpinPage(newID, err == nil) if err != nil { @@ -56,32 +61,60 @@ func (e *Engine) insertTuple(value []byte) (storage.RID, error) { return storage.RID{PageID: newID, SlotID: slotID}, nil } -// nextDataPage returns a pinned page with room for size bytes: a compacted page -// with dead tuples if one fits, otherwise a freshly allocated page. +const ( + // unknownFree marks a page whose tuples were tombstoned since it was last measured. + unknownFree = -1 + // minTrackedFree drops pages too full to be worth revisiting. + minTrackedFree = 64 +) + +// nextDataPage returns a pinned page with room for size bytes: a known page +// with enough free space if one exists, otherwise a freshly allocated page. +// Pages that are too full for this request stay tracked for smaller ones. func (e *Engine) nextDataPage(size int) (*storage.SlottedPage, uint64, error) { - for id := range e.reclaim { - delete(e.reclaim, id) + need := size + storage.SlotSize + for id, free := range e.reclaim { if id == e.activeDataPage { + delete(e.reclaim, id) + continue + } + if free != unknownFree && free < need { continue } p, err := e.bpm.FetchPage(id) if err != nil { + delete(e.reclaim, id) continue } p.Compact() - if p.FreeSpace() >= size+storage.SlotSize { - _ = e.bpm.UnpinPage(id, true) - p, err = e.bpm.FetchPage(id) - if err == nil { + got := p.FreeSpace() + _ = e.bpm.UnpinPage(id, true) + if got >= need { + delete(e.reclaim, id) + if p, err = e.bpm.FetchPage(id); err == nil { return p, id, nil } continue } - _ = e.bpm.UnpinPage(id, true) + if got < minTrackedFree { + delete(e.reclaim, id) + } else { + e.reclaim[id] = got + } } return e.bpm.NewPage() } +// trackFree remembers a page's free space for later inserts. +func (e *Engine) trackFree(id uint64, free int) { + if e.reclaim == nil { + e.reclaim = make(map[uint64]int) + } + if cur, ok := e.reclaim[id]; free == unknownFree || !ok || cur != unknownFree { + e.reclaim[id] = free + } +} + // freeTuple tombstones the tuple at rid and queues its page for space reuse. func (e *Engine) freeTuple(rid storage.RID) { page, err := e.bpm.FetchPage(rid.PageID) @@ -91,10 +124,7 @@ func (e *Engine) freeTuple(rid storage.RID) { _ = page.DeleteTuple(rid.SlotID) _ = e.bpm.UnpinPage(rid.PageID, true) if rid.PageID != e.activeDataPage { - if e.reclaim == nil { - e.reclaim = make(map[uint64]struct{}) - } - e.reclaim[rid.PageID] = struct{}{} + e.trackFree(rid.PageID, unknownFree) } } @@ -106,7 +136,7 @@ func (e *Engine) applyPutInMemory(key uint64, value []byte) error { if err != nil && err != index.ErrKeyNotFound { return err } - rid, err := e.insertTuple(value) + rid, err := e.insertValue(value) if err != nil { return err } @@ -114,7 +144,7 @@ func (e *Engine) applyPutInMemory(key uint64, value []byte) error { return err } if hadOld { - e.freeTuple(oldRID) + e.freeValue(oldRID) } return nil } @@ -127,6 +157,6 @@ func (e *Engine) applyDeleteInMemory(key uint64) error { if err != nil { return err } - e.freeTuple(rid) + e.freeValue(rid) return e.tree.Delete(key) } diff --git a/internal/api/grpc_server.go b/internal/api/grpc_server.go index 9c1a9f1..1fac303 100644 --- a/internal/api/grpc_server.go +++ b/internal/api/grpc_server.go @@ -2,7 +2,6 @@ package api import ( "context" - "errors" "net" beastv1 "github.com/ChromaBeast/beastdb/api/proto" @@ -66,24 +65,15 @@ func (s *GRPCServer) Put(ctx context.Context, req *beastv1.PutRequest) (*beastv1 if s.engine.IsReadOnly() { return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") } - if len(req.Value) > MaxRecordSize { - return nil, status.Errorf(codes.InvalidArgument, "record value exceeds max 4000 bytes: %d bytes provided", len(req.Value)) + if len(req.Value) > MaxValueSize { + return nil, status.Errorf(codes.InvalidArgument, "record value exceeds max %d bytes: %d bytes provided", MaxValueSize, len(req.Value)) } cond, err := conditionFromProto(req.GetCondition()) if err != nil { return nil, err } if err := s.engine.PutIf(req.Key, req.Value, cond); err != nil { - if errors.Is(err, ErrConditionFailed) { - return nil, status.Errorf(codes.Aborted, "precondition failed") - } - if errors.Is(err, ErrReadOnlyReplica) { - return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") - } - if errors.Is(err, ErrRecordTooLarge) { - return nil, status.Errorf(codes.InvalidArgument, "record value exceeds max 4000 bytes") - } - return nil, status.Errorf(codes.Internal, "put error: %v", err) + return nil, writeStatus(err, "put") } return &beastv1.PutResponse{Success: true}, nil } @@ -97,15 +87,8 @@ func (s *GRPCServer) Delete(ctx context.Context, req *beastv1.DeleteRequest) (*b if err != nil { return nil, err } - err = s.engine.DeleteIf(req.Key, cond) - if errors.Is(err, ErrConditionFailed) { - return nil, status.Errorf(codes.Aborted, "precondition failed") - } - if errors.Is(err, ErrReadOnlyReplica) { - return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") - } - if err != nil { - return nil, status.Errorf(codes.Internal, "delete error: %v", err) + if err := s.engine.DeleteIf(req.Key, cond); err != nil { + return nil, writeStatus(err, "delete") } return &beastv1.DeleteResponse{Success: true}, nil } @@ -167,11 +150,7 @@ func (s *GRPCServer) BatchWrite(ctx context.Context, req *beastv1.BatchWriteRequ return nil, status.Errorf(codes.InvalidArgument, "unsupported operation type: %v", op.OpType) } - ops[i] = BatchOperation{ - Type: opType, - Key: op.Key, - Value: op.Value, - } + ops[i] = BatchOperation{Type: opType, Key: op.Key, Value: op.Value} } if s.engine.IsReadOnly() { @@ -188,16 +167,7 @@ func (s *GRPCServer) BatchWrite(ctx context.Context, req *beastv1.BatchWriteRequ } if err := s.engine.BatchWriteIf(ops, conds); err != nil { - if errors.Is(err, ErrConditionFailed) { - return nil, status.Errorf(codes.Aborted, "precondition failed") - } - if errors.Is(err, ErrReadOnlyReplica) { - return nil, status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") - } - if errors.Is(err, ErrRecordTooLarge) { - return nil, status.Errorf(codes.InvalidArgument, "batch contains record exceeding max 4000 bytes: %v", err) - } - return nil, status.Errorf(codes.Internal, "batch write failed: %v", err) + return nil, writeStatus(err, "batch write") } return &beastv1.BatchWriteResponse{ diff --git a/internal/api/grpc_server_test.go b/internal/api/grpc_server_test.go index c7f743e..a999c39 100644 --- a/internal/api/grpc_server_test.go +++ b/internal/api/grpc_server_test.go @@ -131,3 +131,54 @@ func BenchmarkGRPCGet(b *testing.B) { _, _ = client.Get(ctx, req) } } + +// Values above gRPC's 4 MiB default, batched, must round-trip with the message-size options. +func TestGRPCLargeValuesWithMessageSizeOptions(t *testing.T) { + dir := t.TempDir() + engine, err := NewEngine(filepath.Join(dir, "big.bin"), filepath.Join(dir, "big.wal"), 50) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + server := NewGRPCServer(engine, MessageSizeServerOptions()...) + lis, _ := net.Listen("tcp", "127.0.0.1:0") + go func() { _ = server.Serve(lis) }() + defer server.Stop() + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), MessageSizeDialOption()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + client := beastv1.NewBeastDBServiceClient(conn) + ctx := context.Background() + + var ops []*beastv1.BatchOperation + for k := uint64(1); k <= 3; k++ { + ops = append(ops, &beastv1.BatchOperation{OpType: beastv1.BatchOpType_BATCH_OP_TYPE_PUT, Key: k, Value: bytes.Repeat([]byte{byte(k)}, MaxValueSize)}) + } + if _, err := client.BatchWrite(ctx, &beastv1.BatchWriteRequest{Operations: ops}); err != nil { + t.Fatalf("6 MiB batch: %v", err) + } + got, err := client.Get(ctx, &beastv1.GetRequest{Key: 2}) + if err != nil || !bytes.Equal(got.Value, ops[1].Value) { + t.Fatalf("get large value: %v", err) + } + stream, err := client.Scan(ctx, &beastv1.ScanRequest{StartKey: 1, EndKey: 3}) + if err != nil { + t.Fatal(err) + } + n := 0 + for { + kv, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil || len(kv.Value) != MaxValueSize { + t.Fatalf("scan large value: %v", err) + } + n++ + } + if n != 3 { + t.Fatalf("scan returned %d values", n) + } +} diff --git a/internal/api/grpc_status.go b/internal/api/grpc_status.go new file mode 100644 index 0000000..d235958 --- /dev/null +++ b/internal/api/grpc_status.go @@ -0,0 +1,37 @@ +package api + +import ( + "errors" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// MaxGRPCMessageSize is the gRPC message limit servers and clients should use +// so a maximal batch (MaxBatchBytes of values plus keys and framing) fits. +const MaxGRPCMessageSize = MaxBatchBytes + 1<<20 + +// MessageSizeServerOptions raises gRPC's 4 MiB default to MaxGRPCMessageSize. +func MessageSizeServerOptions() []grpc.ServerOption { + return []grpc.ServerOption{grpc.MaxRecvMsgSize(MaxGRPCMessageSize), grpc.MaxSendMsgSize(MaxGRPCMessageSize)} +} + +// MessageSizeDialOption is the client-side counterpart of MessageSizeServerOptions. +func MessageSizeDialOption() grpc.DialOption { + return grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxGRPCMessageSize), grpc.MaxCallSendMsgSize(MaxGRPCMessageSize)) +} + +// writeStatus maps engine write errors to gRPC status codes. +func writeStatus(err error, op string) error { + switch { + case errors.Is(err, ErrConditionFailed): + return status.Errorf(codes.Aborted, "precondition failed") + case errors.Is(err, ErrReadOnlyReplica): + return status.Errorf(codes.FailedPrecondition, "writes are not permitted on read replica") + case errors.Is(err, ErrRecordTooLarge), errors.Is(err, ErrBatchTooLarge): + return status.Errorf(codes.InvalidArgument, "%s: %v", op, err) + default: + return status.Errorf(codes.Internal, "%s error: %v", op, err) + } +} diff --git a/internal/api/limits_readonly_test.go b/internal/api/limits_readonly_test.go index d34d31d..14bf9cd 100644 --- a/internal/api/limits_readonly_test.go +++ b/internal/api/limits_readonly_test.go @@ -26,17 +26,17 @@ func TestEngineRecordSizeLimit(t *testing.T) { } defer engine.Close() - // Exactly 4000 bytes should succeed - validPayload := bytes.Repeat([]byte("A"), 4000) + // Exactly MaxValueSize bytes should succeed (chained beyond MaxRecordSize) + validPayload := bytes.Repeat([]byte("A"), MaxValueSize) if err := engine.Put(1, validPayload); err != nil { - t.Fatalf("expected 4000 byte put to succeed, got: %v", err) + t.Fatalf("expected MaxValueSize put to succeed, got: %v", err) } - // 4001 bytes must fail with ErrRecordTooLarge - tooLargePayload := bytes.Repeat([]byte("B"), 4001) + // One byte more must fail with ErrRecordTooLarge + tooLargePayload := bytes.Repeat([]byte("B"), MaxValueSize+1) err = engine.Put(2, tooLargePayload) if !errors.Is(err, ErrRecordTooLarge) { - t.Fatalf("expected ErrRecordTooLarge for 4001 bytes, got: %v", err) + t.Fatalf("expected ErrRecordTooLarge for MaxValueSize+1 bytes, got: %v", err) } // BatchWrite with oversized record must also fail @@ -134,7 +134,7 @@ func TestGRPCReadOnlyAndSizeValidation(t *testing.T) { // Disable read-only to test payload size rejection engine.SetReadOnly(false) - _, err = client.Put(ctx, &beastv1.PutRequest{Key: 1, Value: bytes.Repeat([]byte("X"), 4001)}) + _, err = client.Put(ctx, &beastv1.PutRequest{Key: 1, Value: bytes.Repeat([]byte("X"), MaxValueSize+1)}) if status.Code(err) != codes.InvalidArgument { t.Fatalf("expected InvalidArgument on oversized Put, got: %v", err) } diff --git a/internal/replication/large_rep_test.go b/internal/replication/large_rep_test.go new file mode 100644 index 0000000..cbd06b2 --- /dev/null +++ b/internal/replication/large_rep_test.go @@ -0,0 +1,84 @@ +package replication + +import ( + "bytes" + "context" + "net" + "path/filepath" + "testing" + "time" + + "github.com/ChromaBeast/beastdb/internal/api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func filled(n int, b byte) []byte { return bytes.Repeat([]byte{b}, n) } + +// Large values and multi-megabyte batches must replicate in catch-up and live modes. +func TestLargeValuesReplicate(t *testing.T) { + dir := t.TempDir() + leader, err := api.NewEngine(filepath.Join(dir, "l.bin"), filepath.Join(dir, "l.wal"), 50) + if err != nil { + t.Fatal(err) + } + defer leader.Close() + rep := NewLeaderServer(leader.WALPath()) + srv := grpc.NewServer(api.MessageSizeServerOptions()...) + rep.RegisterService(srv) + leader.SetCommitObserver(rep) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + catchUp := filled(1500<<10, 'c') + if err := leader.Put(1, catchUp); err != nil { + t.Fatal(err) + } + + follower, err := api.NewEngine(filepath.Join(dir, "f.bin"), filepath.Join(dir, "f.wal"), 50) + if err != nil { + t.Fatal(err) + } + defer follower.Close() + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), api.MessageSizeDialOption()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = NewFollowerClient("big", follower, conn).SyncLoop(ctx) }() + + live := filled(api.MaxValueSize, 'l') + batch := []api.BatchOperation{ + {Type: api.BatchOpPut, Key: 3, Value: filled(api.MaxValueSize, 'x')}, + {Type: api.BatchOpPut, Key: 4, Value: filled(api.MaxValueSize, 'y')}, + {Type: api.BatchOpPut, Key: 5, Value: filled(api.MaxValueSize, 'z')}, + } + time.Sleep(100 * time.Millisecond) + if err := leader.Put(2, live); err != nil { + t.Fatal(err) + } + if err := leader.BatchWrite(batch); err != nil { + t.Fatal(err) + } + + want := map[uint64][]byte{1: catchUp, 2: live, 3: batch[0].Value, 4: batch[1].Value, 5: batch[2].Value} + deadline := time.Now().Add(5 * time.Second) + for k, v := range want { + for { + got, found, err := follower.Get(k) + if err == nil && found && bytes.Equal(got, v) { + break + } + if time.Now().After(deadline) { + t.Fatalf("follower missing key %d: found=%v len=%d err=%v", k, found, len(got), err) + } + time.Sleep(20 * time.Millisecond) + } + } +} diff --git a/internal/storage/page.go b/internal/storage/page.go index a42953f..e375f6d 100644 --- a/internal/storage/page.go +++ b/internal/storage/page.go @@ -59,31 +59,41 @@ func (p *SlottedPage) FreeSpace() int { } // InsertTuple stores variable-length data, growing data upwards and slot downwards. +// A tombstoned slot is reused before the directory grows, so a page that is +// filled and emptied repeatedly keeps its full capacity. func (p *SlottedPage) InsertTuple(tuple []byte) (uint16, error) { tupleLen := len(tuple) - needed := tupleLen + SlotSize - + h := p.Header() + slotID, reuse := p.freeSlot(h) + needed := tupleLen + if !reuse { + needed += SlotSize + } if p.FreeSpace() < needed { return 0, ErrPageFull } - h := p.Header() h.Upper -= uint16(tupleLen) copy(p.data[h.Upper:h.Upper+uint16(tupleLen)], tuple) - - slotID := h.SlotCount - WriteSlot(p.data, slotID, Slot{ - Offset: h.Upper, - Length: uint16(tupleLen), - }) - - h.SlotCount++ - h.Lower += SlotSize + WriteSlot(p.data, slotID, Slot{Offset: h.Upper, Length: uint16(tupleLen)}) + if !reuse { + h.SlotCount++ + h.Lower += SlotSize + } WritePageHeader(p.data, h) - return slotID, nil } +// freeSlot returns the first tombstoned slot, or the next new slot ID. +func (p *SlottedPage) freeSlot(h PageHeader) (uint16, bool) { + for i := uint16(0); i < h.SlotCount; i++ { + if ReadSlot(p.data, i).Offset == 0 { + return i, true + } + } + return h.SlotCount, false +} + // GetTuple retrieves the tuple bytes referenced by slotID. func (p *SlottedPage) GetTuple(slotID uint16) ([]byte, error) { h := p.Header() diff --git a/internal/storage/page_test.go b/internal/storage/page_test.go index faf3066..3e6b824 100644 --- a/internal/storage/page_test.go +++ b/internal/storage/page_test.go @@ -115,3 +115,20 @@ func BenchmarkSlottedPageGet(b *testing.B) { _, _ = page.GetTuple(slotID) } } + +// A page filled and emptied repeatedly must keep its capacity: tombstoned slots are reused. +func TestInsertReusesTombstonedSlots(t *testing.T) { + p := NewSlottedPage(1) + tuple := make([]byte, 4000) + for i := 0; i < 50; i++ { + slot, err := p.InsertTuple(tuple) + if err != nil { + t.Fatalf("round %d: page lost capacity: %v", i, err) + } + _ = p.DeleteTuple(slot) + p.Compact() + } + if h := p.Header(); h.SlotCount != 1 { + t.Fatalf("slot directory grew to %d entries", h.SlotCount) + } +} diff --git a/internal/wal/record.go b/internal/wal/record.go index 1e06159..0a22811 100644 --- a/internal/wal/record.go +++ b/internal/wal/record.go @@ -12,6 +12,10 @@ const ( // RecLen (4B) + LSN (8B) + Type (1B) + KeyLen (2B) + ValLen (4B) + CRC32 (4B) = 23 Bytes. HeaderSize = 23 + // MaxFrameSize bounds one encoded record. The engine caps batches at 8 MiB + // of values, so anything larger on disk can only be a torn or corrupt header. + MaxFrameSize = 32 << 20 + // Operation Types OpPut byte = 0x01 OpDelete byte = 0x02 @@ -23,6 +27,7 @@ var ( ErrTornWrite = errors.New("wal: torn write or CRC32 checksum mismatch") ErrRecordTooSmall = errors.New("wal: record length smaller than header") ErrPayloadTooLarge = errors.New("wal: record payload exceeds max limit") + ErrFrameTooLarge = errors.New("wal: encoded record exceeds MaxFrameSize") ) // Record represents a single state mutation logged for durability. @@ -70,6 +75,10 @@ func DecodeRecord(r io.Reader) (*Record, error) { if recLen < HeaderSize { return nil, ErrRecordTooSmall } + if recLen > MaxFrameSize { + // A torn header can claim up to 4 GiB; treat it as a torn tail instead of allocating. + return nil, ErrTornWrite + } lsn := binary.LittleEndian.Uint64(header[4:12]) opType := header[12] diff --git a/internal/wal/wal.go b/internal/wal/wal.go index d637fb6..2d038cc 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -66,6 +66,10 @@ func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { } encoded := EncodeRecord(rec) + if len(encoded) > MaxFrameSize { + w.currentLSN-- + return 0, ErrFrameTooLarge + } if _, err := w.file.Write(encoded); err != nil { return 0, err } diff --git a/internal/wal/wal_test.go b/internal/wal/wal_test.go index 1aedf54..62076a8 100644 --- a/internal/wal/wal_test.go +++ b/internal/wal/wal_test.go @@ -2,6 +2,7 @@ package wal import ( "bytes" + "encoding/binary" "os" "path/filepath" "testing" @@ -132,3 +133,12 @@ func BenchmarkWALAppend(b *testing.B) { _, _ = wal.Write(OpPut, key, val) } } + +// A torn header claiming a huge length must read as a torn tail, not trigger a huge allocation. +func TestDecodeRejectsOversizedFrameHeader(t *testing.T) { + header := make([]byte, HeaderSize) + binary.LittleEndian.PutUint32(header[0:4], 0xFFFFFFF0) + if _, err := DecodeRecord(bytes.NewReader(header)); err != ErrTornWrite { + t.Fatalf("expected ErrTornWrite, got %v", err) + } +} From 259eff73ebab3605ee43c163ccc552b04e9b0731 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 17:32:05 +0000 Subject: [PATCH 04/16] fix(engine): verify a chain's CRC before freeing it; count WAL framing in batch limit - freeValue checked chunk magic and chain termination but not total length or CRC, so a mis-linked chain that still ended cleanly after the right number of chunks could free tuples owned by another key. It now verifies length and CRC first and leaves a failing chain in place. - MaxBatchBytes counted only value bytes. Each WAL batch operation adds 15 bytes of framing, so many tiny puts could commit a WAL frame larger than the replication message limit, and followers could never catch up on it. The limit now covers the encoded frame. Both reported by Codex review on #1; regression tests reproduce each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017mWMfRGkKdRgKc5mtSsWEG --- internal/api/engine_large.go | 34 +++++-------- internal/api/engine_large_review_test.go | 61 ++++++++++++++++++++++++ internal/api/engine_limits.go | 26 ++++++++++ internal/api/engine_ops.go | 4 +- internal/api/grpc_status.go | 6 ++- 5 files changed, 105 insertions(+), 26 deletions(-) create mode 100644 internal/api/engine_large_review_test.go create mode 100644 internal/api/engine_limits.go diff --git a/internal/api/engine_large.go b/internal/api/engine_large.go index ff26145..a18be3e 100644 --- a/internal/api/engine_large.go +++ b/internal/api/engine_large.go @@ -17,7 +17,7 @@ import ( const ( // MaxValueSize is the largest value accepted by Put, PutIf and BatchWrite. MaxValueSize = 2 << 20 - // MaxBatchBytes bounds the summed value bytes of one BatchWrite. + // MaxBatchBytes bounds one BatchWrite's encoded WAL frame: its values plus 15 bytes per operation. MaxBatchBytes = 8 << 20 largeSlotFlag = 0x8000 @@ -156,8 +156,10 @@ func (e *Engine) readValue(rid storage.RID) ([]byte, error) { return out, nil } -// freeValue releases the tuples behind rid. A chain that fails validation is -// left in place rather than risk tombstoning tuples that belong to other keys. +// freeValue releases the tuples behind rid. A chain that fails any check, +// including its length and CRC, is left in place: a mis-linked chain can end +// cleanly after the right number of chunks while running through another +// key's tuples, and freeing it would destroy that key's value. func (e *Engine) freeValue(rid storage.RID) { if !isLargeRID(rid) { e.freeTuple(rid) @@ -168,7 +170,13 @@ func (e *Engine) freeValue(rid storage.RID) { return } var chunks []storage.RID - if e.walkChain(m, func(r storage.RID, _ []byte) { chunks = append(chunks, r) }) != nil { + size, crc := 0, uint32(0) + visit := func(r storage.RID, p []byte) { + chunks = append(chunks, r) + size += len(p) + crc = crc32.Update(crc, crc32.IEEETable, p) + } + if e.walkChain(m, visit) != nil || uint32(size) != m.length || crc != m.crc { return } for _, r := range chunks { @@ -176,21 +184,3 @@ func (e *Engine) freeValue(rid storage.RID) { } e.freeTuple(manifestOf(rid)) } - -// checkValueSizes enforces per-value and per-batch limits before anything is logged. -func checkValueSizes(ops []BatchOperation) error { - total := 0 - for _, op := range ops { - if op.Type != BatchOpPut { - continue - } - if len(op.Value) > MaxValueSize { - return ErrRecordTooLarge - } - total += len(op.Value) - } - if total > MaxBatchBytes { - return ErrBatchTooLarge - } - return nil -} diff --git a/internal/api/engine_large_review_test.go b/internal/api/engine_large_review_test.go new file mode 100644 index 0000000..e7bd6a1 --- /dev/null +++ b/internal/api/engine_large_review_test.go @@ -0,0 +1,61 @@ +package api + +import ( + "encoding/binary" + "errors" + "testing" + + "github.com/ChromaBeast/beastdb/internal/storage" +) + +func chainRIDs(t *testing.T, e *Engine, key uint64) []storage.RID { + t.Helper() + rid, err := e.tree.Find(key) + if err != nil { + t.Fatal(err) + } + m, err := e.readManifest(rid) + if err != nil { + t.Fatal(err) + } + var out []storage.RID + if err := e.walkChain(m, func(r storage.RID, _ []byte) { out = append(out, r) }); err != nil { + t.Fatal(err) + } + return out +} + +// A mis-linked chain that still has the right length and terminates must not +// free chunks owned by another key: the value's length and CRC must check out first. +func TestFreeRefusesChainThatFailsCRC(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + _ = e.Put(1, pattern(20<<10, 1)) + _ = e.Put(2, pattern(20<<10, 2)) + a, b := chainRIDs(t, e, 1), chainRIDs(t, e, 2) + page, _ := e.bpm.FetchPage(a[0].PageID) + tuple, _ := page.GetTuple(a[0].SlotID) + binary.LittleEndian.PutUint64(tuple[1:9], b[1].PageID) + binary.LittleEndian.PutUint16(tuple[9:11], b[1].SlotID) + _ = e.bpm.UnpinPage(a[0].PageID, true) + + if err := e.Delete(1); err != nil { + t.Fatal(err) + } + mustGet(t, e, 2, pattern(20<<10, 2)) +} + +// The WAL frames each batch operation with 15 bytes; that framing must count +// toward MaxBatchBytes so a committed batch always fits a replication message. +func TestBatchLimitCountsWALFraming(t *testing.T) { + e := openLarge(t, t.TempDir()) + defer e.Close() + n := MaxBatchBytes/16 + 1 // one-byte values: fits the old value-only limit + ops := make([]BatchOperation, n) + for i := range ops { + ops[i] = BatchOperation{Type: BatchOpPut, Key: uint64(i), Value: []byte{1}} + } + if err := e.BatchWrite(ops); !errors.Is(err, ErrBatchTooLarge) { + t.Fatalf("expected ErrBatchTooLarge for a batch whose WAL frame exceeds the limit, got %v", err) + } +} diff --git a/internal/api/engine_limits.go b/internal/api/engine_limits.go new file mode 100644 index 0000000..90f71a8 --- /dev/null +++ b/internal/api/engine_limits.go @@ -0,0 +1,26 @@ +package api + +// walBatchOpOverhead is the WAL batch framing per operation: type (1), key +// length (2), the 8-byte key and value length (4); see wal.EncodeBatchPayload. +const walBatchOpOverhead = 15 + +// checkValueSizes enforces per-value and per-batch limits before anything is +// logged. The batch limit covers the encoded WAL frame, framing included, so a +// committed batch always fits in one replication message. +func checkValueSizes(ops []BatchOperation) error { + total := 4 // operation count + for _, op := range ops { + total += walBatchOpOverhead + if op.Type != BatchOpPut { + continue + } + if len(op.Value) > MaxValueSize { + return ErrRecordTooLarge + } + total += len(op.Value) + } + if total > MaxBatchBytes { + return ErrBatchTooLarge + } + return nil +} diff --git a/internal/api/engine_ops.go b/internal/api/engine_ops.go index c637262..a164e58 100644 --- a/internal/api/engine_ops.go +++ b/internal/api/engine_ops.go @@ -18,8 +18,8 @@ var ( ErrRecordExists = errors.New("record already exists") // ErrRecordTooLarge indicates a value exceeds MaxValueSize (2 MiB). ErrRecordTooLarge = errors.New("record value exceeds maximum supported size of 2 MiB") - // ErrBatchTooLarge indicates a batch's values sum to more than MaxBatchBytes (8 MiB). - ErrBatchTooLarge = errors.New("batch values exceed maximum supported total of 8 MiB") + // ErrBatchTooLarge indicates a batch's encoded WAL frame exceeds MaxBatchBytes (8 MiB). + ErrBatchTooLarge = errors.New("batch exceeds maximum supported size of 8 MiB (values plus 15 bytes per operation)") // ErrReadOnlyReplica indicates mutation attempts on a read-only replica node. ErrReadOnlyReplica = errors.New("writes are not permitted on read-only replica") ) diff --git a/internal/api/grpc_status.go b/internal/api/grpc_status.go index d235958..0277d09 100644 --- a/internal/api/grpc_status.go +++ b/internal/api/grpc_status.go @@ -8,8 +8,10 @@ import ( "google.golang.org/grpc/status" ) -// MaxGRPCMessageSize is the gRPC message limit servers and clients should use -// so a maximal batch (MaxBatchBytes of values plus keys and framing) fits. +// MaxGRPCMessageSize is the gRPC message limit servers and clients should use. +// A committed batch's WAL frame is at most MaxBatchBytes, so the 1 MiB margin +// covers the replication message around it; a client request with protobuf +// framing near the limit is rejected by gRPC before anything is written. const MaxGRPCMessageSize = MaxBatchBytes + 1<<20 // MessageSizeServerOptions raises gRPC's 4 MiB default to MaxGRPCMessageSize. From e815ba8d25aba827b27a073571265a2033f83daa Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:17:38 +0530 Subject: [PATCH 05/16] feat(auth): per-method gRPC role authorization and API token expiry Co-Authored-By: Claude Sonnet 5.5 --- internal/api/grpc_auth.go | 71 +++++++++++++++++++--- internal/api/grpc_auth_roles_test.go | 83 ++++++++++++++++++++++++++ internal/api/grpc_auth_test.go | 10 ++-- internal/web/auth/token_expiry_test.go | 54 +++++++++++++++++ internal/web/auth/token_store.go | 31 +++++++++- internal/web/handler/token_api.go | 20 ++++++- 6 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 internal/api/grpc_auth_roles_test.go create mode 100644 internal/web/auth/token_expiry_test.go diff --git a/internal/api/grpc_auth.go b/internal/api/grpc_auth.go index 3f0a115..fc77c09 100644 --- a/internal/api/grpc_auth.go +++ b/internal/api/grpc_auth.go @@ -16,6 +16,53 @@ type TokenValidator interface { Validate(raw string) bool } +// RoleTokenValidator is an optional extension of TokenValidator that also +// reports the role bound to a valid token. Validators that implement it get +// per-method authorization; plain TokenValidators are treated as admin +// (their historical behavior). +type RoleTokenValidator interface { + TokenValidator + ValidateRole(raw string) (role string, ok bool) +} + +// Roles understood by the gRPC authorizer. +const ( + RoleAdmin = "admin" + RoleWriter = "writer" + RoleViewer = "viewer" + RoleReplication = "replication" +) + +// ValidTokenRole reports whether role is a recognized API token role. +func ValidTokenRole(role string) bool { + switch role { + case RoleAdmin, RoleWriter, RoleViewer, RoleReplication: + return true + } + return false +} + +// methodRoles is the deny-by-default method -> allowed roles map. Methods not +// listed here are rejected for every caller. +var methodRoles = map[string][]string{ + "/beastdb.v1.BeastDBService/Get": {RoleViewer, RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Scan": {RoleViewer, RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Put": {RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Delete": {RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/BatchWrite": {RoleWriter, RoleAdmin}, + "/beastdb.v1.ReplicationService/StreamWAL": {RoleReplication, RoleAdmin}, + "/beastdb.v1.ReplicationService/Ack": {RoleReplication, RoleAdmin}, +} + +func roleAllowed(role, method string) bool { + for _, r := range methodRoles[method] { + if r == role { + return true + } + } + return false +} + // AuthInterceptor enforces Bearer token authentication on incoming gRPC RPCs. type AuthInterceptor struct { staticToken string @@ -37,7 +84,7 @@ func NewAuthInterceptor(staticToken string, validator ...TokenValidator) *AuthIn // Unary returns a grpc.UnaryServerInterceptor. func (a *AuthInterceptor) Unary() grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - if err := a.authorize(ctx); err != nil { + if err := a.authorize(ctx, info.FullMethod); err != nil { return nil, err } return handler(ctx, req) @@ -47,14 +94,14 @@ func (a *AuthInterceptor) Unary() grpc.UnaryServerInterceptor { // Stream returns a grpc.StreamServerInterceptor. func (a *AuthInterceptor) Stream() grpc.StreamServerInterceptor { return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - if err := a.authorize(ss.Context()); err != nil { + if err := a.authorize(ss.Context(), info.FullMethod); err != nil { return err } return handler(srv, ss) } } -func (a *AuthInterceptor) authorize(ctx context.Context) error { +func (a *AuthInterceptor) authorize(ctx context.Context, fullMethod string) error { if a.staticToken == "" && a.validator == nil { return nil } @@ -71,11 +118,21 @@ func (a *AuthInterceptor) authorize(ctx context.Context) error { return status.Errorf(codes.Unauthenticated, "authorization must be Bearer token") } token := strings.TrimSpace(header[7:]) + role := "" if a.staticToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(a.staticToken)) == 1 { - return nil + role = RoleAdmin + } else if rv, ok := a.validator.(RoleTokenValidator); ok && rv != nil { + if r, valid := rv.ValidateRole(token); valid { + role = r + } + } else if a.validator != nil && a.validator.Validate(token) { + role = RoleAdmin // legacy validator without roles } - if a.validator != nil && a.validator.Validate(token) { - return nil + if role == "" { + return status.Errorf(codes.Unauthenticated, "invalid authorization token") + } + if !roleAllowed(role, fullMethod) { + return status.Errorf(codes.PermissionDenied, "role %q is not permitted to call %s", role, fullMethod) } - return status.Errorf(codes.Unauthenticated, "invalid authorization token") + return nil } diff --git a/internal/api/grpc_auth_roles_test.go b/internal/api/grpc_auth_roles_test.go new file mode 100644 index 0000000..80ecd62 --- /dev/null +++ b/internal/api/grpc_auth_roles_test.go @@ -0,0 +1,83 @@ +package api + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type fakeRoleValidator map[string]string + +func (f fakeRoleValidator) Validate(raw string) bool { _, ok := f[raw]; return ok } +func (f fakeRoleValidator) ValidateRole(raw string) (string, bool) { + r, ok := f[raw] + return r, ok +} + +type legacyValidator struct{ tok string } + +func (l legacyValidator) Validate(raw string) bool { return raw == l.tok } + +func bearer(tok string) context.Context { + return metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+tok)) +} + +func TestAuthorizeMethodRoleMatrix(t *testing.T) { + const svc = "/beastdb.v1.BeastDBService/" + const rep = "/beastdb.v1.ReplicationService/" + v := fakeRoleValidator{"v": "viewer", "w": "writer", "a": "admin", "r": "replication", "x": "bogus"} + ai := NewAuthInterceptor("static", v) + + allow := map[string]map[string]bool{ + svc + "Get": {"viewer": true, "writer": true, "admin": true}, + svc + "Scan": {"viewer": true, "writer": true, "admin": true}, + svc + "Put": {"writer": true, "admin": true}, + svc + "Delete": {"writer": true, "admin": true}, + svc + "BatchWrite": {"writer": true, "admin": true}, + rep + "StreamWAL": {"replication": true, "admin": true}, + rep + "Ack": {"replication": true, "admin": true}, + } + tokens := map[string]string{"viewer": "v", "writer": "w", "admin": "a", "replication": "r", "bogus": "x", "static": "static"} + for method, ok := range allow { + for role, tok := range tokens { + want := ok[role] || role == "static" && ok["admin"] + err := ai.authorize(bearer(tok), method) + if want && err != nil { + t.Errorf("%s %s: expected allow, got %v", role, method, err) + } + if !want { + if status.Code(err) != codes.PermissionDenied { + t.Errorf("%s %s: expected PermissionDenied, got %v", role, method, err) + } + } + } + } +} + +func TestAuthorizeDenyByDefaultAndCodes(t *testing.T) { + ai := NewAuthInterceptor("static", fakeRoleValidator{"a": "admin"}) + for _, tok := range []string{"static", "a"} { + if c := status.Code(ai.authorize(bearer(tok), "/beastdb.v1.BeastDBService/Unknown")); c != codes.PermissionDenied { + t.Errorf("unknown method with %s: got %v", tok, c) + } + } + if c := status.Code(ai.authorize(bearer("nope"), "/beastdb.v1.BeastDBService/Get")); c != codes.Unauthenticated { + t.Errorf("bad token: got %v", c) + } + if c := status.Code(ai.authorize(context.Background(), "/beastdb.v1.BeastDBService/Get")); c != codes.Unauthenticated { + t.Errorf("no metadata: got %v", c) + } +} + +func TestAuthorizeLegacyValidatorIsAdmin(t *testing.T) { + ai := NewAuthInterceptor("", legacyValidator{"legacy"}) + if err := ai.authorize(bearer("legacy"), "/beastdb.v1.BeastDBService/Put"); err != nil { + t.Fatalf("legacy validator should act as admin: %v", err) + } + if c := status.Code(ai.authorize(bearer("bad"), "/beastdb.v1.BeastDBService/Put")); c != codes.Unauthenticated { + t.Fatalf("got %v", c) + } +} diff --git a/internal/api/grpc_auth_test.go b/internal/api/grpc_auth_test.go index fedb5ad..37b1a2b 100644 --- a/internal/api/grpc_auth_test.go +++ b/internal/api/grpc_auth_test.go @@ -7,30 +7,32 @@ import ( "google.golang.org/grpc/metadata" ) +const getM = "/beastdb.v1.BeastDBService/Get" + func TestAuthInterceptor(t *testing.T) { ai := NewAuthInterceptor("secret-token-123") // 1. Missing metadata -> unauthenticated - err := ai.authorize(context.Background()) + err := ai.authorize(context.Background(), getM) if err == nil { t.Fatalf("expected error for missing metadata, got nil") } // 2. Wrong token -> unauthenticated badCtx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer wrong")) - if err := ai.authorize(badCtx); err == nil { + if err := ai.authorize(badCtx, getM); err == nil { t.Fatalf("expected error for bad token, got nil") } // 3. Valid token -> allowed goodCtx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer secret-token-123")) - if err := ai.authorize(goodCtx); err != nil { + if err := ai.authorize(goodCtx, getM); err != nil { t.Fatalf("expected success for valid token, got %v", err) } // 4. Empty configured token -> open access openAI := NewAuthInterceptor("") - if err := openAI.authorize(context.Background()); err != nil { + if err := openAI.authorize(context.Background(), getM); err != nil { t.Fatalf("expected success when token is empty, got %v", err) } } diff --git a/internal/web/auth/token_expiry_test.go b/internal/web/auth/token_expiry_test.go new file mode 100644 index 0000000..c2e53c8 --- /dev/null +++ b/internal/web/auth/token_expiry_test.go @@ -0,0 +1,54 @@ +package auth + +import ( + "encoding/json" + "testing" + "time" +) + +func TestTokenExpiryAndRole(t *testing.T) { + engine := newMockEngine() + scan := func(start, end uint64) ([]TokenRecord, error) { + var recs []TokenRecord + for k, v := range engine.data { + if k >= start && k <= end { + recs = append(recs, TokenRecord{Key: k, Value: v}) + } + } + return recs, nil + } + ts := NewTokenStore(engine, scan) + + past := time.Now().Add(-time.Minute) + future := time.Now().Add(time.Hour) + expired, _, _ := ts.CreateWithExpiry("old", "viewer", "t", &past) + live, _, _ := ts.CreateWithExpiry("new", "writer", "t", &future) + forever, _, _ := ts.Create("forever", "viewer", "t") + + if ts.Validate(expired) { + t.Fatal("expired token must not validate") + } + if role, ok := ts.ValidateRole(live); !ok || role != "writer" { + t.Fatalf("live token: %q %v", role, ok) + } + if role, ok := ts.ValidateRole(forever); !ok || role != "viewer" { + t.Fatalf("no-expiry token: %q %v", role, ok) + } + + // Persisted and reloaded: expiry still enforced. + ts2 := NewTokenStore(engine, scan) + if ts2.Validate(expired) || !ts2.Validate(live) { + t.Fatal("expiry not preserved across reload") + } +} + +func TestLegacyStoredTokenWithoutExpiry(t *testing.T) { + raw := "bst_live_legacy" + rec, _ := json.Marshal(map[string]any{"id": "tok_1", "name": "n", "token_hash": hashToken(raw), "role": "admin"}) + ts := NewTokenStore(newMockEngine(), func(start, end uint64) ([]TokenRecord, error) { + return []TokenRecord{{Key: tokenKey("tok_1"), Value: rec}}, nil + }) + if role, ok := ts.ValidateRole(raw); !ok || role != "admin" { + t.Fatalf("legacy token should validate as admin, got %q %v", role, ok) + } +} diff --git a/internal/web/auth/token_store.go b/internal/web/auth/token_store.go index 5e0fde2..456a2ce 100644 --- a/internal/web/auth/token_store.go +++ b/internal/web/auth/token_store.go @@ -21,6 +21,13 @@ type APIToken struct { Role string `json:"role"` CreatedAt time.Time `json:"created_at"` CreatedBy string `json:"created_by"` + // ExpiresAt is optional; nil (or absent in older stored tokens) means no expiry. + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// Expired reports whether the token has an expiry that is at or before now. +func (t APIToken) Expired(now time.Time) bool { + return t.ExpiresAt != nil && !now.Before(*t.ExpiresAt) } // TokenEngine is the storage interface needed for token persistence. @@ -67,6 +74,11 @@ func hashToken(raw string) string { // Create generates a new API token, saves it to partition 0x05, and activates it. func (ts *TokenStore) Create(name, role, createdBy string) (string, *APIToken, error) { + return ts.CreateWithExpiry(name, role, createdBy, nil) +} + +// CreateWithExpiry is Create with an optional expiry time (nil = never expires). +func (ts *TokenStore) CreateWithExpiry(name, role, createdBy string, expiresAt *time.Time) (string, *APIToken, error) { idBytes := make([]byte, 4) tokenBytes := make([]byte, 20) if _, err := rand.Read(idBytes); err != nil { @@ -90,6 +102,10 @@ func (ts *TokenStore) Create(name, role, createdBy string) (string, *APIToken, e CreatedAt: time.Now().UTC(), CreatedBy: createdBy, } + if expiresAt != nil { + e := expiresAt.UTC() + tok.ExpiresAt = &e + } data, err := json.Marshal(tok) if err != nil { @@ -135,11 +151,22 @@ func (ts *TokenStore) Delete(id string) error { // Validate checks whether a given raw token is valid in < 100ns without disk I/O. func (ts *TokenStore) Validate(raw string) bool { + _, ok := ts.ValidateRole(raw) + return ok +} + +// ValidateRole returns the token's role if it exists and has not expired. +// Lookup is by SHA-256 hash of the presented token (map lookup on a hash, so +// no raw-secret comparison timing leak). +func (ts *TokenStore) ValidateRole(raw string) (string, bool) { th := hashToken(raw) ts.mu.RLock() - _, exists := ts.tokenIndex[th] + tok, exists := ts.tokenIndex[th] ts.mu.RUnlock() - return exists + if !exists || tok.Expired(time.Now()) { + return "", false + } + return tok.Role, true } // Reload repopulates the in-memory token cache from partition 0x05. diff --git a/internal/web/handler/token_api.go b/internal/web/handler/token_api.go index b446b16..47ddaf6 100644 --- a/internal/web/handler/token_api.go +++ b/internal/web/handler/token_api.go @@ -4,7 +4,9 @@ import ( "encoding/json" "net/http" "strings" + "time" + "github.com/ChromaBeast/beastdb/internal/api" "github.com/ChromaBeast/beastdb/internal/web/auth" ) @@ -52,6 +54,8 @@ func (h *APIHandler) CreateToken(w http.ResponseWriter, r *http.Request) { var body struct { Name string `json:"name"` Role string `json:"role"` + // ExpiresInSeconds optionally makes the token expire after N seconds (0 = never). + ExpiresInSeconds int64 `json:"expires_in_seconds"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { http.Error(w, "Bad Request", http.StatusBadRequest) @@ -69,7 +73,21 @@ func (h *APIHandler) CreateToken(w http.ResponseWriter, r *http.Request) { tokRole = "admin" } - rawToken, meta, err := h.tokens.Create(name, tokRole, username) + if !api.ValidTokenRole(tokRole) { + http.Error(w, "Invalid role: must be admin, writer, viewer or replication", http.StatusBadRequest) + return + } + if body.ExpiresInSeconds < 0 || body.ExpiresInSeconds > 10*365*24*3600 { + http.Error(w, "Invalid expires_in_seconds", http.StatusBadRequest) + return + } + var expiresAt *time.Time + if body.ExpiresInSeconds > 0 { + t := time.Now().Add(time.Duration(body.ExpiresInSeconds) * time.Second) + expiresAt = &t + } + + rawToken, meta, err := h.tokens.CreateWithExpiry(name, tokRole, username, expiresAt) if err != nil { http.Error(w, "Failed to create token: "+err.Error(), http.StatusInternalServerError) return From 7ed1da2b0e7c3cdffffcdaf2bfb5c74b95d6a6b2 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:19:18 +0530 Subject: [PATCH 06/16] feat(bench): add beastbench YCSB-style load generator and benchmarking guide Co-Authored-By: Claude Sonnet 5.5 --- cmd/beastbench/beastbench_test.go | 185 +++++++++++++++++ cmd/beastbench/client.go | 103 ++++++++++ cmd/beastbench/dist.go | 82 ++++++++ cmd/beastbench/histogram.go | 131 ++++++++++++ cmd/beastbench/main.go | 151 ++++++++++++++ cmd/beastbench/runner.go | 318 ++++++++++++++++++++++++++++++ cmd/beastbench/selfhost.go | 39 ++++ cmd/beastbench/workload.go | 73 +++++++ docs/benchmarking.md | 172 ++++++++++++++++ 9 files changed, 1254 insertions(+) create mode 100644 cmd/beastbench/beastbench_test.go create mode 100644 cmd/beastbench/client.go create mode 100644 cmd/beastbench/dist.go create mode 100644 cmd/beastbench/histogram.go create mode 100644 cmd/beastbench/main.go create mode 100644 cmd/beastbench/runner.go create mode 100644 cmd/beastbench/selfhost.go create mode 100644 cmd/beastbench/workload.go create mode 100644 docs/benchmarking.md diff --git a/cmd/beastbench/beastbench_test.go b/cmd/beastbench/beastbench_test.go new file mode 100644 index 0000000..5bd0323 --- /dev/null +++ b/cmd/beastbench/beastbench_test.go @@ -0,0 +1,185 @@ +package main + +import ( + "bytes" + "encoding/json" + "math/rand" + "sort" + "sync/atomic" + "testing" + "time" +) + +func TestHistogramQuantiles(t *testing.T) { + h := NewHistogram() + var vals []int + for i := 1; i <= 100000; i++ { + d := time.Duration(i) * time.Microsecond + h.Record(d) + vals = append(vals, int(d)) + } + sort.Ints(vals) + for _, q := range []float64{.5, .95, .99} { + want := float64(vals[int(q*float64(len(vals)))-1]) + got := float64(h.Quantile(q)) + if got < want || got > want*1.02 { + t.Errorf("q=%v got %v want ~%v", q, got, want) + } + } + if h.Max() != 100000*time.Microsecond || h.Min() != time.Microsecond { + t.Errorf("min/max wrong: %v %v", h.Min(), h.Max()) + } + if h.Count() != 100000 { + t.Errorf("count %d", h.Count()) + } +} + +func TestHistogramBucketsRoundTrip(t *testing.T) { + for _, v := range []uint64{0, 1, 127, 128, 255, 256, 257, 1000, 1 << 20, 1<<40 + 12345, 1<<63 + 5} { + lo, hi := bucketBounds(bucketIndex(v)) + if v < lo || v > hi { + t.Errorf("v=%d not in [%d,%d]", v, lo, hi) + } + } +} + +func TestHistogramMergeAndEmpty(t *testing.T) { + a, b := NewHistogram(), NewHistogram() + if a.Quantile(.5) != 0 || a.Mean() != 0 || a.Min() != 0 { + t.Error("empty histogram should report zeros") + } + a.Record(time.Millisecond) + b.Record(3 * time.Millisecond) + a.Merge(b) + if a.Count() != 2 || a.Max() != 3*time.Millisecond || a.Min() != time.Millisecond { + t.Errorf("merge wrong: %d %v %v", a.Count(), a.Min(), a.Max()) + } +} + +func TestZipfianSkewAndRange(t *testing.T) { + const n = 1000 + z := NewZipfian(n, DefaultZipfTheta) + r := rand.New(rand.NewSource(1)) + counts := make([]int, n) + for i := 0; i < 200000; i++ { + v := z.Next(r) + if v >= n { + t.Fatalf("out of range: %d", v) + } + counts[v]++ + } + if counts[0] < counts[10] || counts[10] < counts[500] { + t.Errorf("not monotonically skewed: %d %d %d", counts[0], counts[10], counts[500]) + } + if counts[0] < 200000/20 { + t.Errorf("item 0 too cold: %d", counts[0]) + } +} + +func TestUniformAndDeterminism(t *testing.T) { + u := Uniform{N: 10} + r := rand.New(rand.NewSource(7)) + seen := map[uint64]bool{} + for i := 0; i < 1000; i++ { + seen[u.Next(r)] = true + } + if len(seen) != 10 { + t.Errorf("uniform missed buckets: %d", len(seen)) + } + z := NewZipfian(100, .9) + a, b := rand.New(rand.NewSource(3)), rand.New(rand.NewSource(3)) + for i := 0; i < 100; i++ { + if z.Next(a) != z.Next(b) { + t.Fatal("same seed must give same sequence") + } + } +} + +func TestLatestFavoursNewest(t *testing.T) { + f := &atomic.Uint64{} + f.Store(1000) + l := Latest{Zipf: NewZipfian(1000, DefaultZipfTheta), Frontier: f} + r := rand.New(rand.NewSource(2)) + newest := 0 + for i := 0; i < 10000; i++ { + v := l.Next(r) + if v >= 1000 { + t.Fatalf("out of range %d", v) + } + if v >= 990 { + newest++ + } + } + if newest < 2000 { + t.Errorf("latest not skewed to newest: %d", newest) + } +} + +func TestWorkloadMixes(t *testing.T) { + for name, w := range workloads { + var sum float64 + for _, p := range w.Mix { + sum += p + } + if sum < .999 || sum > 1.001 { + t.Errorf("workload %s mix sums to %v", name, sum) + } + } + w := workloads["A"] + if w.pick(0.1) != OpRead || w.pick(0.9) != OpUpdate { + t.Error("A pick wrong") + } + if _, err := lookupWorkload("Z"); err == nil { + t.Error("expected error for unknown workload") + } +} + +func TestSmokeAllWorkloads(t *testing.T) { + for _, wl := range []string{"A", "B", "C", "D", "E", "F"} { + wl := wl + t.Run(wl, func(t *testing.T) { + var out, errb bytes.Buffer + err := realMain([]string{"--self-contained", "--json", "--workload", wl, "--records", "200", + "--ops", "300", "--warmup", "20", "--threads", "4", "--value-size", "64", "--seed", "5", "--max-scan-len", "10"}, &out, &errb) + if err != nil { + t.Fatalf("%v (%s)", err, errb.String()) + } + var rep Report + if err := json.Unmarshal(out.Bytes(), &rep); err != nil { + t.Fatal(err) + } + if len(rep.Phases) != 2 || rep.Phases[1].Ops != 300 { + t.Fatalf("unexpected phases: %+v", rep.Phases) + } + for _, p := range rep.Phases { + for _, o := range p.PerOp { + if o.Errors != 0 { + t.Errorf("%s/%s: %d errors", p.Phase, o.Op, o.Errors) + } + if o.P50Us > o.P99Us || o.P99Us > o.MaxUs { + t.Errorf("%s/%s percentiles not monotonic: %+v", p.Phase, o.Op, o) + } + } + } + if rep.Phases[0].Ops != 200 { + t.Errorf("load ops %d", rep.Phases[0].Ops) + } + }) + } +} + +func TestTextOutputAndBadFlags(t *testing.T) { + var out, errb bytes.Buffer + if err := realMain([]string{"--self-contained", "--records", "50", "--ops", "50", "--threads", "2"}, &out, &errb); err != nil { + t.Fatal(err) + } + if !bytes.Contains(out.Bytes(), []byte("not a competitive benchmark")) || !bytes.Contains(out.Bytes(), []byte("p99(us)")) { + t.Errorf("text output missing pieces:\n%s", out.String()) + } + if err := realMain([]string{"--workload", "Q"}, &out, &errb); err == nil { + t.Error("expected error") + } + if err := realMain([]string{"--value-size", "1"}, &out, &errb); err == nil { + t.Error("expected error") + } +} diff --git a/cmd/beastbench/client.go b/cmd/beastbench/client.go new file mode 100644 index 0000000..2812911 --- /dev/null +++ b/cmd/beastbench/client.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "github.com/ChromaBeast/beastdb/internal/api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// tokenAuth mirrors cmd/beastctl: Bearer token in the authorization header. +type tokenAuth struct{ token string } + +func (t tokenAuth) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + if t.token == "" { + return nil, nil + } + return map[string]string{"authorization": "Bearer " + t.token}, nil +} + +// RequireTransportSecurity is false to match beastctl, so tokens work on plain +// loopback test servers; use the TLS flags for anything non-local. +func (t tokenAuth) RequireTransportSecurity() bool { return false } + +// ConnOpts describes how to reach the server. +type ConnOpts struct { + Addr, Token string + TLSCert, TLSKey, TLSCA string + TLSServerName string + Conns int +} + +func (o ConnOpts) creds() (credentials.TransportCredentials, error) { + if o.TLSCert == "" && o.TLSCA == "" { + return insecure.NewCredentials(), nil + } + cfg := &tls.Config{MinVersion: tls.VersionTLS13, ServerName: o.TLSServerName} + if o.TLSCA != "" { + pem, err := os.ReadFile(o.TLSCA) + if err != nil { + return nil, fmt.Errorf("read CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no certificates in %s", o.TLSCA) + } + cfg.RootCAs = pool + } + if o.TLSCert != "" || o.TLSKey != "" { + cert, err := tls.LoadX509KeyPair(o.TLSCert, o.TLSKey) + if err != nil { + return nil, fmt.Errorf("load client cert: %w", err) + } + cfg.Certificates = []tls.Certificate{cert} + } + return credentials.NewTLS(cfg), nil +} + +// Dial opens o.Conns connections and verifies the server answers. +func Dial(o ConnOpts) ([]beastv1.BeastDBServiceClient, func(), error) { + creds, err := o.creds() + if err != nil { + return nil, nil, err + } + n := o.Conns + if n < 1 { + n = 1 + } + var conns []*grpc.ClientConn + closeAll := func() { + for _, c := range conns { + c.Close() + } + } + var out []beastv1.BeastDBServiceClient + for i := 0; i < n; i++ { + opts := []grpc.DialOption{grpc.WithTransportCredentials(creds), api.MessageSizeDialOption()} + if o.Token != "" { + opts = append(opts, grpc.WithPerRPCCredentials(tokenAuth{o.Token})) + } + conn, err := grpc.NewClient(o.Addr, opts...) + if err != nil { + closeAll() + return nil, nil, fmt.Errorf("dial %s: %w", o.Addr, err) + } + conns = append(conns, conn) + out = append(out, beastv1.NewBeastDBServiceClient(conn)) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := out[0].Get(ctx, &beastv1.GetRequest{Key: 0}); err != nil { + closeAll() + return nil, nil, fmt.Errorf("could not reach BeastDB at %s: %w", o.Addr, err) + } + return out, closeAll, nil +} diff --git a/cmd/beastbench/dist.go b/cmd/beastbench/dist.go new file mode 100644 index 0000000..c5eb65a --- /dev/null +++ b/cmd/beastbench/dist.go @@ -0,0 +1,82 @@ +package main + +import ( + "math" + "math/rand" + "sync/atomic" +) + +// KeyChooser picks a record index in [0, n). +type KeyChooser interface { + Next(r *rand.Rand) uint64 +} + +// Uniform picks indexes uniformly. +type Uniform struct{ N uint64 } + +func (u Uniform) Next(r *rand.Rand) uint64 { return uint64(r.Int63n(int64(u.N))) } + +// Zipfian implements the YCSB / Gray et al. generator. Index 0 is the most +// popular item. Parameters are precomputed so Next is O(1) and the value is +// safe for concurrent use (each caller supplies its own *rand.Rand). +type Zipfian struct { + n uint64 + theta, alpha, zeta float64 + zeta2, eta float64 +} + +// DefaultZipfTheta is YCSB's default skew constant. +const DefaultZipfTheta = 0.99 + +func zetaN(n uint64, theta float64) float64 { + var s float64 + for i := uint64(1); i <= n; i++ { + s += 1 / math.Pow(float64(i), theta) + } + return s +} + +// NewZipfian builds a generator over n items. +func NewZipfian(n uint64, theta float64) *Zipfian { + z := &Zipfian{n: n, theta: theta} + z.zeta2 = zetaN(2, theta) + z.zeta = zetaN(n, theta) + z.alpha = 1 / (1 - theta) + z.eta = (1 - math.Pow(2/float64(n), 1-theta)) / (1 - z.zeta2/z.zeta) + return z +} + +func (z *Zipfian) Next(r *rand.Rand) uint64 { + u := r.Float64() + uz := u * z.zeta + if uz < 1 { + return 0 + } + if uz < 1+math.Pow(0.5, z.theta) { + return 1 % z.n + } + v := uint64(float64(z.n) * math.Pow(z.eta*u-z.eta+1, z.alpha)) + if v >= z.n { + v = z.n - 1 + } + return v +} + +// Latest favours the most recently inserted records (workload D): it draws a +// zipfian offset back from the current insert frontier. +type Latest struct { + Zipf *Zipfian + Frontier *atomic.Uint64 // number of records currently present +} + +func (l Latest) Next(r *rand.Rand) uint64 { + n := l.Frontier.Load() + if n == 0 { + return 0 + } + off := l.Zipf.Next(r) + if off >= n { + off = n - 1 + } + return n - 1 - off +} diff --git a/cmd/beastbench/histogram.go b/cmd/beastbench/histogram.go new file mode 100644 index 0000000..e368129 --- /dev/null +++ b/cmd/beastbench/histogram.go @@ -0,0 +1,131 @@ +package main + +import ( + "math" + "math/bits" + "time" +) + +// Histogram is a small HDR-style log-linear histogram of nanosecond values. +// Each power-of-two range is split into 128 linear sub-buckets, bounding the +// relative quantile error to under 1%. It is not safe for concurrent use; +// give each worker its own and Merge afterwards. +type Histogram struct { + counts [histBuckets]uint64 + total uint64 + sum float64 + min uint64 + max uint64 +} + +const ( + histSubBits = 7 + histSub = 1 << histSubBits + histBuckets = (64-histSubBits)*histSub + 2*histSub +) + +// NewHistogram returns an empty histogram. +func NewHistogram() *Histogram { return &Histogram{min: math.MaxUint64} } + +func bucketIndex(v uint64) int { + shift := bits.Len64(v) - histSubBits - 1 + if shift < 0 { + shift = 0 + } + return shift*histSub + int(v>>uint(shift)) +} + +// bucketBounds returns the inclusive value range covered by bucket idx. +func bucketBounds(idx int) (lo, hi uint64) { + if idx < 2*histSub { + return uint64(idx), uint64(idx) + } + shift := uint(idx/histSub - 1) + m := uint64(idx%histSub + histSub) + lo = m << shift + return lo, lo + (uint64(1) << shift) - 1 +} + +// Record adds one observation. +func (h *Histogram) Record(d time.Duration) { + if d < 0 { + d = 0 + } + v := uint64(d) + h.counts[bucketIndex(v)]++ + h.total++ + h.sum += float64(v) + if v < h.min { + h.min = v + } + if v > h.max { + h.max = v + } +} + +// Count returns the number of observations. +func (h *Histogram) Count() uint64 { return h.total } + +// Max returns the largest observation (exact). +func (h *Histogram) Max() time.Duration { return time.Duration(h.max) } + +// Min returns the smallest observation (exact), or 0 when empty. +func (h *Histogram) Min() time.Duration { + if h.total == 0 { + return 0 + } + return time.Duration(h.min) +} + +// Mean returns the arithmetic mean. +func (h *Histogram) Mean() time.Duration { + if h.total == 0 { + return 0 + } + return time.Duration(h.sum / float64(h.total)) +} + +// Quantile returns the value at quantile q in [0,1] (bucket upper bound, +// clamped to the exact max). +func (h *Histogram) Quantile(q float64) time.Duration { + if h.total == 0 { + return 0 + } + if q >= 1 { + return h.Max() + } + rank := uint64(math.Ceil(q * float64(h.total))) + if rank < 1 { + rank = 1 + } + var seen uint64 + for i, c := range h.counts { + if c == 0 { + continue + } + seen += c + if seen >= rank { + _, hi := bucketBounds(i) + if hi > h.max { + hi = h.max + } + return time.Duration(hi) + } + } + return h.Max() +} + +// Merge adds all observations of o into h. +func (h *Histogram) Merge(o *Histogram) { + for i, c := range o.counts { + h.counts[i] += c + } + h.total += o.total + h.sum += o.sum + if o.total > 0 && o.min < h.min { + h.min = o.min + } + if o.max > h.max { + h.max = o.max + } +} diff --git a/cmd/beastbench/main.go b/cmd/beastbench/main.go new file mode 100644 index 0000000..561dd38 --- /dev/null +++ b/cmd/beastbench/main.go @@ -0,0 +1,151 @@ +// Command beastbench is a reproducible YCSB-style load generator for BeastDB. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/signal" + "runtime" + "strings" + "time" +) + +// Report is the full machine-readable result. +type Report struct { + Tool string `json:"tool"` + GoVersion string `json:"go_version"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + NumCPU int `json:"num_cpu"` + SelfHosted bool `json:"self_hosted"` + Addr string `json:"addr"` + Config Config `json:"config"` + Phases []PhaseResult `json:"phases"` +} + +func main() { + if err := realMain(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "beastbench:", err) + os.Exit(1) + } +} + +func realMain(args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("beastbench", flag.ContinueOnError) + fs.SetOutput(stderr) + var ( + cfg Config + co ConnOpts + self = fs.Bool("self-contained", false, "start an in-process engine+gRPC server on a temp dir (smoke runs only)") + poolSize = fs.Int("pool-size", 128, "buffer pool frames for --self-contained") + asJSON = fs.Bool("json", false, "emit JSON instead of text") + skipLoad = fs.Bool("skip-load", false, "skip the load phase (dataset already loaded with the same --records/--value-size/--seed)") + loadOnly = fs.Bool("load-only", false, "only run the load phase") + timeout = fs.Duration("op-timeout", 30*time.Second, "per-operation timeout") + ) + fs.StringVar(&co.Addr, "addr", "127.0.0.1:50051", "BeastDB gRPC address") + fs.StringVar(&co.Token, "token", strings.TrimSpace(os.Getenv("BEASTDB_API_TOKEN")), "Bearer token (default $BEASTDB_API_TOKEN)") + fs.StringVar(&co.TLSCert, "tls-cert", os.Getenv("BEASTDB_TLS_CERT"), "client TLS certificate (PEM)") + fs.StringVar(&co.TLSKey, "tls-key", os.Getenv("BEASTDB_TLS_KEY"), "client TLS key (PEM)") + fs.StringVar(&co.TLSCA, "tls-ca", os.Getenv("BEASTDB_TLS_CA"), "CA certificate to verify the server (PEM)") + fs.StringVar(&co.TLSServerName, "tls-server-name", "", "override TLS server name") + fs.IntVar(&co.Conns, "conns", 1, "gRPC connections (threads are spread round-robin)") + fs.StringVar(&cfg.Workload, "workload", "A", "workload A-F") + fs.StringVar(&cfg.Dist, "dist", "zipfian", "key distribution: zipfian or uniform (workload D always uses read-latest)") + fs.Float64Var(&cfg.ZipfTheta, "zipf-theta", DefaultZipfTheta, "zipfian skew constant in (0,1)") + fs.Uint64Var(&cfg.Records, "records", 10000, "records loaded before the run") + fs.Uint64Var(&cfg.Ops, "ops", 50000, "operations in the measured run phase") + fs.Uint64Var(&cfg.WarmupOps, "warmup", 0, "unmeasured warmup operations before the run phase") + fs.IntVar(&cfg.Threads, "threads", 8, "concurrent client workers") + fs.IntVar(&cfg.ValueSize, "value-size", 100, "value size in bytes (8..2097152)") + fs.IntVar(&cfg.MaxScanLen, "max-scan-len", 100, "workload E scan length is uniform in [1,max-scan-len]") + fs.Int64Var(&cfg.Seed, "seed", 1, "RNG seed; same seed gives the same operation sequence per thread") + if err := fs.Parse(args); err != nil { + return err + } + cfg.Workload = strings.ToUpper(cfg.Workload) + cfg.Dist = strings.ToLower(cfg.Dist) + cfg.Timeout = *timeout + if _, err := lookupWorkload(cfg.Workload); err != nil { + return err + } + switch { + case cfg.Dist != "zipfian" && cfg.Dist != "uniform": + return fmt.Errorf("--dist must be zipfian or uniform") + case cfg.Records < 2, cfg.Threads < 1, cfg.MaxScanLen < 1: + return fmt.Errorf("need --records >= 2, --threads >= 1, --max-scan-len >= 1") + case cfg.ValueSize < 8 || cfg.ValueSize > 2<<20: + return fmt.Errorf("--value-size must be in [8, 2097152]") + case cfg.ZipfTheta <= 0 || cfg.ZipfTheta >= 1: + return fmt.Errorf("--zipf-theta must be in (0,1)") + } + if uint64(cfg.Threads) > cfg.Records { + cfg.Threads = int(cfg.Records) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + if *self { + addr, stopSrv, err := StartSelfHosted(*poolSize) + if err != nil { + return err + } + defer stopSrv() + co.Addr = addr + co.TLSCert, co.TLSKey, co.TLSCA, co.Token = "", "", "", "" + } + cl, closeConns, err := Dial(co) + if err != nil { + return err + } + defer closeConns() + + rep := Report{Tool: "beastbench", GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, + NumCPU: runtime.NumCPU(), SelfHosted: *self, Addr: co.Addr, Config: cfg} + + if !*skipLoad { + res, err := Load(ctx, cl, cfg) + rep.Phases = append(rep.Phases, res) + if err != nil { + return err + } + } + if !*loadOnly { + res, err := Run(ctx, cl, cfg) + rep.Phases = append(rep.Phases, res) + if err != nil { + return err + } + } + if *asJSON { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + return enc.Encode(rep) + } + printText(stdout, rep) + return nil +} + +func printText(w io.Writer, r Report) { + c := r.Config + fmt.Fprintf(w, "beastbench workload=%s dist=%s records=%d ops=%d warmup=%d threads=%d value=%dB seed=%d\n", + c.Workload, c.Dist, c.Records, c.Ops, c.WarmupOps, c.Threads, c.ValueSize, c.Seed) + fmt.Fprintf(w, "%s/%s go=%s cpus=%d self-contained=%v addr=%s\n", r.GOOS, r.GOARCH, r.GoVersion, r.NumCPU, r.SelfHosted, r.Addr) + if r.SelfHosted { + fmt.Fprintln(w, "NOTE: self-contained loopback smoke run; not a competitive benchmark.") + } + for _, p := range r.Phases { + fmt.Fprintf(w, "\n[%s] %d ops in %.2fs = %.0f ops/s\n", p.Phase, p.Ops, p.Seconds, p.Throughput) + fmt.Fprintf(w, " %-8s %10s %10s %10s %10s %10s %10s %7s %7s %7s\n", + "op", "count", "ops/s", "p50(us)", "p95(us)", "p99(us)", "max(us)", "errors", "misses", "confl") + for _, o := range p.PerOp { + fmt.Fprintf(w, " %-8s %10d %10.0f %10.1f %10.1f %10.1f %10.1f %7d %7d %7d\n", + o.Op, o.Count, o.Throughput, o.P50Us, o.P95Us, o.P99Us, o.MaxUs, o.Errors, o.Misses, o.Conflicts) + } + } +} diff --git a/cmd/beastbench/runner.go b/cmd/beastbench/runner.go new file mode 100644 index 0000000..69cd245 --- /dev/null +++ b/cmd/beastbench/runner.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "math" + "math/rand" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Config holds everything that determines a run. +type Config struct { + Workload string `json:"workload"` + Dist string `json:"distribution"` + ZipfTheta float64 `json:"zipf_theta,omitempty"` + Records uint64 `json:"records"` + Ops uint64 `json:"ops"` + WarmupOps uint64 `json:"warmup_ops"` + Threads int `json:"threads"` + ValueSize int `json:"value_size"` + MaxScanLen int `json:"max_scan_len"` + Seed int64 `json:"seed"` + Timeout time.Duration `json:"-"` +} + +// OpStats is the per-operation result. +type OpStats struct { + Op string `json:"op"` + Count uint64 `json:"count"` + Errors uint64 `json:"errors"` + Misses uint64 `json:"misses,omitempty"` // reads/scans that found nothing + Conflicts uint64 `json:"conflicts,omitempty"` // conditional writes rejected (rmw) + Throughput float64 `json:"ops_per_sec"` + MeanUs float64 `json:"mean_us"` + P50Us float64 `json:"p50_us"` + P95Us float64 `json:"p95_us"` + P99Us float64 `json:"p99_us"` + MaxUs float64 `json:"max_us"` +} + +// PhaseResult summarises one phase. +type PhaseResult struct { + Phase string `json:"phase"` + Seconds float64 `json:"seconds"` + Ops uint64 `json:"ops"` + Throughput float64 `json:"ops_per_sec"` + PerOp []OpStats `json:"per_op"` +} + +func us(d time.Duration) float64 { return float64(d) / float64(time.Microsecond) } + +func recordKey(idx uint64) uint64 { return idx + 1 } + +// makeValue deterministically fills a value for (idx, version). +func makeValue(buf []byte, idx uint64, version uint64) { + x := idx*0x9E3779B97F4A7C15 + version + 1 + for i := 0; i < len(buf); i += 8 { + x ^= x << 13 + x ^= x >> 7 + x ^= x << 17 + var tmp [8]byte + binary.LittleEndian.PutUint64(tmp[:], x) + copy(buf[i:], tmp[:]) + } +} + +type counters struct { + hist [numOps]*Histogram + errs [numOps]uint64 + misses [numOps]uint64 + conflicts [numOps]uint64 +} + +func newCounters() *counters { + c := &counters{} + for i := range c.hist { + c.hist[i] = NewHistogram() + } + return c +} + +// Load inserts cfg.Records records with the given parallelism. +func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + var firstErr atomic.Value + start := time.Now() + per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + wg.Add(1) + go func(t int) { + defer wg.Done() + c := cs[t] + client := cl[t%len(cl)] + buf := make([]byte, cfg.ValueSize) + lo := uint64(t) * per + hi := lo + per + if hi > cfg.Records { + hi = cfg.Records + } + for i := lo; i < hi; i++ { + if ctx.Err() != nil { + return + } + makeValue(buf, i, 0) + t0 := time.Now() + _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) + c.hist[OpInsert].Record(time.Since(t0)) + if err != nil { + c.errs[OpInsert]++ + firstErr.CompareAndSwap(nil, err) + return + } + } + }(t) + } + wg.Wait() + el := time.Since(start) + res := summarize("load", el, cs) + if e, ok := firstErr.Load().(error); ok { + return res, fmt.Errorf("load failed: %w", e) + } + return res, ctx.Err() +} + +// callPut returns conflict=true when a conditional write was rejected (ABORTED). +func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) + if err != nil { + if status.Code(err) == codes.Aborted { + return true, nil + } + return false, err + } + return false, nil +} + +// Run executes warmup (unrecorded) then the measured run phase. +func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + w, err := lookupWorkload(cfg.Workload) + if err != nil { + return PhaseResult{}, err + } + frontier := &atomic.Uint64{} + frontier.Store(cfg.Records) + + var chooser KeyChooser + switch { + case w.Latest: + chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} + case cfg.Dist == "uniform": + chooser = Uniform{N: cfg.Records} + default: + chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) + } + + phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + start := time.Now() + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + n := total / uint64(cfg.Threads) + if uint64(t) < total%uint64(cfg.Threads) { + n++ + } + wg.Add(1) + go func(t int, n uint64) { + defer wg.Done() + r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) + worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) + }(t, n) + } + wg.Wait() + return cs, time.Since(start) + } + + if cfg.WarmupOps > 0 { + phase(cfg.WarmupOps, 1_000_003, false) + } + cs, el := phase(cfg.Ops, 0, true) + return summarize("run", el, cs), ctx.Err() +} + +func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, + ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { + buf := make([]byte, cfg.ValueSize) + for i := uint64(0); i < n && ctx.Err() == nil; i++ { + op := w.pick(r.Float64()) + t0 := time.Now() + var err error + var miss, conflict bool + switch op { + case OpRead: + idx := ch.Next(r) + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) + miss = err == nil && !found + case OpUpdate: + idx := ch.Next(r) + r.Read(buf) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpInsert: + idx := frontier.Add(1) - 1 + makeValue(buf, idx, 0) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpScan: + idx := ch.Next(r) + ln := 1 + r.Intn(cfg.MaxScanLen) + var got int + got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) + miss = err == nil && got == 0 + case OpRMW: + idx := ch.Next(r) + var old []byte + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) + if err == nil && !found { + miss = true + break + } + if err == nil { + nv := append([]byte(nil), old...) + if len(nv) > 0 { + nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) + } + conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ + Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) + } + } + if ctx.Err() != nil { + return + } + if !record { + continue + } + c.hist[op].Record(time.Since(t0)) + if err != nil { + c.errs[op]++ + } + if miss { + c.misses[op]++ + } + if conflict { + c.conflicts[op]++ + } + } +} + +func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) + if err != nil { + return false, err + } + if out != nil { + *out = resp.Value + } + return resp.Found, nil +} + +func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) + if err != nil { + return 0, err + } + n := 0 + for { + _, err := st.Recv() + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + n++ + } +} + +func summarize(name string, el time.Duration, cs []*counters) PhaseResult { + res := PhaseResult{Phase: name, Seconds: el.Seconds()} + for op := OpType(0); op < numOps; op++ { + h := NewHistogram() + var errs, miss, conf uint64 + for _, c := range cs { + h.Merge(c.hist[op]) + errs += c.errs[op] + miss += c.misses[op] + conf += c.conflicts[op] + } + if h.Count() == 0 { + continue + } + res.Ops += h.Count() + res.PerOp = append(res.PerOp, OpStats{ + Op: op.String(), Count: h.Count(), Errors: errs, Misses: miss, Conflicts: conf, + Throughput: float64(h.Count()) / el.Seconds(), + MeanUs: us(h.Mean()), P50Us: us(h.Quantile(.50)), P95Us: us(h.Quantile(.95)), + P99Us: us(h.Quantile(.99)), MaxUs: us(h.Max()), + }) + } + res.Throughput = float64(res.Ops) / el.Seconds() + return res +} diff --git a/cmd/beastbench/selfhost.go b/cmd/beastbench/selfhost.go new file mode 100644 index 0000000..55e35fd --- /dev/null +++ b/cmd/beastbench/selfhost.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "net" + "os" + "path/filepath" + + "github.com/ChromaBeast/beastdb/internal/api" +) + +// StartSelfHosted runs an in-process engine and gRPC server on a temp dir +// listening on loopback. The returned stop func removes everything. +func StartSelfHosted(poolSize int) (addr string, stop func(), err error) { + dir, err := os.MkdirTemp("", "beastbench-*") + if err != nil { + return "", nil, err + } + eng, err := api.NewEngine(filepath.Join(dir, "beast.bin"), filepath.Join(dir, "beast.wal"), poolSize) + if err != nil { + os.RemoveAll(dir) + return "", nil, fmt.Errorf("start engine: %w", err) + } + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + eng.Close() + os.RemoveAll(dir) + return "", nil, err + } + srv := api.NewGRPCServer(eng, api.MessageSizeServerOptions()...) + done := make(chan struct{}) + go func() { _ = srv.Serve(lis); close(done) }() + return lis.Addr().String(), func() { + srv.Stop() + <-done + eng.Close() + os.RemoveAll(dir) + }, nil +} diff --git a/cmd/beastbench/workload.go b/cmd/beastbench/workload.go new file mode 100644 index 0000000..26ec9ed --- /dev/null +++ b/cmd/beastbench/workload.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" +) + +// OpType is a benchmarked operation kind. +type OpType int + +const ( + OpRead OpType = iota + OpUpdate + OpInsert + OpScan + OpRMW + numOps +) + +var opNames = [numOps]string{"read", "update", "insert", "scan", "rmw"} + +func (o OpType) String() string { return opNames[o] } + +// Workload is a YCSB-style operation mix; proportions sum to 1. +type Workload struct { + Name string + Desc string + Mix [numOps]float64 + Latest bool // read-latest key distribution (D) + Uniform bool // workload forces a distribution regardless of flag (none today) +} + +var workloads = map[string]Workload{ + "A": {Name: "A", Desc: "50% read / 50% update", Mix: mix(OpRead, .5, OpUpdate, .5)}, + "B": {Name: "B", Desc: "95% read / 5% update", Mix: mix(OpRead, .95, OpUpdate, .05)}, + "C": {Name: "C", Desc: "100% read", Mix: mix(OpRead, 1.0)}, + "D": {Name: "D", Desc: "95% read-latest / 5% insert", Mix: mix(OpRead, .95, OpInsert, .05), Latest: true}, + "E": {Name: "E", Desc: "95% short scan / 5% insert", Mix: mix(OpScan, .95, OpInsert, .05)}, + "F": {Name: "F", Desc: "50% read / 50% read-modify-write (conditional put)", Mix: mix(OpRead, .5, OpRMW, .5)}, +} + +func mix(kv ...any) [numOps]float64 { + var m [numOps]float64 + for i := 0; i < len(kv); i += 2 { + m[kv[i].(OpType)] = kv[i+1].(float64) + } + return m +} + +func lookupWorkload(name string) (Workload, error) { + w, ok := workloads[strings.ToUpper(name)] + if !ok { + return w, fmt.Errorf("unknown workload %q (want A-F)", name) + } + return w, nil +} + +// pick maps u in [0,1) to an operation. +func (w Workload) pick(u float64) OpType { + var acc float64 + last := OpRead + for op := OpType(0); op < numOps; op++ { + if w.Mix[op] == 0 { + continue + } + last = op + acc += w.Mix[op] + if u < acc { + return op + } + } + return last +} diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 0000000..231bf37 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,172 @@ +# Benchmarking BeastDB with `beastbench` + +`cmd/beastbench` is a reproducible, YCSB-style load generator that drives a +BeastDB server over gRPC. This document explains how to run it and how to run +*equivalent* workloads against Redis, Pebble and Badger so that a comparison is +fair. It deliberately contains no results: numbers depend on your hardware, +settings and dataset, and only numbers you measured yourself are meaningful. + +## Running beastbench + +``` +go build -o beastbench ./cmd/beastbench + +# against a running server (flags follow cmd/beastctl: token and TLS) +./beastbench --addr 127.0.0.1:50051 --token "$BEASTDB_API_TOKEN" \ + --workload A --dist zipfian --records 10000000 --ops 5000000 \ + --threads 64 --value-size 1024 --warmup 200000 --seed 42 --json > a.json + +# TLS / mTLS +./beastbench --addr db:50051 --tls-ca ca.pem --tls-cert client.pem --tls-key client-key.pem ... + +# quick smoke run: in-process engine + gRPC on a temp dir (loopback, no auth) +./beastbench --self-contained --workload B --records 5000 --ops 5000 --threads 4 +``` + +Flags: `--addr --token --tls-ca --tls-cert --tls-key --tls-server-name --conns` +(connection), `--workload A-F`, `--dist zipfian|uniform`, `--zipf-theta`, +`--records --ops --warmup --threads --value-size --max-scan-len --seed`, +`--json`, `--skip-load`, `--load-only`, `--self-contained --pool-size`. +`--token` defaults to `$BEASTDB_API_TOKEN`, the TLS flags to `$BEASTDB_TLS_*`. + +### Phases + +1. **Load**: inserts `--records` records (keys 1..N, deterministic values of + `--value-size` bytes) in parallel. Reported separately. +2. **Warmup** (optional): `--warmup` unmeasured operations of the chosen mix. +3. **Run**: `--ops` measured operations split across `--threads` workers. + +Each worker has its own RNG seeded from `--seed`, so a given +(seed, threads, ops) produces the same operation and key sequence. Timing and +server behaviour still vary run to run; repeat runs and report the spread. + +### Workloads + +| Name | Mix | Keys | +|---|---|---| +| A | 50% read, 50% update | zipfian / uniform | +| B | 95% read, 5% update | zipfian / uniform | +| C | 100% read | zipfian / uniform | +| D | 95% read, 5% insert | read-latest (zipfian offset from newest insert) | +| E | 95% scan (length uniform 1..`--max-scan-len`), 5% insert | zipfian / uniform start key | +| F | 50% read, 50% read-modify-write | zipfian / uniform | + +Workload F's read-modify-write is `Get` followed by `Put` with a +`VALUE_EQUALS` condition on the value read. A rejected condition (ABORTED) is +counted as a conflict, not an error, and is included in the `rmw` latency. +Latency for `rmw` covers both RPCs. + +Notes on semantics: + +- Zipfian uses the YCSB generator (theta 0.99 by default). Keys are *not* + hash-scrambled, so the hottest keys are the lowest ones; this keeps scans + contiguous but concentrates the hot set in one region of the B+ tree. +- Workloads D/E allocate new keys above the loaded range. A D read may hit a key + whose insert is still in flight; that is reported as a `miss`. +- Percentiles come from a log-linear histogram (128 sub-buckets per power of + two, under 1% error); max is exact. Latency is client-observed per call, + including gRPC and network time. There is no coordinated-omission correction: + workers are closed-loop, so a stalled server lowers offered load. Say so when + you report tail latencies. + +### Output + +Text by default; `--json` emits config, host info (Go version, OS, CPUs, +self-contained flag) and per-phase, per-operation count, errors, misses, +conflicts, ops/s, mean, p50, p95, p99 and max in microseconds. Keep the JSON +with any number you quote. + +## Fair comparison against Redis, Pebble and Badger + +A comparison is only as honest as its weakest control. Fix all of the following +and record them next to results. + +### Environment + +- **Same machine, same disk, run sequentially**, never concurrently. Record CPU + model, core count, RAM, disk model and filesystem, OS and kernel, Go version + and every system's version/commit. +- Disable CPU frequency scaling surprises (performance governor), keep the + machine otherwise idle, and use a fresh data directory per run on the same + device. +- **Dataset larger than RAM.** If the dataset fits in the page cache you are + benchmarking memory, not the storage engine. Choose `records x (value size + + per-key overhead)` at least 2-4x physical RAM (or constrain RAM with a + cgroup/container memory limit and state it). Check with `free`/`vmstat` that + the page cache did not absorb the working set, and drop caches between + phases if you intend a cold start (`sync; echo 3 > /proc/sys/vm/drop_caches`). +- State the **cache budgets** explicitly and make them comparable: BeastDB + `--pool-size` (4 KiB frames), Pebble block cache, Badger block/index cache, + Redis `maxmemory`. Do not leave one system on defaults while tuning another. +- Same key count, key size (BeastDB keys are `uint64`; use 8-byte keys + elsewhere, big-endian so ordering matches), value size, and value contents. + Values from `beastbench` are incompressible-ish pseudo-random bytes; disable + or equalise compression (Pebble/Badger default compression off or on must + match; Redis stores raw). + +### Durability: the biggest source of unfair numbers + +Compare like with like. State what each system guarantees when an +acknowledged write is followed by a power loss. + +| System | Per-write durable (fsync on ack) | Relaxed | +|---|---|---| +| BeastDB | Each write is acknowledged only after the WAL is synced (`docs/v1-semantics.md`); benchmark as shipped | no relaxed mode is documented; document any change you make | +| Redis | `appendonly yes`, `appendfsync always` | `appendfsync everysec`, or no AOF (RDB only) | +| Pebble | `pebble.Sync` on every write (`WriteOptions{Sync: true}`) | `pebble.NoSync` | +| Badger | `SyncWrites: true` | `SyncWrites: false` (default) | + +Run the per-write-durable configuration for all systems as the primary +comparison, and report any relaxed configuration as a separate, clearly +labelled table. A durable system compared against a non-durable one tells you +nothing. Also note that group commit/batching behaviour differs and is part of +what is being measured when many threads write concurrently. + +### Matching the access path + +- `beastbench` measures a *networked* client-server round trip over gRPC. Redis + has a network protocol too (use `redis-benchmark` or a small RESP client, + no pipelining unless you also pipeline everywhere). Pebble and Badger are + embedded libraries with no network hop; comparing them to a gRPC call + mostly measures the RPC. Either (a) wrap them in an equivalent gRPC/TCP + service and run the same `--threads`, or (b) benchmark BeastDB's engine + in-process (`go test ./internal/api -bench`) and compare to the embedded + libraries in-process. Do not mix the two. +- Use the same concurrency (`--threads`) and report results at several + concurrency levels, not one. +- Same operation mix, key distribution, warmup, run length, and seed-derived + key sequence. Implement the equivalent of each workload in the other + system's harness: A/B/C/D as get/put, E as an iterator seeking to the start + key and reading up to N entries (Redis: a sorted-set or key range scheme and + say which), F as get then a conditional/transactional write (Redis + `WATCH`/`MULTI` or a Lua CAS; Pebble/Badger: a transaction or CAS loop). + Document any place the semantics differ. +- Load the dataset the same way (same ordering and parallelism) and let + background work finish (compaction in Pebble/Badger, checkpoint in BeastDB, + `BGSAVE`/AOF rewrite in Redis) before the measured phase, or measure it + deliberately and say so. + +### Existing harnesses + +You may use YCSB itself (Redis binding exists; Pebble/Badger need a custom or +community binding) or `redis-benchmark`. If you do, map its parameters to the +above (recordcount, operationcount, fieldcount=1, fieldlength, request +distribution, threadcount) and note that YCSB's latency measurement differs +from `beastbench`'s. + +### Reporting + +- Publish the exact command lines, config files, versions, hardware and the raw + JSON/CSV outputs. +- Report several repetitions (at least 5) with median and range, not a single + best run. +- Report p50/p95/p99/max latency, not only throughput, plus error and conflict + counts. +- State what you did *not* control. Do not generalise from one workload, + dataset size or machine; do not claim one system is faster in general. + +### Smoke runs are not benchmarks + +`--self-contained` runs client and server in one process on loopback, with +whatever disk the temp dir lands on and no auth or TLS. Use it to validate the +harness and catch regressions in your own environment only. From c9d5113fa0dbe157de8df47bafe967d5edcc7aa6 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:23:00 +0530 Subject: [PATCH 07/16] feat(obs): Prometheus /metrics endpoint with gRPC/HTTP request metrics Hand-written text exposition (no new deps). Authenticated by admin session or bearer token (--api-token / BEASTDB_METRICS_TOKEN); no loopback bypass. Adds buffer-pool hit/miss counters and checkpoint timestamps, gRPC metrics interceptors, runbook docs with scrape config and alert rules. Co-Authored-By: Claude Sonnet 5.5 --- cmd/server/main.go | 6 +- cmd/server/web.go | 4 +- docs/runbooks/operator-guide.md | 74 +++++++++++ internal/api/engine.go | 4 + internal/api/engine_checkpoint.go | 8 +- internal/api/engine_stats.go | 32 +++-- internal/api/grpc_metrics.go | 35 ++++++ internal/api/grpc_metrics_test.go | 42 +++++++ internal/metrics/diskfree_other.go | 6 + internal/metrics/diskfree_unix.go | 15 +++ internal/metrics/metrics.go | 175 +++++++++++++++++++++++++++ internal/storage/buffer_pool.go | 11 ++ internal/web/handler/metrics.go | 155 ++++++++++++++++++++++++ internal/web/handler/metrics_test.go | 164 +++++++++++++++++++++++++ internal/web/server.go | 22 +++- 15 files changed, 738 insertions(+), 15 deletions(-) create mode 100644 internal/api/grpc_metrics.go create mode 100644 internal/api/grpc_metrics_test.go create mode 100644 internal/metrics/diskfree_other.go create mode 100644 internal/metrics/diskfree_unix.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/web/handler/metrics.go create mode 100644 internal/web/handler/metrics_test.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 624c994..8e34914 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -74,6 +74,8 @@ func main() { tokenStore := initTokenStore(engine) grpcOpts := api.MessageSizeServerOptions() + // Metrics interceptors run first so they also observe auth rejections. + grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(api.MetricsUnaryInterceptor()), grpc.ChainStreamInterceptor(api.MetricsStreamInterceptor())) if *tlsCert != "" && *tlsKey != "" { creds, err := loadServerTLS(*tlsCert, *tlsKey, *tlsCA) if err != nil { @@ -84,7 +86,7 @@ func main() { } if *apiToken != "" || tokenStore != nil { authInterceptor := api.NewAuthInterceptor(*apiToken, tokenStore) - grpcOpts = append(grpcOpts, grpc.UnaryInterceptor(authInterceptor.Unary()), grpc.StreamInterceptor(authInterceptor.Stream())) + grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(authInterceptor.Unary()), grpc.ChainStreamInterceptor(authInterceptor.Stream())) if *apiToken != "" { log.Printf("🔐 gRPC service protected with Bearer token authentication.") } @@ -145,7 +147,7 @@ func main() { } } - stopWeb := startWebServer(*webAddr, *dataDir, *role, *adminPassword, engine, partitions, tokenStore) + stopWeb := startWebServer(*webAddr, *dataDir, *role, *adminPassword, engine, partitions, tokenStore, *apiToken, os.Getenv("BEASTDB_METRICS_TOKEN")) defer stopWeb() lis, err := net.Listen("tcp", grpcBind) diff --git a/cmd/server/web.go b/cmd/server/web.go index c15d202..62fdb8d 100644 --- a/cmd/server/web.go +++ b/cmd/server/web.go @@ -10,7 +10,7 @@ import ( ) // startWebServer boots the HTTP admin console and seeds or synchronizes the admin credentials. -func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.Engine, partitions []handler.PartitionEntry, tokenStore *auth.TokenStore) func() { +func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.Engine, partitions []handler.PartitionEntry, tokenStore *auth.TokenStore, metricsTokens ...string) func() { if webAddr == "" { return func() {} } @@ -20,7 +20,7 @@ func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.En log.Fatalf("Failed to initialize session secret: %v", secretErr) } - webSrv, webErr := web.NewServer(webAddr, engine, secret, role, Version, partitions, tokenStore) + webSrv, webErr := web.NewServerWithOptions(webAddr, engine, secret, role, Version, partitions, web.Options{DataDir: dataDir, MetricsTokens: metricsTokens}, tokenStore) if webErr != nil { log.Fatalf("Failed to create web server: %v", webErr) } diff --git a/docs/runbooks/operator-guide.md b/docs/runbooks/operator-guide.md index afabf9e..61da4e7 100644 --- a/docs/runbooks/operator-guide.md +++ b/docs/runbooks/operator-guide.md @@ -64,6 +64,80 @@ The embedded HTTP server exposes unauthenticated health endpoints for Kubernetes --- +### Prometheus Metrics (`GET /metrics`) + +The web server exposes Prometheus text exposition (format 0.0.4) at `/metrics` on `--web-addr`. It is **never anonymous**: there is no loopback exemption, because a reverse proxy or sidecar on the same host makes every remote peer look like `127.0.0.1`. A request is accepted only with one of: + +- `Authorization: Bearer ` equal to the gRPC `--api-token`, or to `BEASTDB_METRICS_TOKEN` (preferred: a dedicated scrape-only secret), or +- an admin web session cookie. + +With neither token configured, only admin sessions work (Prometheus cannot scrape). Set `BEASTDB_METRICS_TOKEN` and keep the web listener on loopback or a private network/TLS-terminating proxy, since the bearer token travels in cleartext over plain HTTP. + +| Metric | Meaning | +|---|---| +| `beastdb_requests_total{protocol,method,code}` | gRPC (code name) and HTTP (status, route pattern) request counts; errors are non-`OK` / 4xx-5xx | +| `beastdb_request_duration_seconds` (histogram) | latency per protocol/method (streams: total stream duration) | +| `beastdb_grpc_requests_in_flight` | executing gRPC calls | +| `beastdb_wal_size_bytes`, `beastdb_wal_current_lsn` | WAL file size and highest LSN | +| `beastdb_checkpoint_last_lsn`, `beastdb_checkpoint_age_seconds` | last checkpoint by this process (series absent until the first checkpoint after start) | +| `beastdb_buffer_pool_{size,cached,pinned,dirty}_pages`, `beastdb_buffer_pool_hits_total` / `_misses_total` / `_hit_ratio` | buffer pool state; ratio is lifetime, use `rate()` of the counters for recent behavior | +| `beastdb_data_file_size_bytes`, `beastdb_disk_free_bytes` | data file size, free space on the data dir filesystem (unix only) | +| `beastdb_read_only` | 1 when writes are rejected (follower or fail-stop after a failed apply) | +| `beastdb_replica_applied_lsn` | followers only: last applied LSN | +| `beastdb_build_info{version,role}` | always 1 | + +Replica lag is computed in PromQL: `max(beastdb_wal_current_lsn{role="leader"}) - beastdb_replica_applied_lsn` (add a `role` label via relabeling, or key on `instance`). + +Sample scrape config (`prometheus.yml`): + +```yaml +scrape_configs: + - job_name: beastdb + metrics_path: /metrics + scrape_interval: 15s + authorization: + type: Bearer + credentials_file: /etc/prometheus/beastdb.token # contains BEASTDB_METRICS_TOKEN + static_configs: + - targets: ["leader.internal:8080"] + labels: { role: leader } + - targets: ["replica1.internal:8080", "replica2.internal:8080"] + labels: { role: follower } +``` + +Example alert rules: + +```yaml +groups: + - name: beastdb + rules: + - alert: BeastDBDown + expr: up{job="beastdb"} == 0 + for: 2m + - alert: BeastDBHighErrorRate + expr: | + sum by (instance) (rate(beastdb_requests_total{protocol="grpc",code!~"OK|NotFound|AlreadyExists|FailedPrecondition"}[5m])) + / sum by (instance) (rate(beastdb_requests_total{protocol="grpc"}[5m])) > 0.05 + for: 5m + - alert: BeastDBHighP99Latency + expr: histogram_quantile(0.99, sum by (instance, le) (rate(beastdb_request_duration_seconds_bucket{protocol="grpc",method!~".*Sync.*|.*Scan.*"}[5m]))) > 0.25 + for: 10m + - alert: BeastDBWALGrowing + expr: beastdb_wal_size_bytes > 1073741824 or beastdb_checkpoint_age_seconds > 600 + for: 10m + - alert: BeastDBDiskLow + expr: beastdb_disk_free_bytes < 5 * 1024^3 + for: 5m + - alert: BeastDBReplicaLagging + expr: max(beastdb_wal_current_lsn{role="leader"}) - on() group_right() beastdb_replica_applied_lsn > 10000 + for: 5m + - alert: BeastDBFailStop + expr: beastdb_read_only{role="leader"} == 1 + for: 1m +``` + +--- + ## 5. Replica Promotion & Failover BeastDB V1 does not employ automatic distributed consensus. In the event of leader loss: diff --git a/internal/api/engine.go b/internal/api/engine.go index 19c13dd..decd4a5 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sync" + "sync/atomic" "github.com/ChromaBeast/beastdb/internal/index" "github.com/ChromaBeast/beastdb/internal/storage" @@ -24,6 +25,9 @@ type Engine struct { failed error // set by halt; blocks writes until restart reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree mu sync.RWMutex + + lastCkptLSN atomic.Uint64 // LSN at the last successful checkpoint (metrics) + lastCkptUnix atomic.Int64 // unix seconds of the last successful checkpoint } // NewEngine initializes the database engine components with recovery support. diff --git a/internal/api/engine_checkpoint.go b/internal/api/engine_checkpoint.go index 91036ba..8a2e085 100644 --- a/internal/api/engine_checkpoint.go +++ b/internal/api/engine_checkpoint.go @@ -54,7 +54,13 @@ func (e *Engine) Checkpoint() error { if err := e.disk.Sync(); err != nil { return fmt.Errorf("checkpoint sync data file: %w", err) } - return e.wal.Checkpoint() + ckptLSN := e.wal.CurrentLSN() + if err := e.wal.Checkpoint(); err != nil { + return err + } + e.lastCkptLSN.Store(ckptLSN) + e.lastCkptUnix.Store(time.Now().Unix()) + return nil } // StartAutoCheckpoint checkpoints whenever the WAL exceeds maxWALBytes, polling every diff --git a/internal/api/engine_stats.go b/internal/api/engine_stats.go index c4aa1f2..f776a03 100644 --- a/internal/api/engine_stats.go +++ b/internal/api/engine_stats.go @@ -15,6 +15,13 @@ type StorageMetrics struct { WALSizeBytes int64 `json:"walSizeBytes"` CurrentLSN uint64 `json:"currentLSN"` ActiveDataPage uint64 `json:"activeDataPage"` + + BufferHits uint64 `json:"bufferHits"` + BufferMisses uint64 `json:"bufferMisses"` + LastCheckpointLSN uint64 `json:"lastCheckpointLSN"` + // LastCheckpointUnix is 0 until a checkpoint completes in this process. + LastCheckpointUnix int64 `json:"lastCheckpointUnix"` + ReadOnly bool `json:"readOnly"` } // StorageMetrics collects live runtime storage indicators. @@ -25,15 +32,22 @@ func (e *Engine) StorageMetrics() StorageMetrics { numPages := e.disk.NumPages() poolSize, cached, pinned, dirty := e.bpm.Stats() + hits, misses := e.bpm.HitMiss() + return StorageMetrics{ - TotalPages: numPages, - DiskSizeBytes: numPages * storage.PageSize, - PoolSize: poolSize, - CachedPages: cached, - PinnedPages: pinned, - DirtyPages: dirty, - WALSizeBytes: e.wal.FileSize(), - CurrentLSN: e.wal.CurrentLSN(), - ActiveDataPage: e.activeDataPage, + BufferHits: hits, + BufferMisses: misses, + LastCheckpointLSN: e.lastCkptLSN.Load(), + LastCheckpointUnix: e.lastCkptUnix.Load(), + ReadOnly: e.readOnly, + TotalPages: numPages, + DiskSizeBytes: numPages * storage.PageSize, + PoolSize: poolSize, + CachedPages: cached, + PinnedPages: pinned, + DirtyPages: dirty, + WALSizeBytes: e.wal.FileSize(), + CurrentLSN: e.wal.CurrentLSN(), + ActiveDataPage: e.activeDataPage, } } diff --git a/internal/api/grpc_metrics.go b/internal/api/grpc_metrics.go new file mode 100644 index 0000000..b501bc5 --- /dev/null +++ b/internal/api/grpc_metrics.go @@ -0,0 +1,35 @@ +package api + +import ( + "context" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/status" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +// MetricsUnaryInterceptor records count, status code and latency of unary RPCs. +func MetricsUnaryInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler) (any, error) { + metrics.InFlight.Add(1) + start := time.Now() + resp, err := h(ctx, req) + metrics.InFlight.Add(-1) + metrics.Default.Observe("grpc", info.FullMethod, status.Code(err).String(), time.Since(start)) + return resp, err + } +} + +// MetricsStreamInterceptor records count, status code and total duration of streaming RPCs. +func MetricsStreamInterceptor() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, h grpc.StreamHandler) error { + metrics.InFlight.Add(1) + start := time.Now() + err := h(srv, ss) + metrics.InFlight.Add(-1) + metrics.Default.Observe("grpc", info.FullMethod, status.Code(err).String(), time.Since(start)) + return err + } +} diff --git a/internal/api/grpc_metrics_test.go b/internal/api/grpc_metrics_test.go new file mode 100644 index 0000000..a582b97 --- /dev/null +++ b/internal/api/grpc_metrics_test.go @@ -0,0 +1,42 @@ +package api + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +func TestMetricsInterceptors(t *testing.T) { + u := MetricsUnaryInterceptor() + _, err := u(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/t.S/Unary"}, + func(context.Context, any) (any, error) { return nil, status.Error(codes.PermissionDenied, "no") }) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("error not propagated: %v", err) + } + s := MetricsStreamInterceptor() + if err := s(nil, nil, &grpc.StreamServerInfo{FullMethod: "/t.S/Stream"}, + func(any, grpc.ServerStream) error { return nil }); err != nil { + t.Fatal(err) + } + var b strings.Builder + metrics.Default.WriteRequests(metrics.NewWriter(&b)) + out := b.String() + for _, want := range []string{ + `beastdb_requests_total{protocol="grpc",method="/t.S/Unary",code="PermissionDenied"} 1`, + `beastdb_requests_total{protocol="grpc",method="/t.S/Stream",code="OK"} 1`, + `beastdb_request_duration_seconds_count{protocol="grpc",method="/t.S/Unary"} 1`, + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q", want) + } + } + if metrics.InFlight.Load() != 0 { + t.Fatalf("in-flight leaked: %d", metrics.InFlight.Load()) + } +} diff --git a/internal/metrics/diskfree_other.go b/internal/metrics/diskfree_other.go new file mode 100644 index 0000000..5e22402 --- /dev/null +++ b/internal/metrics/diskfree_other.go @@ -0,0 +1,6 @@ +//go:build !unix + +package metrics + +// DiskFreeBytes is unsupported on this platform. +func DiskFreeBytes(string) (uint64, bool) { return 0, false } diff --git a/internal/metrics/diskfree_unix.go b/internal/metrics/diskfree_unix.go new file mode 100644 index 0000000..17c4433 --- /dev/null +++ b/internal/metrics/diskfree_unix.go @@ -0,0 +1,15 @@ +//go:build unix + +package metrics + +import "syscall" + +// DiskFreeBytes returns bytes available to unprivileged users on the +// filesystem holding path. +func DiskFreeBytes(path string) (uint64, bool) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, false + } + return uint64(st.Bavail) * uint64(st.Bsize), true +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..77646f4 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,175 @@ +// Package metrics is a tiny dependency-free Prometheus text-format (0.0.4) +// exposition layer: request counters, latency histograms and gauge helpers. +package metrics + +import ( + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// DefaultBuckets are latency histogram upper bounds in seconds. +var DefaultBuckets = []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10} + +type hist struct { + counts []uint64 // per-bucket (non-cumulative), len = len(bounds)+1 (+Inf) + sum float64 + n uint64 +} + +type reqKey struct{ proto, method, code string } + +// RequestMetrics records request counts (by protocol, method, status code) +// and latency histograms (by protocol, method). +type RequestMetrics struct { + mu sync.Mutex + count map[reqKey]uint64 + hists map[[2]string]*hist +} + +// NewRequestMetrics creates an empty collector. +func NewRequestMetrics() *RequestMetrics { + return &RequestMetrics{count: map[reqKey]uint64{}, hists: map[[2]string]*hist{}} +} + +// Observe records one finished request. code is a short status string +// (gRPC code name or HTTP status number). +func (r *RequestMetrics) Observe(proto, method, code string, d time.Duration) { + r.mu.Lock() + defer r.mu.Unlock() + r.count[reqKey{proto, method, code}]++ + hk := [2]string{proto, method} + h := r.hists[hk] + if h == nil { + h = &hist{counts: make([]uint64, len(DefaultBuckets)+1)} + r.hists[hk] = h + } + s := d.Seconds() + i := sort.SearchFloat64s(DefaultBuckets, s) // first bound >= s + h.counts[i]++ + h.sum += s + h.n++ +} + +// Default is the process-wide request collector shared by the gRPC +// interceptors and the HTTP middleware. +var Default = NewRequestMetrics() + +// InFlight tracks currently executing gRPC requests. +var InFlight atomic.Int64 + +// Writer emits Prometheus text exposition. +type Writer struct { + w io.Writer + seen map[string]bool +} + +// NewWriter wraps w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w, seen: map[string]bool{}} } + +func (w *Writer) header(name, help, typ string) { + if w.seen[name] { + return + } + w.seen[name] = true + fmt.Fprintf(w.w, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ) +} + +// Escape escapes a label value. +func Escape(v string) string { + return strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`).Replace(v) +} + +func labels(kv []string) string { + if len(kv) == 0 { + return "" + } + parts := make([]string, 0, len(kv)/2) + for i := 0; i+1 < len(kv); i += 2 { + parts = append(parts, kv[i]+`="`+Escape(kv[i+1])+`"`) + } + return "{" + strings.Join(parts, ",") + "}" +} + +func fmtFloat(v float64) string { + switch { + case math.IsInf(v, 1): + return "+Inf" + case math.IsInf(v, -1): + return "-Inf" + case math.IsNaN(v): + return "NaN" + } + return strconv.FormatFloat(v, 'g', -1, 64) +} + +// Gauge writes a gauge sample. kv are alternating label names/values. +func (w *Writer) Gauge(name, help string, v float64, kv ...string) { + w.header(name, help, "gauge") + fmt.Fprintf(w.w, "%s%s %s\n", name, labels(kv), fmtFloat(v)) +} + +// Counter writes a counter sample. +func (w *Writer) Counter(name, help string, v float64, kv ...string) { + w.header(name, help, "counter") + fmt.Fprintf(w.w, "%s%s %s\n", name, labels(kv), fmtFloat(v)) +} + +// WriteRequests emits the request counter and latency histogram families. +func (r *RequestMetrics) WriteRequests(w *Writer) { + r.mu.Lock() + keys := make([]reqKey, 0, len(r.count)) + counts := make(map[reqKey]uint64, len(r.count)) + for k, v := range r.count { + keys = append(keys, k) + counts[k] = v + } + hk := make([][2]string, 0, len(r.hists)) + hs := make(map[[2]string]hist, len(r.hists)) + for k, h := range r.hists { + hk = append(hk, k) + hs[k] = hist{counts: append([]uint64(nil), h.counts...), sum: h.sum, n: h.n} + } + r.mu.Unlock() + + sort.Slice(keys, func(i, j int) bool { + a, b := keys[i], keys[j] + if a.proto != b.proto { + return a.proto < b.proto + } + if a.method != b.method { + return a.method < b.method + } + return a.code < b.code + }) + for _, k := range keys { + w.Counter("beastdb_requests_total", "Requests handled, by protocol, method and status code.", + float64(counts[k]), "protocol", k.proto, "method", k.method, "code", k.code) + } + sort.Slice(hk, func(i, j int) bool { + if hk[i][0] != hk[j][0] { + return hk[i][0] < hk[j][0] + } + return hk[i][1] < hk[j][1] + }) + const name = "beastdb_request_duration_seconds" + for _, k := range hk { + h := hs[k] + w.header(name, "Request latency in seconds.", "histogram") + var cum uint64 + for i, b := range DefaultBuckets { + cum += h.counts[i] + fmt.Fprintf(w.w, "%s_bucket%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1], "le", fmtFloat(b)}), cum) + } + cum += h.counts[len(DefaultBuckets)] + fmt.Fprintf(w.w, "%s_bucket%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1], "le", "+Inf"}), cum) + fmt.Fprintf(w.w, "%s_sum%s %s\n", name, labels([]string{"protocol", k[0], "method", k[1]}), fmtFloat(h.sum)) + fmt.Fprintf(w.w, "%s_count%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1]}), h.n) + } +} diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index 19d4a96..6e14a97 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -3,6 +3,7 @@ package storage import ( "errors" "sync" + "sync/atomic" ) var ( @@ -22,6 +23,8 @@ type BufferPoolManager struct { clockHand int mu sync.RWMutex pagePool sync.Pool // Fix #6: reusable 4KB page buffers + + hits, misses atomic.Uint64 // FetchPage cache hit/miss counters } // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. @@ -48,6 +51,7 @@ func (bpm *BufferPoolManager) FetchPage(pageID uint64) (*SlottedPage, error) { frame.Pin() page := frame.Page bpm.mu.RUnlock() + bpm.hits.Add(1) return page, nil } bpm.mu.RUnlock() @@ -59,9 +63,11 @@ func (bpm *BufferPoolManager) FetchPage(pageID uint64) (*SlottedPage, error) { // Double-check: another goroutine may have loaded the page while we waited. if idx, exists := bpm.pageTable[pageID]; exists { bpm.frames[idx].Pin() + bpm.hits.Add(1) return bpm.frames[idx].Page, nil } + bpm.misses.Add(1) return bpm.loadPage(pageID) } @@ -151,6 +157,11 @@ func (bpm *BufferPoolManager) FlushAll() error { return bpm.disk.Sync() } +// HitMiss returns cumulative FetchPage cache hits and misses. +func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { + return bpm.hits.Load(), bpm.misses.Load() +} + // Stats returns operational metrics for the buffer pool. func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { bpm.mu.RLock() diff --git a/internal/web/handler/metrics.go b/internal/web/handler/metrics.go new file mode 100644 index 0000000..af15807 --- /dev/null +++ b/internal/web/handler/metrics.go @@ -0,0 +1,155 @@ +package handler + +import ( + "bytes" + "crypto/subtle" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ChromaBeast/beastdb/internal/metrics" + "github.com/ChromaBeast/beastdb/internal/web/auth" +) + +// MetricsConfig configures the /metrics endpoint. +type MetricsConfig struct { + Role string + Version string + // DataDir is used for the disk-free gauge (omitted when empty). + DataDir string + // Tokens are static bearer tokens accepted for scraping (e.g. the gRPC + // --api-token and BEASTDB_METRICS_TOKEN). Empty entries are ignored. + Tokens []string + // Sessions authenticates admin browser sessions. May be nil. + Sessions *auth.SessionManager +} + +var processStart = time.Now() + +// authorized reports whether r carries an admin session or a configured +// bearer token. There is deliberately no unauthenticated/loopback bypass: a +// reverse proxy on the same host would make every remote peer look like +// loopback. +func (c MetricsConfig) authorized(r *http.Request) bool { + if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") { + got := []byte(strings.TrimSpace(strings.TrimPrefix(h, "Bearer "))) + for _, t := range c.Tokens { + t = strings.TrimSpace(t) + if t != "" && subtle.ConstantTimeCompare(got, []byte(t)) == 1 { + return true + } + } + return false + } + if c.Sessions != nil { + if _, role, err := c.Sessions.ValidateSession(r); err == nil && role == auth.RoleAdmin { + return true + } + } + return false +} + +// MetricsHandler serves Prometheus text exposition (version 0.0.4). +func MetricsHandler(engine EngineReader, cfg MetricsConfig) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + if !cfg.authorized(r) { + w.Header().Set("WWW-Authenticate", `Bearer realm="beastdb-metrics"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + var buf bytes.Buffer + WriteMetrics(&buf, engine, cfg) + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + _, _ = w.Write(buf.Bytes()) + }) +} + +// WriteMetrics renders all metrics. +func WriteMetrics(buf *bytes.Buffer, engine EngineReader, cfg MetricsConfig) { + w := metrics.NewWriter(buf) + w.Gauge("beastdb_build_info", "Build information; value is always 1.", 1, + "version", cfg.Version, "role", cfg.Role) + w.Gauge("beastdb_process_start_time_seconds", "Process start time, unix seconds.", float64(processStart.Unix())) + w.Gauge("beastdb_uptime_seconds", "Seconds since process start.", time.Since(processStart).Seconds()) + + if engine != nil { + m := engine.StorageMetrics() + w.Gauge("beastdb_wal_size_bytes", "Current WAL file size in bytes.", float64(m.WALSizeBytes)) + w.Gauge("beastdb_wal_current_lsn", "Highest LSN written to the WAL.", float64(m.CurrentLSN)) + w.Gauge("beastdb_checkpoint_last_lsn", "LSN of the last checkpoint completed by this process (0 if none).", float64(m.LastCheckpointLSN)) + if m.LastCheckpointUnix > 0 { + w.Gauge("beastdb_checkpoint_last_timestamp_seconds", "Unix time of the last checkpoint.", float64(m.LastCheckpointUnix)) + age := time.Since(time.Unix(m.LastCheckpointUnix, 0)).Seconds() + if age < 0 { + age = 0 + } + w.Gauge("beastdb_checkpoint_age_seconds", "Seconds since the last checkpoint.", age) + } + w.Gauge("beastdb_buffer_pool_size_pages", "Buffer pool capacity in pages.", float64(m.PoolSize)) + w.Gauge("beastdb_buffer_pool_cached_pages", "Pages resident in the buffer pool.", float64(m.CachedPages)) + w.Gauge("beastdb_buffer_pool_pinned_pages", "Pinned buffer pool pages.", float64(m.PinnedPages)) + w.Gauge("beastdb_buffer_pool_dirty_pages", "Dirty buffer pool pages.", float64(m.DirtyPages)) + w.Counter("beastdb_buffer_pool_hits_total", "Buffer pool fetches served from memory.", float64(m.BufferHits)) + w.Counter("beastdb_buffer_pool_misses_total", "Buffer pool fetches that read from disk.", float64(m.BufferMisses)) + if total := m.BufferHits + m.BufferMisses; total > 0 { + w.Gauge("beastdb_buffer_pool_hit_ratio", "Lifetime buffer pool hit ratio (0..1).", float64(m.BufferHits)/float64(total)) + } + w.Gauge("beastdb_data_file_size_bytes", "Data file size in bytes.", float64(m.DiskSizeBytes)) + w.Gauge("beastdb_data_pages", "Total pages in the data file.", float64(m.TotalPages)) + ro := 0.0 + if m.ReadOnly { + ro = 1 + } + w.Gauge("beastdb_read_only", "1 if the engine rejects writes (follower or fail-stop).", ro) + if cfg.Role != "leader" { + w.Gauge("beastdb_replica_applied_lsn", + "Last LSN applied by this follower. Lag = leader beastdb_wal_current_lsn minus this.", float64(engine.CurrentLSN())) + } + } + if cfg.DataDir != "" { + if free, ok := metrics.DiskFreeBytes(filepath.Clean(cfg.DataDir)); ok { + w.Gauge("beastdb_disk_free_bytes", "Free bytes on the data directory filesystem.", float64(free)) + } + } + w.Gauge("beastdb_grpc_requests_in_flight", "gRPC requests currently executing.", float64(metrics.InFlight.Load())) + metrics.Default.WriteRequests(w) +} + +type statusRecorder struct { + http.ResponseWriter + code int +} + +func (s *statusRecorder) WriteHeader(c int) { + if s.code == 0 { + s.code = c + } + s.ResponseWriter.WriteHeader(c) +} + +func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter } + +// HTTPMetrics records request count, status and latency for every HTTP +// request. The route label is the matched mux pattern (bounded cardinality). +func HTTPMetrics(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w} + next.ServeHTTP(rec, r) + code := rec.code + if code == 0 { + code = http.StatusOK + } + route := r.Pattern + if route == "" { + route = "unmatched" + } + metrics.Default.Observe("http", route, strconv.Itoa(code), time.Since(start)) + }) +} diff --git a/internal/web/handler/metrics_test.go b/internal/web/handler/metrics_test.go new file mode 100644 index 0000000..7b04a64 --- /dev/null +++ b/internal/web/handler/metrics_test.go @@ -0,0 +1,164 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +var sampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{([a-zA-Z_][a-zA-Z0-9_]*="([^"\\]|\\.)*"(,|))*\})? (-?[0-9][0-9eE.+-]*|\+Inf|-Inf|NaN)$`) + +// validateExposition checks Prometheus text format basics and histogram +// consistency (cumulative buckets, +Inf bucket == _count). +func validateExposition(t *testing.T, body string) { + t.Helper() + typed := map[string]string{} + infBucket := map[string]string{} + count := map[string]string{} + for _, line := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") { + if line == "" { + t.Fatalf("blank line in exposition") + } + if strings.HasPrefix(line, "# TYPE ") { + f := strings.Fields(line) + if len(f) != 4 { + t.Fatalf("bad TYPE line %q", line) + } + if _, dup := typed[f[2]]; dup { + t.Fatalf("duplicate TYPE for %s", f[2]) + } + typed[f[2]] = f[3] + continue + } + if strings.HasPrefix(line, "#") { + continue + } + if !sampleRe.MatchString(line) { + t.Fatalf("invalid sample line %q", line) + } + if strings.Contains(line, `le="+Inf"`) { + i := strings.LastIndex(line, " ") + key := strings.Replace(line[:i], `,le="+Inf"`, "", 1) + infBucket[strings.Replace(key, "_bucket", "", 1)] = line[i+1:] + } else if strings.Contains(line, "_count") { + i := strings.LastIndex(line, " ") + count[strings.Replace(line[:i], "_count", "", 1)] = line[i+1:] + } + } + for k, v := range infBucket { + if count[k] != v { + t.Fatalf("histogram %s: +Inf bucket %s != count %s", k, v, count[k]) + } + } + if len(infBucket) == 0 { + t.Fatal("no histogram series found") + } + // Bucket monotonicity. + var prevKey string + var prev float64 + for _, line := range strings.Split(body, "\n") { + if !strings.Contains(line, "_bucket{") { + continue + } + i := strings.LastIndex(line, " ") + v, _ := strconv.ParseFloat(line[i+1:], 64) + j := strings.Index(line, ",le=") + key := line[:j] + if key == prevKey && v < prev { + t.Fatalf("non-cumulative buckets at %q", line) + } + prevKey, prev = key, v + } +} + +func TestMetricsAuthAndFormat(t *testing.T) { + metrics.Default.Observe("grpc", "/beastdb.v1.KV/Get", "OK", 3*time.Millisecond) + metrics.Default.Observe("grpc", "/beastdb.v1.KV/Get", "NotFound", 30*time.Second) + metrics.Default.Observe("http", "/healthz", "200", time.Millisecond) + + h := MetricsHandler(&mockEngineReader{lsn: 7}, MetricsConfig{ + Role: "follower", Version: `v"1`, DataDir: t.TempDir(), Tokens: []string{"s3cret", ""}, + }) + + // No credentials, and an empty-bearer attempt, must be rejected. + for _, hdr := range []string{"", "Bearer ", "Bearer wrong", "Basic s3cret"} { + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + if hdr != "" { + req.Header.Set("Authorization", hdr) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("auth header %q: got %d want 401", hdr, rec.Code) + } + } + // Loopback peers get no special treatment. + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.RemoteAddr = "127.0.0.1:1234" + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("loopback anonymous: got %d want 401", rec.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.Header.Set("Authorization", "Bearer s3cret") + rec = httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("authorized: got %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain; version=0.0.4") { + t.Fatalf("content type %q", ct) + } + body := rec.Body.String() + validateExposition(t, body) + for _, want := range []string{ + `beastdb_build_info{version="v\"1",role="follower"} 1`, + "beastdb_wal_size_bytes ", "beastdb_wal_current_lsn ", "beastdb_buffer_pool_dirty_pages ", + "beastdb_buffer_pool_pinned_pages ", "beastdb_replica_applied_lsn 7", + `beastdb_requests_total{protocol="grpc",method="/beastdb.v1.KV/Get",code="NotFound"} 1`, + `beastdb_request_duration_seconds_bucket{protocol="grpc",method="/beastdb.v1.KV/Get",le="+Inf"} 2`, + "beastdb_disk_free_bytes ", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in output", want) + } + } + + if rec := httptest.NewRecorder(); func() int { + h.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/metrics", nil)) + return rec.Code + }() != http.StatusMethodNotAllowed { + t.Fatal("POST should be 405") + } +} + +func TestMetricsNoTokensRejectsBearer(t *testing.T) { + h := MetricsHandler(&mockEngineReader{}, MetricsConfig{Role: "leader", Version: "x"}) + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.Header.Set("Authorization", "Bearer ") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("got %d want 401", rec.Code) + } +} + +func TestHTTPMetricsMiddleware(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/teapot", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) }) + h := HTTPMetrics(mux) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/teapot", nil)) + var b strings.Builder + metrics.Default.WriteRequests(metrics.NewWriter(&b)) + if !strings.Contains(b.String(), `beastdb_requests_total{protocol="http",method="/teapot",code="418"}`) { + t.Fatalf("http request not recorded:\n%s", b.String()) + } +} diff --git a/internal/web/server.go b/internal/web/server.go index 1426ffe..7d35e84 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -30,6 +30,20 @@ type EngineBackend interface { // partitions is the optional project-defined partition registry served to the Studio. // Call Serve() to start accepting connections. func NewServer(addr string, engine EngineBackend, secret []byte, role, version string, partitions []handler.PartitionEntry, tokenStore ...*auth.TokenStore) (*Server, error) { + return NewServerWithOptions(addr, engine, secret, role, version, partitions, Options{}, tokenStore...) +} + +// Options carries optional settings for NewServerWithOptions. +type Options struct { + // DataDir enables the disk-free metric. + DataDir string + // MetricsTokens are bearer tokens accepted by /metrics in addition to + // admin sessions. With none configured only admin sessions may scrape. + MetricsTokens []string +} + +// NewServerWithOptions is NewServer plus optional settings. +func NewServerWithOptions(addr string, engine EngineBackend, secret []byte, role, version string, partitions []handler.PartitionEntry, opts Options, tokenStore ...*auth.TokenStore) (*Server, error) { staticFS, err := fs.Sub(staticFiles, "static") if err != nil { return nil, err @@ -63,6 +77,12 @@ func NewServer(addr string, engine EngineBackend, secret []byte, role, version s mux.HandleFunc("/healthz", handler.HealthzHandler(version)) mux.HandleFunc("/readyz", handler.ReadyzHandler(engine, role)) + // Prometheus metrics: admin session or bearer token required (never anonymous). + mux.Handle("/metrics", handler.MetricsHandler(engine, handler.MetricsConfig{ + Role: role, Version: version, DataDir: opts.DataDir, + Tokens: opts.MetricsTokens, Sessions: sessions, + })) + // Public auth routes mux.HandleFunc("/login", handler.LoginHandler(deps)) mux.HandleFunc("/logout", handler.LogoutHandler(deps)) @@ -114,7 +134,7 @@ func NewServer(addr string, engine EngineBackend, secret []byte, role, version s srv := &http.Server{ Addr: addr, - Handler: securityHeaders(mux), + Handler: securityHeaders(handler.HTTPMetrics(mux)), } return &Server{httpServer: srv}, nil From 7ccd824c54ff69f8a66cc5088e508d06259a5cc0 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:23:27 +0530 Subject: [PATCH 08/16] perf(wal,engine): group commit - concurrent writers share one fsync WAL gains Append (assign LSN, no fsync), WaitDurable/Sync with a shared syncedLSN so one fsync covers all appended records, and a sticky fsync error. Write keeps its signature and durability but now group-commits too. Engine writes (Put/PutIfAbsent/PutIf/Delete/DeleteIf/BatchWrite[If] and replicated apply) go through a commit queue: a committer goroutine runs a group of writes under the engine lock (append + apply), issues one fsync, notifies the observer in LSN order, then releases the lock and acks writers. Reads and replication never see a write before it is durable. A failed fsync halts the engine and fails every write in the group. Concurrent Put benchmark: 7.82 ms/op -> 0.51 ms/op (~15x). Co-Authored-By: Claude Sonnet 5.5 --- internal/api/engine.go | 15 +-- internal/api/engine_batch.go | 13 +- internal/api/engine_checkpoint.go | 8 ++ internal/api/engine_commit.go | 154 +++++++++++++++++++++++ internal/api/engine_conditional.go | 36 +++--- internal/api/engine_delete.go | 8 +- internal/api/engine_groupcommit_test.go | 161 ++++++++++++++++++++++++ internal/api/engine_ops.go | 30 +++-- internal/wal/group_commit_test.go | 111 ++++++++++++++++ internal/wal/wal.go | 112 +++++++++++++++-- 10 files changed, 584 insertions(+), 64 deletions(-) create mode 100644 internal/api/engine_commit.go create mode 100644 internal/api/engine_groupcommit_test.go create mode 100644 internal/wal/group_commit_test.go diff --git a/internal/api/engine.go b/internal/api/engine.go index 19c13dd..07f454c 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -21,9 +21,11 @@ type Engine struct { observer CommitObserver secIndex *index.SecondaryIndex readOnly bool - failed error // set by halt; blocks writes until restart - reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree + failed error // set by halt; blocks writes until restart + reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree mu sync.RWMutex + cq committer // group-commit queue (engine_commit.go) + pending []pendingNotify // commit events awaiting the group fsync; guarded by mu } // NewEngine initializes the database engine components with recovery support. @@ -114,18 +116,14 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { return nil } keyUint := binary.LittleEndian.Uint64(key) - e.mu.Lock() - defer e.mu.Unlock() - return e.putLocked(keyUint, value) + return e.commit(func() error { return e.putLocked(keyUint, value) }) case wal.OpDelete: if len(key) < 8 { return nil } keyUint := binary.LittleEndian.Uint64(key) - e.mu.Lock() - defer e.mu.Unlock() - return e.deleteLocked(keyUint) + return e.commit(func() error { return e.deleteLocked(keyUint) }) case wal.OpBatch: batchOps, err := wal.DecodeBatchPayload(value) @@ -160,4 +158,3 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { return nil } } - diff --git a/internal/api/engine_batch.go b/internal/api/engine_batch.go index 1fff44e..382869d 100644 --- a/internal/api/engine_batch.go +++ b/internal/api/engine_batch.go @@ -50,8 +50,11 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error return err } - e.mu.Lock() - defer e.mu.Unlock() + return e.commit(func() error { return e.batchWriteLocked(ops, conds) }) +} + +// batchWriteLocked runs under e.mu inside the group-commit pipeline. +func (e *Engine) batchWriteLocked(ops []BatchOperation, conds []KeyCondition) error { if e.failed != nil { return e.failed } @@ -87,7 +90,7 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error } // 2. Commit the entire batch atomically to the WAL - lsn, err := e.wal.Write(wal.OpBatch, nil, payload) + lsn, err := e.wal.Append(wal.OpBatch, nil, payload) if err != nil { return e.halt(err) } @@ -108,9 +111,7 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error } // 4. Notify replication observer of the atomic batch commit as a single unit - if e.observer != nil { - e.observer.OnCommit(lsn, wal.OpBatch, nil, payload) - } + e.queueNotify(lsn, wal.OpBatch, nil, payload) return nil } diff --git a/internal/api/engine_checkpoint.go b/internal/api/engine_checkpoint.go index 91036ba..c13cef3 100644 --- a/internal/api/engine_checkpoint.go +++ b/internal/api/engine_checkpoint.go @@ -11,6 +11,7 @@ import ( // Close flushes all dirty pages to physical disk and releases file handles. func (e *Engine) Close() error { + e.stopCommitter() // drain queued writes before tearing down files e.mu.Lock() defer e.mu.Unlock() @@ -39,6 +40,13 @@ func (e *Engine) Checkpoint() error { e.mu.Lock() defer e.mu.Unlock() + // Writes are applied in memory only inside a commit group that fsyncs before + // releasing e.mu, so nothing un-synced can exist here; sync anyway so data + // pages are never flushed ahead of their WAL records. + if err := e.wal.SyncIfEnabled(); err != nil { + return fmt.Errorf("checkpoint sync wal: %w", err) + } + if err := e.updateMeta(func(m *storage.MetaData) { m.LastCheckpointLSN = e.wal.CurrentLSN() m.ActiveDataPageID = e.activeDataPage diff --git a/internal/api/engine_commit.go b/internal/api/engine_commit.go new file mode 100644 index 0000000..77b5c55 --- /dev/null +++ b/internal/api/engine_commit.go @@ -0,0 +1,154 @@ +package api + +import ( + "errors" + "sync" +) + +// ErrEngineClosed is returned for writes submitted after Close. +var ErrEngineClosed = errors.New("engine: closed") + +// pendingNotify is a commit event held back until its WAL record is fsynced. +type pendingNotify struct { + lsn uint64 + opType byte + key []byte + value []byte +} + +type writeReq struct { + run func() error // executes under e.mu: check, WAL append (no fsync), apply + err error + done chan struct{} + ok bool // run appended a WAL record (so it depends on the batch fsync) +} + +// committer is the group-commit queue. A single goroutine drains it: each pass +// takes every queued write, runs them back to back under the exclusive engine +// lock (appending to the WAL without fsync), issues ONE fsync for the whole +// group, notifies the replication observer in LSN order, and only then +// releases the lock and acknowledges the writers. Because the lock is held +// across the fsync, readers and the observer never see a write before it is +// durable, exactly as with fsync-per-write; the gain is that N queued writers +// pay for one fsync instead of N. +type committer struct { + mu sync.Mutex + queue []*writeReq + started bool + closed bool + wake chan struct{} + done chan struct{} +} + +// commit runs fn through the group-commit pipeline and returns once its WAL +// record (if it wrote one) is durable. fn runs with e.mu held exclusively and +// must record observer events via e.queueNotify rather than calling the +// observer directly. +func (e *Engine) commit(fn func() error) error { + c := &e.cq + req := &writeReq{run: fn, done: make(chan struct{})} + + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return ErrEngineClosed + } + if !c.started { + c.started = true + c.wake = make(chan struct{}, 1) + c.done = make(chan struct{}) + go e.commitLoop() + } + c.queue = append(c.queue, req) + c.mu.Unlock() + + select { + case c.wake <- struct{}{}: + default: + } + <-req.done + return req.err +} + +func (e *Engine) commitLoop() { + c := &e.cq + defer close(c.done) + for { + <-c.wake + for { + c.mu.Lock() + batch := c.queue + c.queue = nil + closed := c.closed + c.mu.Unlock() + if len(batch) == 0 { + if closed { + return + } + break + } + e.runBatch(batch) + } + } +} + +// stopCommitter drains outstanding writes and stops the commit goroutine. +func (e *Engine) stopCommitter() { + c := &e.cq + c.mu.Lock() + c.closed = true + started := c.started + c.mu.Unlock() + if started { + select { + case c.wake <- struct{}{}: + default: + } + <-c.done + } +} + +func (e *Engine) runBatch(batch []*writeReq) { + e.mu.Lock() + e.pending = e.pending[:0] + wrote := false + for _, r := range batch { + before := e.wal.CurrentLSN() + r.err = r.run() + // A request that appended a WAL record must wait for the group fsync, + // even if applying it then failed (the record is still in the log). + r.ok = e.wal.CurrentLSN() != before + wrote = wrote || r.ok + } + + if wrote { + if syncErr := e.wal.SyncIfEnabled(); syncErr != nil { + // The records may or may not be on disk and memory already reflects + // them: fail-stop. Nothing in this group is acknowledged or replicated. + herr := e.halt(syncErr) + for _, r := range batch { + if r.ok { + r.err = herr + } + } + e.pending = e.pending[:0] + } + } + if e.observer != nil { + for _, n := range e.pending { + e.observer.OnCommit(n.lsn, n.opType, n.key, n.value) + } + } + e.pending = e.pending[:0] + e.mu.Unlock() + + for _, r := range batch { + close(r.done) + } +} + +// queueNotify records a commit event to be delivered after the group fsync. +// Caller holds e.mu. +func (e *Engine) queueNotify(lsn uint64, opType byte, key, value []byte) { + e.pending = append(e.pending, pendingNotify{lsn: lsn, opType: opType, key: key, value: value}) +} diff --git a/internal/api/engine_conditional.go b/internal/api/engine_conditional.go index 51e0445..3e97f2c 100644 --- a/internal/api/engine_conditional.go +++ b/internal/api/engine_conditional.go @@ -78,15 +78,15 @@ func (e *Engine) PutIf(key uint64, value []byte, cond Condition) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - if e.failed != nil { - return e.failed - } - if err := e.checkConditionLocked(key, cond); err != nil { - return err - } - return e.putLocked(key, value) + return e.commit(func() error { + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.putLocked(key, value) + }) } // DeleteIf removes key only if cond holds, atomically with the check. @@ -94,13 +94,13 @@ func (e *Engine) DeleteIf(key uint64, cond Condition) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - e.mu.Lock() - defer e.mu.Unlock() - if e.failed != nil { - return e.failed - } - if err := e.checkConditionLocked(key, cond); err != nil { - return err - } - return e.deleteLocked(key) + return e.commit(func() error { + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.deleteLocked(key) + }) } diff --git a/internal/api/engine_delete.go b/internal/api/engine_delete.go index 93c08aa..ff529cb 100644 --- a/internal/api/engine_delete.go +++ b/internal/api/engine_delete.go @@ -11,9 +11,7 @@ func (e *Engine) Delete(key uint64) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - e.mu.Lock() - defer e.mu.Unlock() - return e.deleteLocked(key) + return e.commit(func() error { return e.deleteLocked(key) }) } func (e *Engine) deleteLocked(key uint64) error { @@ -22,7 +20,7 @@ func (e *Engine) deleteLocked(key uint64) error { } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) - lsn, err := e.wal.Write(wal.OpDelete, keyBytes[:], nil) + lsn, err := e.wal.Append(wal.OpDelete, keyBytes[:], nil) if err != nil { return e.halt(err) } @@ -32,6 +30,6 @@ func (e *Engine) deleteLocked(key uint64) error { } e.unindexValueLocked(key) - e.notifyCommit(lsn, wal.OpDelete, keyBytes[:], nil) + e.queueNotify(lsn, wal.OpDelete, keyBytes[:], nil) return nil } diff --git a/internal/api/engine_groupcommit_test.go b/internal/api/engine_groupcommit_test.go new file mode 100644 index 0000000..305b965 --- /dev/null +++ b/internal/api/engine_groupcommit_test.go @@ -0,0 +1,161 @@ +package api + +import ( + "fmt" + "path/filepath" + "sync" + "testing" +) + +type recObs struct { + mu sync.Mutex + lsns []uint64 + eng *Engine +} + +func (o *recObs) OnCommit(lsn uint64, _ byte, _, _ []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // The observer must only ever see records that are already durable. + if o.eng.wal.DurableLSN() < lsn { + panic(fmt.Sprintf("observer saw LSN %d before durable (%d)", lsn, o.eng.wal.DurableLSN())) + } + o.lsns = append(o.lsns, lsn) +} + +// Many concurrent writers of mixed kinds; observer sees strictly increasing, +// durable LSNs; after an abrupt crash (no Close/flush) every acknowledged +// write is recovered. +func TestGroupCommitConcurrentWritersCrashRecovery(t *testing.T) { + dir := t.TempDir() + dbPath, walPath := filepath.Join(dir, "g.bin"), filepath.Join(dir, "g.wal") + e, err := NewEngine(dbPath, walPath, 256) + if err != nil { + t.Fatal(err) + } + obs := &recObs{eng: e} + e.SetCommitObserver(obs) + + const writers, per = 16, 40 + var wg sync.WaitGroup + for g := 0; g < writers; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < per; i++ { + k := uint64(g*1000 + i + 1) + val := []byte(fmt.Sprintf("v-%d", k)) + var err error + switch i % 4 { + case 0: + err = e.Put(k, val) + case 1: + err = e.PutIfAbsent(k, val) + case 2: + err = e.BatchWrite([]BatchOperation{{Type: BatchOpPut, Key: k, Value: val}}) + case 3: + err = e.PutIf(k, val, Condition{Kind: CondAbsent}) + } + if err != nil { + t.Errorf("write %d: %v", k, err) + } + } + }(g) + } + wg.Wait() + + obs.mu.Lock() + if len(obs.lsns) != writers*per { + t.Fatalf("observer got %d events, want %d", len(obs.lsns), writers*per) + } + for i := 1; i < len(obs.lsns); i++ { + if obs.lsns[i] != obs.lsns[i-1]+1 { + t.Fatalf("observer order broken at %d: %d then %d", i, obs.lsns[i-1], obs.lsns[i]) + } + } + obs.mu.Unlock() + + // Simulate crash: stop the committer, drop files without flushing pages. + e.stopCommitter() + _ = e.wal.Close() + _ = e.disk.Close() + + r, err := NewEngine(dbPath, walPath, 256) + if err != nil { + t.Fatal(err) + } + defer r.Close() + for g := 0; g < writers; g++ { + for i := 0; i < per; i++ { + k := uint64(g*1000 + i + 1) + v, ok, err := r.Get(k) + if err != nil || !ok || string(v) != fmt.Sprintf("v-%d", k) { + t.Fatalf("key %d lost after crash: ok=%v err=%v v=%q", k, ok, err, v) + } + } + } +} + +// Conditional writes stay atomic under contention: exactly one PutIfAbsent wins. +func TestGroupCommitConditionalAtomicity(t *testing.T) { + dir := t.TempDir() + e, err := NewEngine(filepath.Join(dir, "a.bin"), filepath.Join(dir, "a.wal"), 64) + if err != nil { + t.Fatal(err) + } + defer e.Close() + var wg sync.WaitGroup + var mu sync.Mutex + wins := 0 + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if e.PutIfAbsent(7, []byte("x")) == nil { + mu.Lock() + wins++ + mu.Unlock() + } + }() + } + wg.Wait() + if wins != 1 { + t.Fatalf("wins=%d", wins) + } +} + +func TestWriteAfterCloseFails(t *testing.T) { + dir := t.TempDir() + e, _ := NewEngine(filepath.Join(dir, "c.bin"), filepath.Join(dir, "c.wal"), 64) + _ = e.Put(1, []byte("a")) + _ = e.Close() + if err := e.Put(2, []byte("b")); err != ErrEngineClosed { + t.Fatalf("got %v", err) + } +} + +func BenchmarkEngineConcurrentWrite(b *testing.B) { + dir := b.TempDir() + e, err := NewEngine(filepath.Join(dir, "b.bin"), filepath.Join(dir, "b.wal"), 1024) + if err != nil { + b.Fatal(err) + } + defer e.Close() + payload := []byte(`{"name":"benchmark-record","val":12345}`) + var mu sync.Mutex + var next uint64 + b.SetParallelism(8) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + mu.Lock() + next++ + k := next + mu.Unlock() + if err := e.Put(k, payload); err != nil { + b.Error(err) + return + } + } + }) +} diff --git a/internal/api/engine_ops.go b/internal/api/engine_ops.go index a164e58..5d457ea 100644 --- a/internal/api/engine_ops.go +++ b/internal/api/engine_ops.go @@ -24,7 +24,7 @@ var ( ErrReadOnlyReplica = errors.New("writes are not permitted on read-only replica") ) -// Put logs mutation to WAL, stores tuple in slotted page, and indexes key in B+ Tree. +// Put logs mutation (group-committed: returns once the WAL record is fsynced) to WAL, stores tuple in slotted page, and indexes key in B+ Tree. func (e *Engine) Put(key uint64, value []byte) error { if e.IsReadOnly() { return ErrReadOnlyReplica @@ -32,9 +32,7 @@ func (e *Engine) Put(key uint64, value []byte) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - return e.putLocked(key, value) + return e.commit(func() error { return e.putLocked(key, value) }) } // PutIfAbsent inserts only when key is not already present. @@ -45,16 +43,16 @@ func (e *Engine) PutIfAbsent(key uint64, value []byte) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - _, err := e.tree.Find(key) - if err == nil { - return ErrRecordExists - } - if err != index.ErrKeyNotFound { - return err - } - return e.putLocked(key, value) + return e.commit(func() error { + _, err := e.tree.Find(key) + if err == nil { + return ErrRecordExists + } + if err != index.ErrKeyNotFound { + return err + } + return e.putLocked(key, value) + }) } func (e *Engine) putLocked(key uint64, value []byte) error { @@ -63,7 +61,7 @@ func (e *Engine) putLocked(key uint64, value []byte) error { } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) - lsn, err := e.wal.Write(wal.OpPut, keyBytes[:], value) + lsn, err := e.wal.Append(wal.OpPut, keyBytes[:], value) if err != nil { return e.halt(err) } @@ -72,7 +70,7 @@ func (e *Engine) putLocked(key uint64, value []byte) error { } e.indexValueLocked(key, value) - e.notifyCommit(lsn, wal.OpPut, keyBytes[:], value) + e.queueNotify(lsn, wal.OpPut, keyBytes[:], value) return nil } diff --git a/internal/wal/group_commit_test.go b/internal/wal/group_commit_test.go new file mode 100644 index 0000000..b5724ba --- /dev/null +++ b/internal/wal/group_commit_test.go @@ -0,0 +1,111 @@ +package wal + +import ( + "path/filepath" + "sync" + "testing" +) + +// Concurrent Write calls must all be durable on return, get unique monotonic +// LSNs, and replay completely after reopen. +func TestWALGroupCommitConcurrentWriters(t *testing.T) { + path := filepath.Join(t.TempDir(), "gc.wal") + w, err := OpenWAL(path, true) + if err != nil { + t.Fatal(err) + } + const writers, per = 32, 25 + var wg sync.WaitGroup + lsns := make(chan uint64, writers*per) + for g := 0; g < writers; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < per; i++ { + lsn, err := w.Write(OpPut, []byte("k"), []byte("v")) + if err != nil { + t.Error(err) + return + } + if w.DurableLSN() < lsn { + t.Errorf("acknowledged LSN %d not durable (durable=%d)", lsn, w.DurableLSN()) + } + lsns <- lsn + } + }() + } + wg.Wait() + close(lsns) + seen := map[uint64]bool{} + for l := range lsns { + if seen[l] { + t.Fatalf("duplicate LSN %d", l) + } + seen[l] = true + } + if len(seen) != writers*per { + t.Fatalf("got %d LSNs", len(seen)) + } + _ = w.Close() + + var last uint64 + rep, err := Replay(path, func(r *Record) error { + if r.LSN != last+1 { + t.Fatalf("non-monotonic replay: %d after %d", r.LSN, last) + } + last = r.LSN + return nil + }) + if err != nil || rep.TotalRecords != writers*per { + t.Fatalf("replay: %v total=%d", err, rep.TotalRecords) + } +} + +func TestWALAppendNotDurableUntilWait(t *testing.T) { + w, err := OpenWAL(filepath.Join(t.TempDir(), "a.wal"), true) + if err != nil { + t.Fatal(err) + } + defer w.Close() + lsn, _ := w.Append(OpPut, []byte("k"), []byte("v")) + if w.DurableLSN() >= lsn { + t.Fatal("Append must not mark record durable") + } + if err := w.WaitDurable(lsn); err != nil || w.DurableLSN() < lsn { + t.Fatalf("WaitDurable: %v durable=%d", err, w.DurableLSN()) + } +} + +func TestWALCheckpointMarksDurableAndLSNContinues(t *testing.T) { + w, _ := OpenWAL(filepath.Join(t.TempDir(), "c.wal"), true) + defer w.Close() + l1, _ := w.Append(OpPut, []byte("k"), []byte("v")) + if err := w.Checkpoint(); err != nil { + t.Fatal(err) + } + if w.DurableLSN() < l1 { + t.Fatal("checkpoint must cover appended records") + } + l2, _ := w.Write(OpPut, []byte("k"), []byte("v")) + if l2 != l1+1 { + t.Fatalf("LSN after checkpoint: %d", l2) + } +} + +func BenchmarkWALConcurrentSyncedWrite(b *testing.B) { + w, err := OpenWAL(filepath.Join(b.TempDir(), "b.wal"), true) + if err != nil { + b.Fatal(err) + } + defer w.Close() + b.SetParallelism(8) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := w.Write(OpPut, []byte("key"), []byte("value")); err != nil { + b.Error(err) + return + } + } + }) +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 2d038cc..42cc68f 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -8,12 +8,22 @@ import ( ) // WAL manages the sequential append-only write-ahead log file. +// +// Durability is group-committed: Append assigns an LSN and writes the frame +// under mu (cheap), while the fsync is performed by whichever caller reaches +// syncMu first and covers every record appended up to that moment, so +// concurrent writers share one fsync. Lock order is always syncMu, then mu. type WAL struct { mu sync.Mutex file *os.File path string currentLSN uint64 syncOnWrite bool + + syncMu sync.Mutex // serializes fsyncs; guards syncedLSN + syncedLSN uint64 // highest LSN known to be fsynced + errMu sync.Mutex // guards syncErr (read by Append without syncMu) + syncErr error // sticky: after a failed fsync the log is untrustworthy } // OpenWAL opens an existing WAL file or creates a new one at the specified path. @@ -49,17 +59,22 @@ func OpenWAL(path string, syncOnWrite bool) (*WAL, error) { } w.currentLSN = lastLSN + w.syncedLSN = lastLSN // everything already on disk at open is treated as durable return w, nil } -// Write appends a state mutation record to the WAL, returning the assigned LSN. -func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { +// Append writes a record to the log file and assigns its LSN but does NOT +// fsync. The record is not durable until WaitDurable(lsn) or Sync returns. +func (w *WAL) Append(opType byte, key, value []byte) (uint64, error) { w.mu.Lock() defer w.mu.Unlock() - w.currentLSN++ + if err := w.loadSyncErr(); err != nil { + return 0, err + } + rec := &Record{ - LSN: w.currentLSN, + LSN: w.currentLSN + 1, Type: opType, Key: key, Value: value, @@ -67,27 +82,98 @@ func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { encoded := EncodeRecord(rec) if len(encoded) > MaxFrameSize { - w.currentLSN-- return 0, ErrFrameTooLarge } if _, err := w.file.Write(encoded); err != nil { return 0, err } + w.currentLSN = rec.LSN + return rec.LSN, nil +} +// Write appends a state mutation record to the WAL, returning the assigned LSN. +// When syncOnWrite is set it returns only after the record is fsynced; concurrent +// callers share fsyncs (group commit). +func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { + lsn, err := w.Append(opType, key, value) + if err != nil { + return 0, err + } if w.syncOnWrite { - if err := w.file.Sync(); err != nil { + if err := w.WaitDurable(lsn); err != nil { return 0, err } } + return lsn, nil +} + +// SyncOnWrite reports whether this log was opened with fsync-per-commit semantics. +func (w *WAL) SyncOnWrite() bool { return w.syncOnWrite } + +func (w *WAL) loadSyncErr() error { + w.errMu.Lock() + defer w.errMu.Unlock() + return w.syncErr +} + +// WaitDurable blocks until the record with the given LSN has been fsynced. +// If another caller's fsync already covered it, no further fsync is issued. +func (w *WAL) WaitDurable(lsn uint64) error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + if w.syncedLSN >= lsn { + return nil + } + return w.syncLocked() +} + +// syncLocked fsyncs everything appended so far. Caller holds syncMu. +func (w *WAL) syncLocked() error { + w.mu.Lock() + target := w.currentLSN + f := w.file + w.mu.Unlock() - return w.currentLSN, nil + // Appends may proceed while fsync runs; they simply miss this sync and are + // covered by the next one. target only claims what was written before it. + if err := f.Sync(); err != nil { + w.errMu.Lock() + w.syncErr = err + w.errMu.Unlock() + return err + } + if target > w.syncedLSN { + w.syncedLSN = target + } + return nil } // Sync forces all buffered kernel writes to physical non-volatile storage. func (w *WAL) Sync() error { - w.mu.Lock() - defer w.mu.Unlock() - return w.file.Sync() + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + return w.syncLocked() +} + +// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. +func (w *WAL) SyncIfEnabled() error { + if !w.syncOnWrite { + return nil + } + return w.Sync() +} + +// DurableLSN returns the highest LSN known to be fsynced. +func (w *WAL) DurableLSN() uint64 { + w.syncMu.Lock() + defer w.syncMu.Unlock() + return w.syncedLSN } // CurrentLSN returns the latest assigned Log Sequence Number. @@ -115,6 +201,8 @@ func (w *WAL) Path() string { // Close flushes buffered writes and closes the log file. func (w *WAL) Close() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() w.mu.Lock() defer w.mu.Unlock() _ = w.file.Sync() @@ -126,12 +214,16 @@ func (w *WAL) Close() error { // The caller MUST have flushed all dirty buffer pool pages to disk first so the // archived WAL records are no longer needed for recovery. func (w *WAL) Checkpoint() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() w.mu.Lock() defer w.mu.Unlock() if err := w.file.Sync(); err != nil { return err } + // Everything appended so far is now durable (and about to be archived). + w.syncedLSN = w.currentLSN if err := w.file.Close(); err != nil { return err } From 65051a7880134a7644c7a4dd011805fd624443cd Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:36:15 +0530 Subject: [PATCH 09/16] fix(storage): enforce write-ahead barrier before dirty page writes Group commit applies writes to pages before the group's fsync, so the buffer pool could flush a page ahead of its WAL records. Add a WAL barrier (WAL.EnsureDurable) called before eviction, FlushPage and FlushAll; a failing barrier blocks the page write. Adds regression tests and the upgrade spec. Co-Authored-By: Claude Sonnet 5.5 --- docs/upgrade-spec.md | 30 ++++++ internal/api/engine.go | 4 + internal/storage/buffer_pool.go | 28 +++++- internal/storage/buffer_pool_eviction.go | 2 +- internal/storage/wal_barrier_test.go | 112 +++++++++++++++++++++++ internal/wal/wal.go | 12 +++ 6 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 docs/upgrade-spec.md create mode 100644 internal/storage/wal_barrier_test.go diff --git a/docs/upgrade-spec.md b/docs/upgrade-spec.md new file mode 100644 index 0000000..ea293a1 --- /dev/null +++ b/docs/upgrade-spec.md @@ -0,0 +1,30 @@ +# BeastDB upgrade spec (security, usability, performance) + +Source: review of 2026-11-12. Complements [ROADMAP.md](../ROADMAP.md) and [v1-semantics.md](v1-semantics.md). Claims here are goals, not shipped features. + +## Findings that drive the work + +| Area | Finding | Evidence | +|---|---|---| +| Perf | WAL fsyncs inside its mutex on every write; ~4 ms/write | `internal/wal/wal.go` Write | +| Perf | One global `sync.RWMutex`; scans hold it for the cursor lifetime | `internal/api/engine_ops.go` Scan | +| Perf | Benchmarks are in-process, 1 MB pool, 2k records; no competitor comparison | `docs/benchmarks.md` | +| Sec | gRPC auth checks token validity only; roles ignored | `internal/api/grpc_auth.go` | +| Sec | No encryption at rest (data, WAL, backups) | repo-wide search | +| Sec | Static token has no expiry/rotation | `grpc_auth.go` | +| UX | uint64 keys only, no namespaces, one Python example, no /metrics | proto, `examples/` | + +## Work packages (parallel, disjoint files) + +1. **WP1 WAL group commit** (`internal/wal`, write path in `internal/api`): batch concurrent writers into one fsync, same durability. Target: >=10x concurrent write throughput. Must keep crash/fault tests green. +2. **WP2 gRPC role authorization** (`internal/api/grpc_auth*.go`, token store): per-method roles (viewer = Get/Scan; admin/writer = mutations; replication role for StreamWAL/Ack), token expiry. Deny by default. +3. **WP3 YCSB-style harness** (new `cmd/beastbench`): workloads A-F over gRPC, reports throughput and p50/p95/p99, reproducible flags, doc update. +4. **WP4 Prometheus /metrics** (`internal/web/handler`): request/error/latency, WAL size and sync latency, dirty pages, checkpoint age, LSN. + +## Next wave (not started) + +Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log. + +## Studio dashboard design + +Figma design for a Firestore/Supabase-style console: project overview, collections/table editor, SQL-free query bar, API keys and roles, metrics, backups. Aimed at non-experts. Tracked in Figma; implementation in `studio/`. diff --git a/internal/api/engine.go b/internal/api/engine.go index 07f454c..b3bc26a 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -41,6 +41,10 @@ func NewEngine(dbPath, walPath string, poolSize int) (*Engine, error) { _ = disk.Close() return nil, err } + // Write-ahead barrier: no dirty data page may reach the data file before + // the WAL records that describe it are durable (group commit applies pages + // before the group's fsync). + bpm.SetWALBarrier(w.EnsureDurable) if disk.NumPages() == 0 { return initFreshV1(disk, bpm, w) diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index 19d4a96..53e4a67 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -22,6 +22,30 @@ type BufferPoolManager struct { clockHand int mu sync.RWMutex pagePool sync.Pool // Fix #6: reusable 4KB page buffers + + // walBarrier, when set, is called before any dirty page is written to the + // data file. It must return only once every WAL record appended so far is + // durable, which preserves the write-ahead invariant even though writes are + // applied to pages before their group's fsync completes. + walBarrier func() error +} + +// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. +func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { + bpm.mu.Lock() + defer bpm.mu.Unlock() + bpm.walBarrier = fn +} + +// writeBack writes one dirty frame to the data file after the WAL barrier. +// Caller must hold bpm.mu (write lock). +func (bpm *BufferPoolManager) writeBack(f *Frame) error { + if bpm.walBarrier != nil { + if err := bpm.walBarrier(); err != nil { + return err + } + } + return bpm.disk.WritePage(f.PageID, f.Page.Data()) } // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. @@ -127,7 +151,7 @@ func (bpm *BufferPoolManager) FlushPage(pageID uint64) error { frame := bpm.frames[idx] if frame.Page != nil && frame.IsDirty { - if err := bpm.disk.WritePage(frame.PageID, frame.Page.Data()); err != nil { + if err := bpm.writeBack(frame); err != nil { return err } frame.IsDirty = false @@ -142,7 +166,7 @@ func (bpm *BufferPoolManager) FlushAll() error { for _, frame := range bpm.frames { if frame.Page != nil && frame.IsDirty { - if err := bpm.disk.WritePage(frame.PageID, frame.Page.Data()); err != nil { + if err := bpm.writeBack(frame); err != nil { return err } frame.IsDirty = false diff --git a/internal/storage/buffer_pool_eviction.go b/internal/storage/buffer_pool_eviction.go index 2c2cb6e..8695808 100644 --- a/internal/storage/buffer_pool_eviction.go +++ b/internal/storage/buffer_pool_eviction.go @@ -45,7 +45,7 @@ func (bpm *BufferPoolManager) evictFrame(idx int) error { return nil } if victim.IsDirty { - if err := bpm.disk.WritePage(victim.PageID, victim.Page.Data()); err != nil { + if err := bpm.writeBack(victim); err != nil { return err } victim.IsDirty = false diff --git a/internal/storage/wal_barrier_test.go b/internal/storage/wal_barrier_test.go new file mode 100644 index 0000000..1e03faa --- /dev/null +++ b/internal/storage/wal_barrier_test.go @@ -0,0 +1,112 @@ +package storage + +import ( + "bytes" + "errors" + "path/filepath" + "testing" +) + +func newBarrierPool(t *testing.T, size int) (*BufferPoolManager, *DiskManager) { + t.Helper() + disk, err := OpenDiskManager(filepath.Join(t.TempDir(), "barrier.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = disk.Close() }) + return NewBufferPoolManager(disk, size), disk +} + +// A dirty page must not reach the data file when the WAL barrier fails: +// otherwise a page could be persisted ahead of the log records that describe it. +func TestWALBarrierFailureBlocksPageWrite(t *testing.T) { + bpm, disk := newBarrierPool(t, 1) + errBarrier := errors.New("wal not durable") + bpm.SetWALBarrier(func() error { return errBarrier }) + + page, id, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + copy(page.Data()[100:], []byte("uncommitted")) + if err := bpm.UnpinPage(id, true); err != nil { + t.Fatal(err) + } + + // Eviction (pool of one) must fail because the barrier fails. + if _, _, err := bpm.NewPage(); !errors.Is(err, errBarrier) { + t.Fatalf("eviction err = %v, want barrier error", err) + } + if err := bpm.FlushPage(id); !errors.Is(err, errBarrier) { + t.Fatalf("FlushPage err = %v, want barrier error", err) + } + if err := bpm.FlushAll(); !errors.Is(err, errBarrier) { + t.Fatalf("FlushAll err = %v, want barrier error", err) + } + + onDisk := make([]byte, PageSize) + if err := disk.ReadPage(id, onDisk); err != nil { + t.Fatal(err) + } + if bytes.Contains(onDisk, []byte("uncommitted")) { + t.Fatal("dirty page reached the data file although the WAL barrier failed") + } +} + +// The barrier runs before every page write path, and pages are written after it. +func TestWALBarrierRunsBeforeEveryPageWrite(t *testing.T) { + bpm, disk := newBarrierPool(t, 1) + calls := 0 + bpm.SetWALBarrier(func() error { calls++; return nil }) + + page, id, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + copy(page.Data()[100:], []byte("committed")) + if err := bpm.UnpinPage(id, true); err != nil { + t.Fatal(err) + } + + // Evict via a second page (pool size 1). + _, id2, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("barrier calls after eviction = %d, want 1", calls) + } + onDisk := make([]byte, PageSize) + if err := disk.ReadPage(id, onDisk); err != nil { + t.Fatal(err) + } + if !bytes.Contains(onDisk, []byte("committed")) { + t.Fatal("evicted page was not written after the barrier succeeded") + } + + if err := bpm.UnpinPage(id2, true); err != nil { + t.Fatal(err) + } + if err := bpm.FlushPage(id2); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("barrier calls after FlushPage = %d, want 2", calls) + } + + // Dirty again, then FlushAll. + p, err := bpm.FetchPage(id2) + if err != nil { + t.Fatal(err) + } + copy(p.Data()[200:], []byte("more")) + if err := bpm.UnpinPage(id2, true); err != nil { + t.Fatal(err) + } + if err := bpm.FlushAll(); err != nil { + t.Fatal(err) + } + if calls != 3 { + t.Fatalf("barrier calls after FlushAll = %d, want 3", calls) + } +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 42cc68f..7261209 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -169,6 +169,18 @@ func (w *WAL) SyncIfEnabled() error { return w.Sync() } +// EnsureDurable returns once every record appended so far is fsynced. It is a +// no-op when the log was not opened with syncOnWrite, and costs no fsync when +// a previous sync already covers the current LSN. The buffer pool calls it +// before writing a dirty page so a data page never reaches disk ahead of the +// WAL records that describe it. +func (w *WAL) EnsureDurable() error { + if !w.syncOnWrite { + return nil + } + return w.WaitDurable(w.CurrentLSN()) +} + // DurableLSN returns the highest LSN known to be fsynced. func (w *WAL) DurableLSN() uint64 { w.syncMu.Lock() From cf7ed6c80dbcafaa3ec451d7113bc42fd037266b Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:37:45 +0530 Subject: [PATCH 10/16] docs: document group commit, WAL barrier, gRPC roles, metrics, beastbench and Studio design Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 ++ ROADMAP.md | 2 +- docs/architecture.md | 4 ++-- docs/benchmarks.md | 6 ++++++ docs/runbooks/operator-guide.md | 4 ++++ docs/studio.md | 24 ++++++++++++++++++++++++ docs/upgrade-spec.md | 24 +++++++++++++++++------- docs/v1-semantics.md | 25 +++++++++++++++++++++++-- 8 files changed, 79 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2a431de..169640d 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,8 @@ BeastDB is optimized for **read-heavy OLTP** — deterministic point reads and o - [**V1 Semantics & Guarantees**](docs/v1-semantics.md) — Durability acknowledgment, limits, and replica reads. - [**System Benchmarks**](docs/benchmarks.md) — Reproducible throughput, tail latencies, and cache-eviction profiles. +- [**Benchmarking Guide**](docs/benchmarking.md) — `beastbench` YCSB-style workloads and how to compare fairly with other databases. +- [**Upgrade Spec**](docs/upgrade-spec.md) — Security, usability and performance work packages and what is still planned. - [**Operator Guide**](docs/runbooks/operator-guide.md) — TLS configuration, cluster roles, manual failover, and `/healthz` probes. - [**Disaster Recovery**](docs/runbooks/disaster-recovery.md) — Physical backup drills, checksum verification, and RTO/RPO SLA. - [**Client Integration**](docs/clients.md) — Protobuf contracts for Go, Python, TypeScript, and Rust. diff --git a/ROADMAP.md b/ROADMAP.md index 15ddabb..3d88fe0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -122,7 +122,7 @@ Order is dependency-based, not a calendar promise. Initial planning range for on ## Beyond V1 gates -See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite), limit-based scan pagination, and values up to 2 MiB with 8 MiB batches ([ADR 0001](docs/adr/0001-large-values.md)). +See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases and the [upgrade spec](docs/upgrade-spec.md) for the current security, usability and performance work. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite), limit-based scan pagination, values up to 2 MiB with 8 MiB batches ([ADR 0001](docs/adr/0001-large-values.md)), WAL group commit with a write-ahead buffer-pool barrier, per-method gRPC role authorization with token expiry, a Prometheus `/metrics` endpoint, and the `beastbench` YCSB-style load generator ([benchmarking guide](docs/benchmarking.md)). ## References diff --git a/docs/architecture.md b/docs/architecture.md index 59dceb7..96d531e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ A modular, zero-allocation database architecture built from first principles in 3. The RESP parser processes byte streams in-place into command frames without heap allocations. ### Write Execution Flow -1. **WAL Append:** Before modifying any page in memory, a log record is written to the WAL and synced (`fdatasync`). +1. **Group commit:** Writers enqueue their operation. One committer goroutine drains the queue, takes the engine lock exclusively, and for each queued write checks its condition, appends the record to the WAL (no `fsync` yet) and applies it in memory. It then issues **one** `fsync` for the whole group, notifies replication in LSN order, releases the lock, and only then acknowledges the writers. The lock is held across the `fsync` so readers and replicas never see a write before it is durable. 2. **Buffer Pool Pin:** The targeted 4KB disk page is fetched into a memory frame and pinned. 3. **Slotted Page Update:** The tuple is inserted into the slotted page, updating the slot array and free space pointer. 4. **Index Update:** If indexed, the B+ tree is traversed; keys and record pointers are updated, splitting nodes if page capacity is exceeded. @@ -66,6 +66,6 @@ A modular, zero-allocation database architecture built from first principles in --- ## 3. Storage Invariants -- **Write-Ahead Invariant:** Log records must hit disk before dirty database pages containing those modifications are written. +- **Write-Ahead Invariant:** Log records must hit disk before dirty database pages containing those modifications are written. Because group commit applies a write to its page before the group's `fsync`, the buffer pool enforces this with a **WAL barrier** (`WAL.EnsureDurable`) called before every dirty-page write (eviction, `FlushPage`, `FlushAll`). If the barrier fails, the page is not written. The barrier costs no extra `fsync` when the WAL is already durable up to the current LSN. - **Page Isolation:** Every 4KB page maintains its own self-describing header, slot array, and CRC checksum. - **Pin Safety:** A page frame with `pin_count > 0` cannot be evicted from the buffer pool under any circumstances. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 711814e..9e520bc 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -51,6 +51,12 @@ Measured on 12th Gen Intel(R) Core(TM) i5-12450HX: --- +### Concurrent writes and group commit + +Writers now share `fsync`s (group commit, see [architecture](architecture.md)). On one Apple-silicon laptop the engine benchmark `BenchmarkEngineConcurrentWrite` (96 concurrent writer goroutines) measured about 7.8 ms/op before and 0.5 ms/op after; `BenchmarkEnginePointWrite` (one sequential writer) stays `fsync`-bound at about 9 ms/op. These are single-machine, in-process numbers and not a comparison with other databases. For end-to-end gRPC workloads use `beastbench` and follow the [benchmarking guide](benchmarking.md). + +--- + ## 4. Crash Recovery & Disaster Recovery RTO Measurements taken using `cmd/beastctl/backup_drill_test.go`: diff --git a/docs/runbooks/operator-guide.md b/docs/runbooks/operator-guide.md index 61da4e7..e80cff6 100644 --- a/docs/runbooks/operator-guide.md +++ b/docs/runbooks/operator-guide.md @@ -23,6 +23,10 @@ When bound to non-loopback interfaces or when `BEASTDB_ENV=production`, the serv ### Session Secret & API Tokens - Web sessions are authenticated via HMAC-SHA256 signatures backed by `BEASTDB_SESSION_SECRET` or persisted to `/session.key` (0600 file permissions). - gRPC requests require Bearer token authorization when `BEASTDB_API_TOKEN` is configured or API tokens exist in the token store. +- **Roles are enforced per gRPC method** (see [V1 Semantics §4](../v1-semantics.md)): `viewer` (read), `writer` (read and write), `admin` (everything), `replication` (replica streaming only). The static `--api-token` acts as `admin`. Give each application its own token with the smallest role it needs. +- **Followers must authenticate with a `replication` (or `admin`) token.** A viewer or writer token is rejected with `PERMISSION_DENIED` on `StreamWAL`/`Ack`. +- **Token expiry:** `POST /api/tokens` accepts an optional `expires_in_seconds` (0 or absent = never, maximum 10 years) and rejects an unknown `role` with HTTP 400. Studio does not yet expose an expiry field. +- **Upgrade note:** tokens created earlier default to `admin` and keep working. Tokens whose stored role is not one of the four roles now get `PERMISSION_DENIED` on every call; recreate them. --- diff --git a/docs/studio.md b/docs/studio.md index 22aaf8a..a4aa7de 100644 --- a/docs/studio.md +++ b/docs/studio.md @@ -86,6 +86,30 @@ Open `http://localhost:8088` in your browser and sign in with username `admin`. --- +## Design system and planned screens + +A redesign of Studio is specified in Figma (design only; `studio/` is unchanged). It targets people who are not database experts and follows the familiar Firebase/Supabase console layout: sidebar, collection list, records table, detail drawer. + +**Planned screens (10):** Table Editor, Overview, Query (visual builder that shows the equivalent `beastctl`/SDK call), Indexes, Replication, Backups, API Keys & Roles, Metrics, Logs & Audit, Settings. + +**Design tokens** (dark theme; Light mode is not designed): + +| Token | Value | Use | +|---|---|---| +| `accent/default` | `#A8F21A` (logo neon green; matches `brand.lime` in `tailwind.config.ts`) | primary actions, selection, lead chart line | +| `text/on-accent` | `#0A1002` | text on neon (contrast 14.1:1) | +| `bg/base`, `bg/panel`, `bg/raised` | `#0E120E`, `#151A15`, `#1C231B` | green-tinted dark surfaces | +| `border/default` | `#2A342A` | borders | +| `text/primary`, `text/muted` | `#F2F5F0`, `#93A08E` | text (16:1 and 6.4:1 on panel) | +| `status/success` | `#2DD4BF` (teal, so it is not confused with the accent) | healthy, active | +| `status/warning`, `status/danger`, `status/info` | `#FBBF24`, `#F87171`, `#60A5FA` | warnings, errors, info | + +Typography is Inter with Roboto Mono for code. Components: Button (Primary/Secondary/Danger), Status pill, Input, Nav item, Stat card, Toggle, Table row, Card. + +**Not yet backed by the engine:** encryption at rest, two-factor, collections/namespaces and the visual query builder, index suggestions, and a restore-to-new-database flow are shown in the design but are roadmap items (see [upgrade spec](upgrade-spec.md)). The mocks use placeholder data. + +--- + ## 🔥 Firebase-Style UI & Universal Capabilities - **3-Pane Collection Explorer**: Firebase Console style navigation: Left Partition Rail (with color-coded indicators) → Document List (with cover art/avatar badges) → Full-height Document Viewer. diff --git a/docs/upgrade-spec.md b/docs/upgrade-spec.md index ea293a1..2aace34 100644 --- a/docs/upgrade-spec.md +++ b/docs/upgrade-spec.md @@ -14,17 +14,27 @@ Source: review of 2026-11-12. Complements [ROADMAP.md](../ROADMAP.md) and [v1-se | Sec | Static token has no expiry/rotation | `grpc_auth.go` | | UX | uint64 keys only, no namespaces, one Python example, no /metrics | proto, `examples/` | -## Work packages (parallel, disjoint files) +## Work packages -1. **WP1 WAL group commit** (`internal/wal`, write path in `internal/api`): batch concurrent writers into one fsync, same durability. Target: >=10x concurrent write throughput. Must keep crash/fault tests green. -2. **WP2 gRPC role authorization** (`internal/api/grpc_auth*.go`, token store): per-method roles (viewer = Get/Scan; admin/writer = mutations; replication role for StreamWAL/Ack), token expiry. Deny by default. -3. **WP3 YCSB-style harness** (new `cmd/beastbench`): workloads A-F over gRPC, reports throughput and p50/p95/p99, reproducible flags, doc update. -4. **WP4 Prometheus /metrics** (`internal/web/handler`): request/error/latency, WAL size and sync latency, dirty pages, checkpoint age, LSN. +| WP | Scope | Status | +|---|---|---| +| WP1 | **WAL group commit** (`internal/wal`, `internal/api/engine_commit.go`): concurrent writers share one fsync with unchanged durability. Reported ~15x on a concurrent-write benchmark; a single sequential writer is unchanged. Includes a **WAL barrier** in the buffer pool so a data page never reaches disk ahead of its log record. | Merged | +| WP2 | **gRPC role authorization** (`internal/api/grpc_auth.go`, token store): deny-by-default per-method roles `viewer`/`writer`/`admin`/`replication`, optional token expiry. | Merged. No handler test yet for `expires_in_seconds`; Studio has no expiry field. | +| WP3 | **`beastbench`** (`cmd/beastbench`): YCSB-style workloads A–F over gRPC, latency histogram, JSON output, self-contained smoke mode, [benchmarking guide](benchmarking.md). | Merged. No competitor comparison has been run. | +| WP4 | **Prometheus `/metrics`**: request counts and latency, WAL size and LSN, checkpoint age, buffer pool, disk free. Never anonymous; accepts an admin session or a metrics/static bearer token. See the [operator guide](runbooks/operator-guide.md). | Merged | + +## Known limits of what shipped + +- Readers and replication are still blocked for the duration of each group's fsync (the engine keeps one global lock). Lifting this needs MVCC or latch coupling. +- The barrier can add an extra fsync when a dirty page is evicted under memory pressure. +- `LeaderServer.StreamWAL` replays the WAL file from disk and can briefly read a record that is appended but not yet fsynced. +- `TestEngineSystemWorkloadBenchmark` has a fixed 1000 ops/s floor and is timing-sensitive on a loaded machine. +- Checkpoint LSN/age metrics only appear after the first checkpoint since process start; the buffer pool hit ratio is cumulative. ## Next wave (not started) -Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log. +Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log, Studio expiry field and token-API handler tests. ## Studio dashboard design -Figma design for a Firestore/Supabase-style console: project overview, collections/table editor, SQL-free query bar, API keys and roles, metrics, backups. Aimed at non-experts. Tracked in Figma; implementation in `studio/`. +A Firestore/Supabase-style console for non-experts is designed in Figma; the implementation in `studio/` has not been changed. See [Studio design system](studio.md#design-system-and-planned-screens). diff --git a/docs/v1-semantics.md b/docs/v1-semantics.md index 366356c..b9615c7 100644 --- a/docs/v1-semantics.md +++ b/docs/v1-semantics.md @@ -17,7 +17,9 @@ ## 2. Operations & Execution Contracts ### `Put(key uint64, value []byte)` -* **Durability Guarantee:** Synchronous write-ahead log (`WAL`) persistence. A write is acknowledged only after `file.Sync()` (`fdatasync`) completes successfully on disk. +* **Durability Guarantee:** Synchronous write-ahead log (`WAL`) persistence. A write is acknowledged only after an `fsync` covering its WAL record completes successfully on disk. +* **Group commit:** Concurrent writers are queued and share one `fsync` per group. The durability guarantee is unchanged (no write is acknowledged before its record is durable, and readers and replication never observe a write before it is durable), but a group of N writers costs one `fsync` instead of N. A single sequential writer sees no speedup. +* **Failed fsync:** If the group `fsync` fails, the engine halts (fail-stop): every write in the group returns the error, later writes are rejected until restart, and the failure is sticky because retrying `fsync` after an error can report false success. * **Overwrite Semantics:** If `key` exists, the B+ Tree record pointer (`RID`) is atomically updated to the new slotted page location. * **Atomicity:** Single-key atomic mutation. @@ -61,7 +63,26 @@ --- -## 4. Supported Platform & Deployment Matrix +## 4. Authorization (gRPC) + +Authentication is a Bearer token: the static `--api-token` (treated as `admin`) or an API token from the token store. Authorization is **deny by default** per method: + +| Role | Allowed methods | +|---|---| +| `viewer` | `Get`, `Scan` | +| `writer` | `Get`, `Scan`, `Put`, `Delete`, `BatchWrite` | +| `admin` | all client methods and `ReplicationService` | +| `replication` | `ReplicationService.StreamWAL`, `ReplicationService.Ack` only | + +- A missing, malformed, unknown or expired token returns `UNAUTHENTICATED`. A valid token whose role may not call the method returns `PERMISSION_DENIED`. +- Any gRPC method not in the role map is denied for every role, including `admin` and the static token. Register new methods in `methodRoles` (`internal/api/grpc_auth.go`). +- A stored token with a role outside the four above can call nothing. +- Tokens may carry an optional expiry (`expires_at`); a token is rejected once the current time reaches it. Tokens created before expiry support never expire. +- With no static token and no token store, the interceptor is open (development only). + +--- + +## 5. Supported Platform & Deployment Matrix | Component | Supported Version / Target | Notes | |---|---|---| From 06b662e9bfda0245ec06f948298a4e8b282b3a5c Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:43:14 +0530 Subject: [PATCH 11/16] docs: add SECURITY.md with private vulnerability reporting policy Co-Authored-By: Claude Sonnet 5.5 --- CONTRIBUTING.md | 6 +++++ README.md | 1 + SECURITY.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 SECURITY.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b54c642..6e83d08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,12 @@ Thank you for your interest in contributing to BeastDB! We welcome contributions --- +## 🔒 Security Issues + +Do not report vulnerabilities in public issues or pull requests. Follow [SECURITY.md](SECURITY.md). + +--- + ## 🧪 Verification & Testing Before submitting a pull request, ensure all validations pass: diff --git a/README.md b/README.md index 169640d..c5ad977 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ BeastDB is optimized for **read-heavy OLTP** — deterministic point reads and o - [**V1 Semantics & Guarantees**](docs/v1-semantics.md) — Durability acknowledgment, limits, and replica reads. - [**System Benchmarks**](docs/benchmarks.md) — Reproducible throughput, tail latencies, and cache-eviction profiles. - [**Benchmarking Guide**](docs/benchmarking.md) — `beastbench` YCSB-style workloads and how to compare fairly with other databases. +- [**Security Policy**](SECURITY.md) — How to report a vulnerability privately, scope, and safe deployment checklist. - [**Upgrade Spec**](docs/upgrade-spec.md) — Security, usability and performance work packages and what is still planned. - [**Operator Guide**](docs/runbooks/operator-guide.md) — TLS configuration, cluster roles, manual failover, and `/healthz` probes. - [**Disaster Recovery**](docs/runbooks/disaster-recovery.md) — Physical backup drills, checksum verification, and RTO/RPO SLA. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..ebf6904 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,62 @@ +# Security Policy + +## Reporting a vulnerability + +**Please do not open a public issue, pull request or discussion for a security problem.** + +Report it privately through GitHub: + +1. Go to the repository's **Security** tab. +2. Choose **Report a vulnerability** (GitHub private vulnerability reporting). + +Include what you can of: + +- the affected version or commit and how you run BeastDB (flags, TLS, roles, replication); +- steps to reproduce, or a proof of concept; +- the impact you observed (for example unauthorised read or write, data corruption, denial of service); +- any suggested fix. + +If the **Report a vulnerability** button is missing, the maintainers have not enabled it yet. Open a public issue that says only "security contact needed" and contains no technical details, and a maintainer will arrange a private channel. + +### What to expect + +BeastDB is maintained by a small team, so these are goals, not guarantees: + +- acknowledgement of your report within about 5 working days; +- an assessment and a plan within about 14 days; +- a fix or documented mitigation as quickly as the severity allows, with coordinated public disclosure after a fix is available. We will credit you if you want to be credited. + +## Supported versions + +BeastDB has not reached a tagged V1 release (see [ROADMAP.md](ROADMAP.md)). Until then, security fixes are made on the `main` branch only. Once V1 is released, this section will list the supported release lines. + +## Scope + +In scope: + +- authentication and authorization (gRPC bearer tokens and roles, Studio sessions and cookies, the `/metrics` endpoint); +- transport security (TLS and mTLS for clients and replicas); +- the storage engine, WAL, recovery, backup and restore paths, where a flaw could lose or corrupt acknowledged data or expose data to the wrong caller; +- the replication stream; +- the container image and default configuration shipped in this repository. + +Out of scope: + +- findings that require the attacker to already have admin credentials, local root access, or write access to the data directory; +- running with `--dev` or `--insecure-auth` on a reachable network (these flags are documented as for local, non-production use); +- denial of service that needs a very large volume of traffic; +- missing hardening that is already listed as planned in [docs/upgrade-spec.md](docs/upgrade-spec.md) (for example encryption at rest). Reports on how a planned item could be exploited today are still welcome. + +## Deploying safely + +The [Operator Guide](docs/runbooks/operator-guide.md) is the reference. In short: + +- set a strong `BEASTDB_ADMIN_PASSWORD`; the server refuses the default password on non-loopback listeners and when `BEASTDB_ENV=production`; +- enable TLS (and mTLS for replicas) and keep gRPC and Studio off public interfaces; +- give each application its own API token with the smallest role it needs (`viewer`, `writer`, `admin`, `replication`) and set an expiry where you can; +- protect `/metrics` with a dedicated `BEASTDB_METRICS_TOKEN` and scrape it over TLS; +- keep backups off the host and test restores. + +## Safe harbour + +We will not pursue action against researchers who make a good-faith effort to follow this policy: test only against systems you own or have permission to test, avoid accessing or modifying other people's data, do not degrade service for others, and give us reasonable time to fix an issue before disclosing it. From 2cac2c9d771d5af700ebb51f73268cd3a44a80eb Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:45:59 +0530 Subject: [PATCH 12/16] refactor: split wal.go, buffer_pool.go and beastbench runner.go under the 200-line limit No behaviour change: functions moved verbatim into wal_sync.go, wal_lifecycle.go, buffer_pool_wal.go, buffer_pool_stats.go, runner_load.go and runner_run.go. Co-Authored-By: Claude Sonnet 5.5 --- cmd/beastbench/runner.go | 215 -------------------------- cmd/beastbench/runner_load.go | 72 +++++++++ cmd/beastbench/runner_run.go | 158 +++++++++++++++++++ internal/storage/buffer_pool.go | 41 ----- internal/storage/buffer_pool_stats.go | 24 +++ internal/storage/buffer_pool_wal.go | 19 +++ internal/wal/wal.go | 130 ---------------- internal/wal/wal_lifecycle.go | 57 +++++++ internal/wal/wal_sync.go | 79 ++++++++++ 9 files changed, 409 insertions(+), 386 deletions(-) create mode 100644 cmd/beastbench/runner_load.go create mode 100644 cmd/beastbench/runner_run.go create mode 100644 internal/storage/buffer_pool_stats.go create mode 100644 internal/storage/buffer_pool_wal.go create mode 100644 internal/wal/wal_lifecycle.go create mode 100644 internal/wal/wal_sync.go diff --git a/cmd/beastbench/runner.go b/cmd/beastbench/runner.go index 69cd245..197ee7a 100644 --- a/cmd/beastbench/runner.go +++ b/cmd/beastbench/runner.go @@ -1,19 +1,8 @@ package main import ( - "context" "encoding/binary" - "fmt" - "io" - "math" - "math/rand" - "sync" - "sync/atomic" "time" - - beastv1 "github.com/ChromaBeast/beastdb/api/proto" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) // Config holds everything that determines a run. @@ -87,210 +76,6 @@ func newCounters() *counters { return c } -// Load inserts cfg.Records records with the given parallelism. -func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { - cs := make([]*counters, cfg.Threads) - var wg sync.WaitGroup - var firstErr atomic.Value - start := time.Now() - per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) - for t := 0; t < cfg.Threads; t++ { - cs[t] = newCounters() - wg.Add(1) - go func(t int) { - defer wg.Done() - c := cs[t] - client := cl[t%len(cl)] - buf := make([]byte, cfg.ValueSize) - lo := uint64(t) * per - hi := lo + per - if hi > cfg.Records { - hi = cfg.Records - } - for i := lo; i < hi; i++ { - if ctx.Err() != nil { - return - } - makeValue(buf, i, 0) - t0 := time.Now() - _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) - c.hist[OpInsert].Record(time.Since(t0)) - if err != nil { - c.errs[OpInsert]++ - firstErr.CompareAndSwap(nil, err) - return - } - } - }(t) - } - wg.Wait() - el := time.Since(start) - res := summarize("load", el, cs) - if e, ok := firstErr.Load().(error); ok { - return res, fmt.Errorf("load failed: %w", e) - } - return res, ctx.Err() -} - -// callPut returns conflict=true when a conditional write was rejected (ABORTED). -func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) - if err != nil { - if status.Code(err) == codes.Aborted { - return true, nil - } - return false, err - } - return false, nil -} - -// Run executes warmup (unrecorded) then the measured run phase. -func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { - w, err := lookupWorkload(cfg.Workload) - if err != nil { - return PhaseResult{}, err - } - frontier := &atomic.Uint64{} - frontier.Store(cfg.Records) - - var chooser KeyChooser - switch { - case w.Latest: - chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} - case cfg.Dist == "uniform": - chooser = Uniform{N: cfg.Records} - default: - chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) - } - - phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { - cs := make([]*counters, cfg.Threads) - var wg sync.WaitGroup - start := time.Now() - for t := 0; t < cfg.Threads; t++ { - cs[t] = newCounters() - n := total / uint64(cfg.Threads) - if uint64(t) < total%uint64(cfg.Threads) { - n++ - } - wg.Add(1) - go func(t int, n uint64) { - defer wg.Done() - r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) - worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) - }(t, n) - } - wg.Wait() - return cs, time.Since(start) - } - - if cfg.WarmupOps > 0 { - phase(cfg.WarmupOps, 1_000_003, false) - } - cs, el := phase(cfg.Ops, 0, true) - return summarize("run", el, cs), ctx.Err() -} - -func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, - ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { - buf := make([]byte, cfg.ValueSize) - for i := uint64(0); i < n && ctx.Err() == nil; i++ { - op := w.pick(r.Float64()) - t0 := time.Now() - var err error - var miss, conflict bool - switch op { - case OpRead: - idx := ch.Next(r) - var found bool - found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) - miss = err == nil && !found - case OpUpdate: - idx := ch.Next(r) - r.Read(buf) - conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) - case OpInsert: - idx := frontier.Add(1) - 1 - makeValue(buf, idx, 0) - conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) - case OpScan: - idx := ch.Next(r) - ln := 1 + r.Intn(cfg.MaxScanLen) - var got int - got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) - miss = err == nil && got == 0 - case OpRMW: - idx := ch.Next(r) - var old []byte - var found bool - found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) - if err == nil && !found { - miss = true - break - } - if err == nil { - nv := append([]byte(nil), old...) - if len(nv) > 0 { - nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) - } - conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ - Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) - } - } - if ctx.Err() != nil { - return - } - if !record { - continue - } - c.hist[op].Record(time.Since(t0)) - if err != nil { - c.errs[op]++ - } - if miss { - c.misses[op]++ - } - if conflict { - c.conflicts[op]++ - } - } -} - -func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) - if err != nil { - return false, err - } - if out != nil { - *out = resp.Value - } - return resp.Found, nil -} - -func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) - if err != nil { - return 0, err - } - n := 0 - for { - _, err := st.Recv() - if err == io.EOF { - return n, nil - } - if err != nil { - return n, err - } - n++ - } -} - func summarize(name string, el time.Duration, cs []*counters) PhaseResult { res := PhaseResult{Phase: name, Seconds: el.Seconds()} for op := OpType(0); op < numOps; op++ { diff --git a/cmd/beastbench/runner_load.go b/cmd/beastbench/runner_load.go new file mode 100644 index 0000000..0bd5757 --- /dev/null +++ b/cmd/beastbench/runner_load.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Load inserts cfg.Records records with the given parallelism. +func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + var firstErr atomic.Value + start := time.Now() + per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + wg.Add(1) + go func(t int) { + defer wg.Done() + c := cs[t] + client := cl[t%len(cl)] + buf := make([]byte, cfg.ValueSize) + lo := uint64(t) * per + hi := lo + per + if hi > cfg.Records { + hi = cfg.Records + } + for i := lo; i < hi; i++ { + if ctx.Err() != nil { + return + } + makeValue(buf, i, 0) + t0 := time.Now() + _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) + c.hist[OpInsert].Record(time.Since(t0)) + if err != nil { + c.errs[OpInsert]++ + firstErr.CompareAndSwap(nil, err) + return + } + } + }(t) + } + wg.Wait() + el := time.Since(start) + res := summarize("load", el, cs) + if e, ok := firstErr.Load().(error); ok { + return res, fmt.Errorf("load failed: %w", e) + } + return res, ctx.Err() +} + +// callPut returns conflict=true when a conditional write was rejected (ABORTED). +func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) + if err != nil { + if status.Code(err) == codes.Aborted { + return true, nil + } + return false, err + } + return false, nil +} diff --git a/cmd/beastbench/runner_run.go b/cmd/beastbench/runner_run.go new file mode 100644 index 0000000..73e0030 --- /dev/null +++ b/cmd/beastbench/runner_run.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "io" + "math" + "math/rand" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" +) + +// Run executes warmup (unrecorded) then the measured run phase. +func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + w, err := lookupWorkload(cfg.Workload) + if err != nil { + return PhaseResult{}, err + } + frontier := &atomic.Uint64{} + frontier.Store(cfg.Records) + + var chooser KeyChooser + switch { + case w.Latest: + chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} + case cfg.Dist == "uniform": + chooser = Uniform{N: cfg.Records} + default: + chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) + } + + phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + start := time.Now() + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + n := total / uint64(cfg.Threads) + if uint64(t) < total%uint64(cfg.Threads) { + n++ + } + wg.Add(1) + go func(t int, n uint64) { + defer wg.Done() + r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) + worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) + }(t, n) + } + wg.Wait() + return cs, time.Since(start) + } + + if cfg.WarmupOps > 0 { + phase(cfg.WarmupOps, 1_000_003, false) + } + cs, el := phase(cfg.Ops, 0, true) + return summarize("run", el, cs), ctx.Err() +} + +func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, + ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { + buf := make([]byte, cfg.ValueSize) + for i := uint64(0); i < n && ctx.Err() == nil; i++ { + op := w.pick(r.Float64()) + t0 := time.Now() + var err error + var miss, conflict bool + switch op { + case OpRead: + idx := ch.Next(r) + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) + miss = err == nil && !found + case OpUpdate: + idx := ch.Next(r) + r.Read(buf) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpInsert: + idx := frontier.Add(1) - 1 + makeValue(buf, idx, 0) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpScan: + idx := ch.Next(r) + ln := 1 + r.Intn(cfg.MaxScanLen) + var got int + got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) + miss = err == nil && got == 0 + case OpRMW: + idx := ch.Next(r) + var old []byte + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) + if err == nil && !found { + miss = true + break + } + if err == nil { + nv := append([]byte(nil), old...) + if len(nv) > 0 { + nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) + } + conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ + Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) + } + } + if ctx.Err() != nil { + return + } + if !record { + continue + } + c.hist[op].Record(time.Since(t0)) + if err != nil { + c.errs[op]++ + } + if miss { + c.misses[op]++ + } + if conflict { + c.conflicts[op]++ + } + } +} + +func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) + if err != nil { + return false, err + } + if out != nil { + *out = resp.Value + } + return resp.Found, nil +} + +func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) + if err != nil { + return 0, err + } + n := 0 + for { + _, err := st.Recv() + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + n++ + } +} diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index f29df84..7d1c176 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -33,24 +33,6 @@ type BufferPoolManager struct { walBarrier func() error } -// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. -func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { - bpm.mu.Lock() - defer bpm.mu.Unlock() - bpm.walBarrier = fn -} - -// writeBack writes one dirty frame to the data file after the WAL barrier. -// Caller must hold bpm.mu (write lock). -func (bpm *BufferPoolManager) writeBack(f *Frame) error { - if bpm.walBarrier != nil { - if err := bpm.walBarrier(); err != nil { - return err - } - } - return bpm.disk.WritePage(f.PageID, f.Page.Data()) -} - // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. func NewBufferPoolManager(disk *DiskManager, poolSize int) *BufferPoolManager { frames := make([]*Frame, poolSize) @@ -180,26 +162,3 @@ func (bpm *BufferPoolManager) FlushAll() error { } return bpm.disk.Sync() } - -// HitMiss returns cumulative FetchPage cache hits and misses. -func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { - return bpm.hits.Load(), bpm.misses.Load() -} - -// Stats returns operational metrics for the buffer pool. -func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { - bpm.mu.RLock() - defer bpm.mu.RUnlock() - - poolSize = bpm.poolSize - cachedPages = len(bpm.pageTable) - for _, frame := range bpm.frames { - if frame.PinCount() > 0 { - pinnedPages++ - } - if frame.IsDirty { - dirtyPages++ - } - } - return -} diff --git a/internal/storage/buffer_pool_stats.go b/internal/storage/buffer_pool_stats.go new file mode 100644 index 0000000..3fae07a --- /dev/null +++ b/internal/storage/buffer_pool_stats.go @@ -0,0 +1,24 @@ +package storage + +// HitMiss returns cumulative FetchPage cache hits and misses. +func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { + return bpm.hits.Load(), bpm.misses.Load() +} + +// Stats returns operational metrics for the buffer pool. +func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { + bpm.mu.RLock() + defer bpm.mu.RUnlock() + + poolSize = bpm.poolSize + cachedPages = len(bpm.pageTable) + for _, frame := range bpm.frames { + if frame.PinCount() > 0 { + pinnedPages++ + } + if frame.IsDirty { + dirtyPages++ + } + } + return +} diff --git a/internal/storage/buffer_pool_wal.go b/internal/storage/buffer_pool_wal.go new file mode 100644 index 0000000..9ba7820 --- /dev/null +++ b/internal/storage/buffer_pool_wal.go @@ -0,0 +1,19 @@ +package storage + +// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. +func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { + bpm.mu.Lock() + defer bpm.mu.Unlock() + bpm.walBarrier = fn +} + +// writeBack writes one dirty frame to the data file after the WAL barrier. +// Caller must hold bpm.mu (write lock). +func (bpm *BufferPoolManager) writeBack(f *Frame) error { + if bpm.walBarrier != nil { + if err := bpm.walBarrier(); err != nil { + return err + } + } + return bpm.disk.WritePage(f.PageID, f.Page.Data()) +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 7261209..57f1860 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -3,7 +3,6 @@ package wal import ( "io" "os" - "path/filepath" "sync" ) @@ -110,84 +109,6 @@ func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { // SyncOnWrite reports whether this log was opened with fsync-per-commit semantics. func (w *WAL) SyncOnWrite() bool { return w.syncOnWrite } -func (w *WAL) loadSyncErr() error { - w.errMu.Lock() - defer w.errMu.Unlock() - return w.syncErr -} - -// WaitDurable blocks until the record with the given LSN has been fsynced. -// If another caller's fsync already covered it, no further fsync is issued. -func (w *WAL) WaitDurable(lsn uint64) error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - if err := w.loadSyncErr(); err != nil { - return err - } - if w.syncedLSN >= lsn { - return nil - } - return w.syncLocked() -} - -// syncLocked fsyncs everything appended so far. Caller holds syncMu. -func (w *WAL) syncLocked() error { - w.mu.Lock() - target := w.currentLSN - f := w.file - w.mu.Unlock() - - // Appends may proceed while fsync runs; they simply miss this sync and are - // covered by the next one. target only claims what was written before it. - if err := f.Sync(); err != nil { - w.errMu.Lock() - w.syncErr = err - w.errMu.Unlock() - return err - } - if target > w.syncedLSN { - w.syncedLSN = target - } - return nil -} - -// Sync forces all buffered kernel writes to physical non-volatile storage. -func (w *WAL) Sync() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - if err := w.loadSyncErr(); err != nil { - return err - } - return w.syncLocked() -} - -// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. -func (w *WAL) SyncIfEnabled() error { - if !w.syncOnWrite { - return nil - } - return w.Sync() -} - -// EnsureDurable returns once every record appended so far is fsynced. It is a -// no-op when the log was not opened with syncOnWrite, and costs no fsync when -// a previous sync already covers the current LSN. The buffer pool calls it -// before writing a dirty page so a data page never reaches disk ahead of the -// WAL records that describe it. -func (w *WAL) EnsureDurable() error { - if !w.syncOnWrite { - return nil - } - return w.WaitDurable(w.CurrentLSN()) -} - -// DurableLSN returns the highest LSN known to be fsynced. -func (w *WAL) DurableLSN() uint64 { - w.syncMu.Lock() - defer w.syncMu.Unlock() - return w.syncedLSN -} - // CurrentLSN returns the latest assigned Log Sequence Number. func (w *WAL) CurrentLSN() uint64 { w.mu.Lock() @@ -211,57 +132,6 @@ func (w *WAL) Path() string { return w.path } -// Close flushes buffered writes and closes the log file. -func (w *WAL) Close() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - w.mu.Lock() - defer w.mu.Unlock() - _ = w.file.Sync() - return w.file.Close() -} - -// Checkpoint rotates the WAL: syncs the current file, archives it as .bak, -// then opens a fresh append-only log starting from the next LSN. -// The caller MUST have flushed all dirty buffer pool pages to disk first so the -// archived WAL records are no longer needed for recovery. -func (w *WAL) Checkpoint() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - w.mu.Lock() - defer w.mu.Unlock() - - if err := w.file.Sync(); err != nil { - return err - } - // Everything appended so far is now durable (and about to be archived). - w.syncedLSN = w.currentLSN - if err := w.file.Close(); err != nil { - return err - } - - // Archive the old WAL; keep one generation for safety. - archivePath := w.path + ".bak" - _ = os.Remove(archivePath) - if err := os.Rename(w.path, archivePath); err != nil { - return err - } - - // Open a fresh log file for subsequent writes. - newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) - if err != nil { - return err - } - w.file = newFile - - // Persist the rename and the new file's directory entry. - if dir, err := os.Open(filepath.Dir(w.path)); err == nil { - _ = dir.Sync() - _ = dir.Close() - } - return nil -} - func findHighestLSN(file *os.File) (uint64, error) { if _, err := file.Seek(0, io.SeekStart); err != nil { return 0, err diff --git a/internal/wal/wal_lifecycle.go b/internal/wal/wal_lifecycle.go new file mode 100644 index 0000000..5de4ae0 --- /dev/null +++ b/internal/wal/wal_lifecycle.go @@ -0,0 +1,57 @@ +package wal + +import ( + "os" + "path/filepath" +) + +// Close flushes buffered writes and closes the log file. +func (w *WAL) Close() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + w.mu.Lock() + defer w.mu.Unlock() + _ = w.file.Sync() + return w.file.Close() +} + +// Checkpoint rotates the WAL: syncs the current file, archives it as .bak, +// then opens a fresh append-only log starting from the next LSN. +// The caller MUST have flushed all dirty buffer pool pages to disk first so the +// archived WAL records are no longer needed for recovery. +func (w *WAL) Checkpoint() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + w.mu.Lock() + defer w.mu.Unlock() + + if err := w.file.Sync(); err != nil { + return err + } + // Everything appended so far is now durable (and about to be archived). + w.syncedLSN = w.currentLSN + if err := w.file.Close(); err != nil { + return err + } + + // Archive the old WAL; keep one generation for safety. + archivePath := w.path + ".bak" + _ = os.Remove(archivePath) + if err := os.Rename(w.path, archivePath); err != nil { + return err + } + + // Open a fresh log file for subsequent writes. + newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) + if err != nil { + return err + } + w.file = newFile + + // Persist the rename and the new file's directory entry. + if dir, err := os.Open(filepath.Dir(w.path)); err == nil { + _ = dir.Sync() + _ = dir.Close() + } + return nil +} diff --git a/internal/wal/wal_sync.go b/internal/wal/wal_sync.go new file mode 100644 index 0000000..c8f40bc --- /dev/null +++ b/internal/wal/wal_sync.go @@ -0,0 +1,79 @@ +package wal + +func (w *WAL) loadSyncErr() error { + w.errMu.Lock() + defer w.errMu.Unlock() + return w.syncErr +} + +// WaitDurable blocks until the record with the given LSN has been fsynced. +// If another caller's fsync already covered it, no further fsync is issued. +func (w *WAL) WaitDurable(lsn uint64) error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + if w.syncedLSN >= lsn { + return nil + } + return w.syncLocked() +} + +// syncLocked fsyncs everything appended so far. Caller holds syncMu. +func (w *WAL) syncLocked() error { + w.mu.Lock() + target := w.currentLSN + f := w.file + w.mu.Unlock() + + // Appends may proceed while fsync runs; they simply miss this sync and are + // covered by the next one. target only claims what was written before it. + if err := f.Sync(); err != nil { + w.errMu.Lock() + w.syncErr = err + w.errMu.Unlock() + return err + } + if target > w.syncedLSN { + w.syncedLSN = target + } + return nil +} + +// Sync forces all buffered kernel writes to physical non-volatile storage. +func (w *WAL) Sync() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + return w.syncLocked() +} + +// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. +func (w *WAL) SyncIfEnabled() error { + if !w.syncOnWrite { + return nil + } + return w.Sync() +} + +// EnsureDurable returns once every record appended so far is fsynced. It is a +// no-op when the log was not opened with syncOnWrite, and costs no fsync when +// a previous sync already covers the current LSN. The buffer pool calls it +// before writing a dirty page so a data page never reaches disk ahead of the +// WAL records that describe it. +func (w *WAL) EnsureDurable() error { + if !w.syncOnWrite { + return nil + } + return w.WaitDurable(w.CurrentLSN()) +} + +// DurableLSN returns the highest LSN known to be fsynced. +func (w *WAL) DurableLSN() uint64 { + w.syncMu.Lock() + defer w.syncMu.Unlock() + return w.syncedLSN +} From ade5d0ae395b215ebdf0217ade2f3c75ccef011d Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:49:18 +0530 Subject: [PATCH 13/16] docs: replace hard 200-line limit with a one-responsibility-per-file guideline About 400 lines is a prompt to ask whether a file does more than one thing, not a gate. Generated code, tests and data are exempt. Updates GEMINI.md (Antigravity), CONTRIBUTING.md and demo.ps1, and adds AGENTS.md (Codex) and CLAUDE.md (imports AGENTS.md). Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 26 ++++++++++++++++++++++++++ CLAUDE.md | 1 + CONTRIBUTING.md | 14 ++++++++------ GEMINI.md | 9 ++++++--- demo.ps1 | 18 +++++++++--------- 5 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..cb8f16f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# BeastDB: instructions for coding agents + +Read by Codex and other agents that use `AGENTS.md`. Claude Code reads it through `CLAUDE.md`. Antigravity also reads `GEMINI.md`, which carries the same rules; keep the two in step. + +## File modularity (guideline, not a hard limit) + +- One responsibility per file. About **400 lines** is the point to stop and ask "does this file do more than one thing?" If yes, split along that seam (for example sync vs. lifecycle, load vs. run). If no, leave it alone. +- Never split a cohesive file, or merge small files, just to satisfy a line count. +- Exempt: generated code (`*.pb.go`), test files, JSON and raw data. +- Do not add CI or lint checks that fail on line count. A non-exempt file past about 500 lines should say why in the change description. + +## Engine principles + +- **WAL before page:** a data page must never reach the data file before the WAL records that describe it are durable. The buffer pool enforces this with the WAL barrier (`WAL.EnsureDurable`); keep every dirty-page write on that path. +- **Zero-allocation hot paths:** query, hashing, parsing and buffer-pinning paths avoid heap allocations; prefer byte slices and `sync.Pool`. +- **Mechanical sympathy:** align structs to avoid false sharing, prefer flat memory layouts, respect 4 KB page alignment. + +## Scope and honesty + +- BeastDB is a general-purpose, domain-agnostic database. Do not hard-code schema or domain rules from downstream applications; client-specific partition metadata belongs in `partitions.json` or `-partition-config`. +- Do not claim performance relative to other databases without an equivalent-workload benchmark (see `docs/benchmarking.md`). Describe planned features as planned. +- Report vulnerabilities privately; see `SECURITY.md`. + +## Before you finish + +Run `go build ./... && go vet ./... && go test ./... -race`. `TestEngineSystemWorkloadBenchmark` has a fixed 1000 ops/s floor and can fail when the machine is loaded; re-run it alone before treating a failure as a regression. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e83d08..7246279 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,9 +6,11 @@ Thank you for your interest in contributing to BeastDB! We welcome contributions ## 🏛️ Engineering Standards -### 1. Strict File Modularity (< 200 LoC) -- **Constraint:** Every source file (excluding auto-generated `.pb.go` code) must strictly remain readable and **under 200 Lines of Code (LoC)**. -- **Decomposition:** Large subsystems (B+ Tree, Buffer Pool, Hash Table, WAL) must be decomposed into focused, single-responsibility files and companion helpers. +### 1. File Modularity (guideline, not a hard limit) +- **Principle:** One responsibility per file. Treat about **400 lines** as a prompt to ask "does this file do more than one thing?" If it does, split along that seam. If it doesn't, leave it as is. Never split, or merge, files only to hit a number. +- **Exempt:** generated code (`*.pb.go`), test files, and data files (JSON and similar). +- **Review, not CI:** there is no automated line-count gate. A non-exempt file past about 500 lines should say in the pull request why it stays one file. +- **Decomposition:** Large subsystems (B+ Tree, Buffer Pool, Hash Table, WAL) should be organized into focused, single-responsibility files and companion helpers. ### 2. Zero-Allocation Hot Paths - Query execution, hash calculation, parsing, and buffer pinning paths must not trigger unnecessary heap allocations. @@ -38,9 +40,9 @@ go test -v -count=1 ./... # 2. Run benchmarks to verify zero allocations on hot paths go test -bench=. -benchmem ./... -# 3. Enforce the < 200 LoC modularity limit -Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.FullName -notmatch "\.pb\.go" } | ForEach-Object { +# 3. (Optional) List large non-test, non-generated files to review. Informational only. +Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.Name -notmatch "\.pb\.go$|_test\.go$" } | ForEach-Object { $lines = (Get-Content $_.FullName).Count - if ($lines -ge 200) { Write-Error "$lines lines in $($_.Name)" } + if ($lines -ge 400) { Write-Host "$lines lines in $($_.FullName): does it do more than one thing?" } } ``` diff --git a/GEMINI.md b/GEMINI.md index 3ffbbe1..6150a7e 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -4,9 +4,12 @@ These rules govern all interactions, coding, and architecture in this project. --- -## 1. File Modularity & Readability (< 200 LoC) -* **Strict File Length Limit:** In any project, if a file is not JSON or raw data, it must strictly remain readable and **< 200 Lines of Code (LoC)**. -* **Component Decomposition:** Decompose complex systems (B+ trees, buffer pools, hash tables, page managers) into small, single-responsibility files, structs, and reusable components. +## 1. File Modularity & Readability (guideline, not a hard limit) +* **One responsibility per file.** Organize files by what they do. File length is a smell, not a rule: roughly **400 lines** is the point to stop and ask "does this file do more than one thing?" If yes, split it along that seam (for example sync vs. lifecycle, load vs. run). If no, leave it alone. +* **Never split just to hit a number.** Do not cut a cohesive file into arbitrary pieces, move code without a clear responsibility boundary, or merge small files back together to satisfy a count. +* **Exemptions:** generated code (`*.pb.go`), test files, and JSON/raw data have no size expectation. +* **No automated gate.** Do not add CI or lint checks that fail on line count. Past about 500 lines, mention the reason in the change description or review. +* **Component Decomposition:** Decompose complex systems (B+ trees, buffer pools, hash tables, page managers) into focused structs and reusable components. --- diff --git a/demo.ps1 b/demo.ps1 index 9dfe871..435fa20 100644 --- a/demo.ps1 +++ b/demo.ps1 @@ -62,21 +62,21 @@ Write-Host ' [OK] Corrupted disk blocks detected via IEEE CRC32 checksum' -Fore Write-Host ' [OK] Clean WAL recovery and state restoration verified' -ForegroundColor Green Start-Sleep -Seconds 3 -# ACT 3: Code Modularity & Quality Audit (< 200 LoC) -Print-Section "ACT 3" "Strict Architectural Modularity (< 200 LoC per file)" -Simulate-Typing 'audit-codebase --enforce-max-loc 200' -Write-Host " -> Scanning all Go source files across all packages..." -ForegroundColor Gray +# ACT 3: Code Modularity Review (guideline: one responsibility per file) +Print-Section "ACT 3" "Architectural Modularity (one responsibility per file)" +Simulate-Typing 'audit-codebase --review-above 400' +Write-Host " -> Scanning Go source files (generated and test files exempt)..." -ForegroundColor Gray Start-Sleep -Milliseconds 400 -$violations = Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.FullName -notmatch "\.pb\.go" } | ForEach-Object { +$large = Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.Name -notmatch "\.pb\.go$|_test\.go$" } | ForEach-Object { $lines = (Get-Content $_.FullName).Count - if ($lines -ge 200) { "$lines lines: $($_.Name)" } + if ($lines -ge 400) { "$lines lines: $($_.Name)" } } -if ($violations.Count -eq 0) { - Write-Host ' [SUCCESS] AUDIT PASSED: ZERO files exceed 200 lines of code!' -ForegroundColor Green +if ($large.Count -eq 0) { + Write-Host ' [OK] No source file is large enough to need a modularity review' -ForegroundColor Green } else { - $violations | ForEach-Object { Write-Host " [FAIL] VIOLATION: $_" -ForegroundColor Red } + $large | ForEach-Object { Write-Host " [REVIEW] $_ - does it do more than one thing?" -ForegroundColor Yellow } } Start-Sleep -Seconds 2 From c6b851e68293904897925758c078fe33c13a17a6 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:53:16 +0530 Subject: [PATCH 14/16] docs: update phase checklists to the file-modularity guideline Co-Authored-By: Claude Sonnet 5.5 --- docs/phase1_dsa.md | 2 +- docs/phase2_net_cache.md | 2 +- docs/phase3_storage.md | 2 +- docs/phase4_scaling.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/phase1_dsa.md b/docs/phase1_dsa.md index d83f92f..8e47ab8 100644 --- a/docs/phase1_dsa.md +++ b/docs/phase1_dsa.md @@ -67,5 +67,5 @@ Focuses on low-level memory representation, zero-copy byte semantics, and pure G ## Success Verification - [ ] 0 allocations per operation on hot lookup paths. -- [ ] All data structure files adhere to the strict `< 200 LoC` limit. +- [ ] Each data structure file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). - [ ] Comprehensive unit test coverage (>90%) with fuzz tests for edge cases. diff --git a/docs/phase2_net_cache.md b/docs/phase2_net_cache.md index 0ead3d3..228e9aa 100644 --- a/docs/phase2_net_cache.md +++ b/docs/phase2_net_cache.md @@ -73,4 +73,4 @@ Builds the network communication layer, protocol parsing engines, and concurrent - [ ] Connect with standard `redis-cli` and execute `SET`, `GET`, `DEL`, `PING`. - [ ] Handle 10,000 concurrent client connections without deadlocks or goroutine leaks. - [ ] LRU evicts least recently accessed keys accurately under memory pressure. -- [ ] All code files remain strictly under 200 LoC. +- [ ] Each code file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). diff --git a/docs/phase3_storage.md b/docs/phase3_storage.md index ded9acd..9a1dc9c 100644 --- a/docs/phase3_storage.md +++ b/docs/phase3_storage.md @@ -73,4 +73,4 @@ B+ Tree nodes serialized directly inside 4KB slotted pages. - [ ] Process recovery test: Crash during active load; re-open DB and verify 100% data consistency via WAL replay. - [ ] Sequential scan matches exact key ordering across 100,000 inserted keys. - [ ] Zero unpinned frame leaks under heavy concurrent read/write transactions. -- [ ] All source files modularized to `< 200 LoC`. +- [ ] Each source file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). diff --git a/docs/phase4_scaling.md b/docs/phase4_scaling.md index 464a389..3cd4c2f 100644 --- a/docs/phase4_scaling.md +++ b/docs/phase4_scaling.md @@ -76,4 +76,4 @@ Expands the database engine into a production-ready distributed system with gRPC - [ ] Container image build succeeds under 25MB total footprint. - [ ] Server cleanly flushes and terminates on `SIGTERM` without corrupting active pages. - [ ] Sustained 50k+ QPS with p99 latency under 2ms. -- [ ] All source files modularized to `< 200 LoC`. +- [ ] Each source file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). From a52bb001d2b6421ea99bcd557cd6c8632036eadb Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 17:20:30 +0530 Subject: [PATCH 15/16] test(metrics): only require disk-free gauge where the platform reports it DiskFreeBytes is unix-only, so TestMetricsAuthAndFormat failed on Windows CI with a missing beastdb_disk_free_bytes line. Require the gauge only when DiskFreeBytes reports support. Co-Authored-By: Claude Sonnet 5.5 --- internal/web/handler/metrics_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/web/handler/metrics_test.go b/internal/web/handler/metrics_test.go index 7b04a64..5e8d651 100644 --- a/internal/web/handler/metrics_test.go +++ b/internal/web/handler/metrics_test.go @@ -119,14 +119,19 @@ func TestMetricsAuthAndFormat(t *testing.T) { } body := rec.Body.String() validateExposition(t, body) - for _, want := range []string{ + wants := []string{ `beastdb_build_info{version="v\"1",role="follower"} 1`, "beastdb_wal_size_bytes ", "beastdb_wal_current_lsn ", "beastdb_buffer_pool_dirty_pages ", "beastdb_buffer_pool_pinned_pages ", "beastdb_replica_applied_lsn 7", `beastdb_requests_total{protocol="grpc",method="/beastdb.v1.KV/Get",code="NotFound"} 1`, `beastdb_request_duration_seconds_bucket{protocol="grpc",method="/beastdb.v1.KV/Get",le="+Inf"} 2`, - "beastdb_disk_free_bytes ", - } { + } + // Disk-free is only reported on platforms that support it (unix); it is + // omitted elsewhere, e.g. Windows. + if _, ok := metrics.DiskFreeBytes("."); ok { + wants = append(wants, "beastdb_disk_free_bytes ") + } + for _, want := range wants { if !strings.Contains(body, want) { t.Errorf("missing %q in output", want) } From d4667194d09ed8e1a8ea881ba3c7baf043e9b38e Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 17:20:30 +0530 Subject: [PATCH 16/16] perf(engine): let the first writer lead its own commit group Instead of waking a dedicated committer goroutine for every write, the writer that finds no commit in progress runs the queued group on its own goroutine; if more writers queued meanwhile, a drain goroutine serves them. Group commit semantics are unchanged (one fsync per group, lock held across it, observer in LSN order). Aims to remove a cross-goroutine wake-up per uncontended write, which can cost up to a timer tick on Windows. Not yet verified on Windows. Co-Authored-By: Claude Sonnet 5.5 --- internal/api/engine_commit.go | 116 +++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/internal/api/engine_commit.go b/internal/api/engine_commit.go index 77b5c55..21788d3 100644 --- a/internal/api/engine_commit.go +++ b/internal/api/engine_commit.go @@ -23,21 +23,26 @@ type writeReq struct { ok bool // run appended a WAL record (so it depends on the batch fsync) } -// committer is the group-commit queue. A single goroutine drains it: each pass -// takes every queued write, runs them back to back under the exclusive engine -// lock (appending to the WAL without fsync), issues ONE fsync for the whole -// group, notifies the replication observer in LSN order, and only then -// releases the lock and acknowledges the writers. Because the lock is held -// across the fsync, readers and the observer never see a write before it is -// durable, exactly as with fsync-per-write; the gain is that N queued writers -// pay for one fsync instead of N. +// committer is the group-commit queue. Writers enqueue a request; whichever +// writer finds no commit in progress becomes the leader and runs the queued +// group itself, on its own goroutine. This avoids a cross-goroutine wake-up +// for an uncontended write, which costs up to a timer tick on Windows. If more +// writers queued while the leader was busy, the leader hands the remaining work +// to a drain goroutine and returns to its caller. +// +// A group runs back to back under the exclusive engine lock (appending to the +// WAL without fsync), issues ONE fsync for the whole group, notifies the +// replication observer in LSN order, and only then releases the lock and +// acknowledges the writers. Because the lock is held across the fsync, readers +// and the observer never see a write before it is durable, exactly as with +// fsync-per-write; the gain is that N queued writers pay for one fsync. +// Only one group runs at a time, so queue order is LSN order. type committer struct { mu sync.Mutex queue []*writeReq - started bool + running bool // a leader or drain goroutine owns the queue closed bool - wake chan struct{} - done chan struct{} + idle *sync.Cond // signalled (on mu) when running becomes false } // commit runs fn through the group-commit pipeline and returns once its WAL @@ -53,59 +58,80 @@ func (e *Engine) commit(fn func() error) error { c.mu.Unlock() return ErrEngineClosed } - if !c.started { - c.started = true - c.wake = make(chan struct{}, 1) - c.done = make(chan struct{}) - go e.commitLoop() - } c.queue = append(c.queue, req) + lead := !c.running + if lead { + c.running = true + } c.mu.Unlock() - select { - case c.wake <- struct{}{}: - default: + if lead { + e.lead() } <-req.done return req.err } -func (e *Engine) commitLoop() { +// takeBatch removes and returns the queued writes. When the queue is empty it +// clears running (releasing leadership) and returns nil. +func (c *committer) takeBatch() []*writeReq { + c.mu.Lock() + defer c.mu.Unlock() + batch := c.queue + c.queue = nil + if len(batch) == 0 { + c.running = false + if c.idle != nil { + c.idle.Broadcast() + } + } + return batch +} + +// lead runs one group (which includes the leader's own request) on the +// caller's goroutine. If more writers arrived meanwhile, a drain goroutine +// keeps serving them so the leader's caller is not delayed. +func (e *Engine) lead() { + c := &e.cq + e.runBatch(c.takeBatch()) + c.mu.Lock() + more := len(c.queue) > 0 + if !more { + c.running = false + if c.idle != nil { + c.idle.Broadcast() + } + } + c.mu.Unlock() + if more { + go e.drain() + } +} + +// drain serves queued writes until the queue is empty, then releases leadership. +func (e *Engine) drain() { c := &e.cq - defer close(c.done) for { - <-c.wake - for { - c.mu.Lock() - batch := c.queue - c.queue = nil - closed := c.closed - c.mu.Unlock() - if len(batch) == 0 { - if closed { - return - } - break - } - e.runBatch(batch) + batch := c.takeBatch() + if len(batch) == 0 { + return } + e.runBatch(batch) } } -// stopCommitter drains outstanding writes and stops the commit goroutine. +// stopCommitter rejects new writes and waits for in-flight ones to finish. func (e *Engine) stopCommitter() { c := &e.cq c.mu.Lock() c.closed = true - started := c.started - c.mu.Unlock() - if started { - select { - case c.wake <- struct{}{}: - default: - } - <-c.done + if c.idle == nil { + c.idle = sync.NewCond(&c.mu) } + for c.running { + c.idle.Wait() + } + c.mu.Unlock() } func (e *Engine) runBatch(batch []*writeReq) {