From e815ba8d25aba827b27a073571265a2033f83daa Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:17:38 +0530 Subject: [PATCH 01/10] feat(auth): per-method gRPC role authorization and API token expiry Co-Authored-By: Claude Sonnet 5.5 --- internal/api/grpc_auth.go | 71 +++++++++++++++++++--- internal/api/grpc_auth_roles_test.go | 83 ++++++++++++++++++++++++++ internal/api/grpc_auth_test.go | 10 ++-- internal/web/auth/token_expiry_test.go | 54 +++++++++++++++++ internal/web/auth/token_store.go | 31 +++++++++- internal/web/handler/token_api.go | 20 ++++++- 6 files changed, 255 insertions(+), 14 deletions(-) create mode 100644 internal/api/grpc_auth_roles_test.go create mode 100644 internal/web/auth/token_expiry_test.go diff --git a/internal/api/grpc_auth.go b/internal/api/grpc_auth.go index 3f0a115..fc77c09 100644 --- a/internal/api/grpc_auth.go +++ b/internal/api/grpc_auth.go @@ -16,6 +16,53 @@ type TokenValidator interface { Validate(raw string) bool } +// RoleTokenValidator is an optional extension of TokenValidator that also +// reports the role bound to a valid token. Validators that implement it get +// per-method authorization; plain TokenValidators are treated as admin +// (their historical behavior). +type RoleTokenValidator interface { + TokenValidator + ValidateRole(raw string) (role string, ok bool) +} + +// Roles understood by the gRPC authorizer. +const ( + RoleAdmin = "admin" + RoleWriter = "writer" + RoleViewer = "viewer" + RoleReplication = "replication" +) + +// ValidTokenRole reports whether role is a recognized API token role. +func ValidTokenRole(role string) bool { + switch role { + case RoleAdmin, RoleWriter, RoleViewer, RoleReplication: + return true + } + return false +} + +// methodRoles is the deny-by-default method -> allowed roles map. Methods not +// listed here are rejected for every caller. +var methodRoles = map[string][]string{ + "/beastdb.v1.BeastDBService/Get": {RoleViewer, RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Scan": {RoleViewer, RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Put": {RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/Delete": {RoleWriter, RoleAdmin}, + "/beastdb.v1.BeastDBService/BatchWrite": {RoleWriter, RoleAdmin}, + "/beastdb.v1.ReplicationService/StreamWAL": {RoleReplication, RoleAdmin}, + "/beastdb.v1.ReplicationService/Ack": {RoleReplication, RoleAdmin}, +} + +func roleAllowed(role, method string) bool { + for _, r := range methodRoles[method] { + if r == role { + return true + } + } + return false +} + // AuthInterceptor enforces Bearer token authentication on incoming gRPC RPCs. type AuthInterceptor struct { staticToken string @@ -37,7 +84,7 @@ func NewAuthInterceptor(staticToken string, validator ...TokenValidator) *AuthIn // Unary returns a grpc.UnaryServerInterceptor. func (a *AuthInterceptor) Unary() grpc.UnaryServerInterceptor { return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { - if err := a.authorize(ctx); err != nil { + if err := a.authorize(ctx, info.FullMethod); err != nil { return nil, err } return handler(ctx, req) @@ -47,14 +94,14 @@ func (a *AuthInterceptor) Unary() grpc.UnaryServerInterceptor { // Stream returns a grpc.StreamServerInterceptor. func (a *AuthInterceptor) Stream() grpc.StreamServerInterceptor { return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - if err := a.authorize(ss.Context()); err != nil { + if err := a.authorize(ss.Context(), info.FullMethod); err != nil { return err } return handler(srv, ss) } } -func (a *AuthInterceptor) authorize(ctx context.Context) error { +func (a *AuthInterceptor) authorize(ctx context.Context, fullMethod string) error { if a.staticToken == "" && a.validator == nil { return nil } @@ -71,11 +118,21 @@ func (a *AuthInterceptor) authorize(ctx context.Context) error { return status.Errorf(codes.Unauthenticated, "authorization must be Bearer token") } token := strings.TrimSpace(header[7:]) + role := "" if a.staticToken != "" && subtle.ConstantTimeCompare([]byte(token), []byte(a.staticToken)) == 1 { - return nil + role = RoleAdmin + } else if rv, ok := a.validator.(RoleTokenValidator); ok && rv != nil { + if r, valid := rv.ValidateRole(token); valid { + role = r + } + } else if a.validator != nil && a.validator.Validate(token) { + role = RoleAdmin // legacy validator without roles } - if a.validator != nil && a.validator.Validate(token) { - return nil + if role == "" { + return status.Errorf(codes.Unauthenticated, "invalid authorization token") + } + if !roleAllowed(role, fullMethod) { + return status.Errorf(codes.PermissionDenied, "role %q is not permitted to call %s", role, fullMethod) } - return status.Errorf(codes.Unauthenticated, "invalid authorization token") + return nil } diff --git a/internal/api/grpc_auth_roles_test.go b/internal/api/grpc_auth_roles_test.go new file mode 100644 index 0000000..80ecd62 --- /dev/null +++ b/internal/api/grpc_auth_roles_test.go @@ -0,0 +1,83 @@ +package api + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type fakeRoleValidator map[string]string + +func (f fakeRoleValidator) Validate(raw string) bool { _, ok := f[raw]; return ok } +func (f fakeRoleValidator) ValidateRole(raw string) (string, bool) { + r, ok := f[raw] + return r, ok +} + +type legacyValidator struct{ tok string } + +func (l legacyValidator) Validate(raw string) bool { return raw == l.tok } + +func bearer(tok string) context.Context { + return metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer "+tok)) +} + +func TestAuthorizeMethodRoleMatrix(t *testing.T) { + const svc = "/beastdb.v1.BeastDBService/" + const rep = "/beastdb.v1.ReplicationService/" + v := fakeRoleValidator{"v": "viewer", "w": "writer", "a": "admin", "r": "replication", "x": "bogus"} + ai := NewAuthInterceptor("static", v) + + allow := map[string]map[string]bool{ + svc + "Get": {"viewer": true, "writer": true, "admin": true}, + svc + "Scan": {"viewer": true, "writer": true, "admin": true}, + svc + "Put": {"writer": true, "admin": true}, + svc + "Delete": {"writer": true, "admin": true}, + svc + "BatchWrite": {"writer": true, "admin": true}, + rep + "StreamWAL": {"replication": true, "admin": true}, + rep + "Ack": {"replication": true, "admin": true}, + } + tokens := map[string]string{"viewer": "v", "writer": "w", "admin": "a", "replication": "r", "bogus": "x", "static": "static"} + for method, ok := range allow { + for role, tok := range tokens { + want := ok[role] || role == "static" && ok["admin"] + err := ai.authorize(bearer(tok), method) + if want && err != nil { + t.Errorf("%s %s: expected allow, got %v", role, method, err) + } + if !want { + if status.Code(err) != codes.PermissionDenied { + t.Errorf("%s %s: expected PermissionDenied, got %v", role, method, err) + } + } + } + } +} + +func TestAuthorizeDenyByDefaultAndCodes(t *testing.T) { + ai := NewAuthInterceptor("static", fakeRoleValidator{"a": "admin"}) + for _, tok := range []string{"static", "a"} { + if c := status.Code(ai.authorize(bearer(tok), "/beastdb.v1.BeastDBService/Unknown")); c != codes.PermissionDenied { + t.Errorf("unknown method with %s: got %v", tok, c) + } + } + if c := status.Code(ai.authorize(bearer("nope"), "/beastdb.v1.BeastDBService/Get")); c != codes.Unauthenticated { + t.Errorf("bad token: got %v", c) + } + if c := status.Code(ai.authorize(context.Background(), "/beastdb.v1.BeastDBService/Get")); c != codes.Unauthenticated { + t.Errorf("no metadata: got %v", c) + } +} + +func TestAuthorizeLegacyValidatorIsAdmin(t *testing.T) { + ai := NewAuthInterceptor("", legacyValidator{"legacy"}) + if err := ai.authorize(bearer("legacy"), "/beastdb.v1.BeastDBService/Put"); err != nil { + t.Fatalf("legacy validator should act as admin: %v", err) + } + if c := status.Code(ai.authorize(bearer("bad"), "/beastdb.v1.BeastDBService/Put")); c != codes.Unauthenticated { + t.Fatalf("got %v", c) + } +} diff --git a/internal/api/grpc_auth_test.go b/internal/api/grpc_auth_test.go index fedb5ad..37b1a2b 100644 --- a/internal/api/grpc_auth_test.go +++ b/internal/api/grpc_auth_test.go @@ -7,30 +7,32 @@ import ( "google.golang.org/grpc/metadata" ) +const getM = "/beastdb.v1.BeastDBService/Get" + func TestAuthInterceptor(t *testing.T) { ai := NewAuthInterceptor("secret-token-123") // 1. Missing metadata -> unauthenticated - err := ai.authorize(context.Background()) + err := ai.authorize(context.Background(), getM) if err == nil { t.Fatalf("expected error for missing metadata, got nil") } // 2. Wrong token -> unauthenticated badCtx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer wrong")) - if err := ai.authorize(badCtx); err == nil { + if err := ai.authorize(badCtx, getM); err == nil { t.Fatalf("expected error for bad token, got nil") } // 3. Valid token -> allowed goodCtx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer secret-token-123")) - if err := ai.authorize(goodCtx); err != nil { + if err := ai.authorize(goodCtx, getM); err != nil { t.Fatalf("expected success for valid token, got %v", err) } // 4. Empty configured token -> open access openAI := NewAuthInterceptor("") - if err := openAI.authorize(context.Background()); err != nil { + if err := openAI.authorize(context.Background(), getM); err != nil { t.Fatalf("expected success when token is empty, got %v", err) } } diff --git a/internal/web/auth/token_expiry_test.go b/internal/web/auth/token_expiry_test.go new file mode 100644 index 0000000..c2e53c8 --- /dev/null +++ b/internal/web/auth/token_expiry_test.go @@ -0,0 +1,54 @@ +package auth + +import ( + "encoding/json" + "testing" + "time" +) + +func TestTokenExpiryAndRole(t *testing.T) { + engine := newMockEngine() + scan := func(start, end uint64) ([]TokenRecord, error) { + var recs []TokenRecord + for k, v := range engine.data { + if k >= start && k <= end { + recs = append(recs, TokenRecord{Key: k, Value: v}) + } + } + return recs, nil + } + ts := NewTokenStore(engine, scan) + + past := time.Now().Add(-time.Minute) + future := time.Now().Add(time.Hour) + expired, _, _ := ts.CreateWithExpiry("old", "viewer", "t", &past) + live, _, _ := ts.CreateWithExpiry("new", "writer", "t", &future) + forever, _, _ := ts.Create("forever", "viewer", "t") + + if ts.Validate(expired) { + t.Fatal("expired token must not validate") + } + if role, ok := ts.ValidateRole(live); !ok || role != "writer" { + t.Fatalf("live token: %q %v", role, ok) + } + if role, ok := ts.ValidateRole(forever); !ok || role != "viewer" { + t.Fatalf("no-expiry token: %q %v", role, ok) + } + + // Persisted and reloaded: expiry still enforced. + ts2 := NewTokenStore(engine, scan) + if ts2.Validate(expired) || !ts2.Validate(live) { + t.Fatal("expiry not preserved across reload") + } +} + +func TestLegacyStoredTokenWithoutExpiry(t *testing.T) { + raw := "bst_live_legacy" + rec, _ := json.Marshal(map[string]any{"id": "tok_1", "name": "n", "token_hash": hashToken(raw), "role": "admin"}) + ts := NewTokenStore(newMockEngine(), func(start, end uint64) ([]TokenRecord, error) { + return []TokenRecord{{Key: tokenKey("tok_1"), Value: rec}}, nil + }) + if role, ok := ts.ValidateRole(raw); !ok || role != "admin" { + t.Fatalf("legacy token should validate as admin, got %q %v", role, ok) + } +} diff --git a/internal/web/auth/token_store.go b/internal/web/auth/token_store.go index 5e0fde2..456a2ce 100644 --- a/internal/web/auth/token_store.go +++ b/internal/web/auth/token_store.go @@ -21,6 +21,13 @@ type APIToken struct { Role string `json:"role"` CreatedAt time.Time `json:"created_at"` CreatedBy string `json:"created_by"` + // ExpiresAt is optional; nil (or absent in older stored tokens) means no expiry. + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +// Expired reports whether the token has an expiry that is at or before now. +func (t APIToken) Expired(now time.Time) bool { + return t.ExpiresAt != nil && !now.Before(*t.ExpiresAt) } // TokenEngine is the storage interface needed for token persistence. @@ -67,6 +74,11 @@ func hashToken(raw string) string { // Create generates a new API token, saves it to partition 0x05, and activates it. func (ts *TokenStore) Create(name, role, createdBy string) (string, *APIToken, error) { + return ts.CreateWithExpiry(name, role, createdBy, nil) +} + +// CreateWithExpiry is Create with an optional expiry time (nil = never expires). +func (ts *TokenStore) CreateWithExpiry(name, role, createdBy string, expiresAt *time.Time) (string, *APIToken, error) { idBytes := make([]byte, 4) tokenBytes := make([]byte, 20) if _, err := rand.Read(idBytes); err != nil { @@ -90,6 +102,10 @@ func (ts *TokenStore) Create(name, role, createdBy string) (string, *APIToken, e CreatedAt: time.Now().UTC(), CreatedBy: createdBy, } + if expiresAt != nil { + e := expiresAt.UTC() + tok.ExpiresAt = &e + } data, err := json.Marshal(tok) if err != nil { @@ -135,11 +151,22 @@ func (ts *TokenStore) Delete(id string) error { // Validate checks whether a given raw token is valid in < 100ns without disk I/O. func (ts *TokenStore) Validate(raw string) bool { + _, ok := ts.ValidateRole(raw) + return ok +} + +// ValidateRole returns the token's role if it exists and has not expired. +// Lookup is by SHA-256 hash of the presented token (map lookup on a hash, so +// no raw-secret comparison timing leak). +func (ts *TokenStore) ValidateRole(raw string) (string, bool) { th := hashToken(raw) ts.mu.RLock() - _, exists := ts.tokenIndex[th] + tok, exists := ts.tokenIndex[th] ts.mu.RUnlock() - return exists + if !exists || tok.Expired(time.Now()) { + return "", false + } + return tok.Role, true } // Reload repopulates the in-memory token cache from partition 0x05. diff --git a/internal/web/handler/token_api.go b/internal/web/handler/token_api.go index b446b16..47ddaf6 100644 --- a/internal/web/handler/token_api.go +++ b/internal/web/handler/token_api.go @@ -4,7 +4,9 @@ import ( "encoding/json" "net/http" "strings" + "time" + "github.com/ChromaBeast/beastdb/internal/api" "github.com/ChromaBeast/beastdb/internal/web/auth" ) @@ -52,6 +54,8 @@ func (h *APIHandler) CreateToken(w http.ResponseWriter, r *http.Request) { var body struct { Name string `json:"name"` Role string `json:"role"` + // ExpiresInSeconds optionally makes the token expire after N seconds (0 = never). + ExpiresInSeconds int64 `json:"expires_in_seconds"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { http.Error(w, "Bad Request", http.StatusBadRequest) @@ -69,7 +73,21 @@ func (h *APIHandler) CreateToken(w http.ResponseWriter, r *http.Request) { tokRole = "admin" } - rawToken, meta, err := h.tokens.Create(name, tokRole, username) + if !api.ValidTokenRole(tokRole) { + http.Error(w, "Invalid role: must be admin, writer, viewer or replication", http.StatusBadRequest) + return + } + if body.ExpiresInSeconds < 0 || body.ExpiresInSeconds > 10*365*24*3600 { + http.Error(w, "Invalid expires_in_seconds", http.StatusBadRequest) + return + } + var expiresAt *time.Time + if body.ExpiresInSeconds > 0 { + t := time.Now().Add(time.Duration(body.ExpiresInSeconds) * time.Second) + expiresAt = &t + } + + rawToken, meta, err := h.tokens.CreateWithExpiry(name, tokRole, username, expiresAt) if err != nil { http.Error(w, "Failed to create token: "+err.Error(), http.StatusInternalServerError) return From 7ed1da2b0e7c3cdffffcdaf2bfb5c74b95d6a6b2 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:19:18 +0530 Subject: [PATCH 02/10] feat(bench): add beastbench YCSB-style load generator and benchmarking guide Co-Authored-By: Claude Sonnet 5.5 --- cmd/beastbench/beastbench_test.go | 185 +++++++++++++++++ cmd/beastbench/client.go | 103 ++++++++++ cmd/beastbench/dist.go | 82 ++++++++ cmd/beastbench/histogram.go | 131 ++++++++++++ cmd/beastbench/main.go | 151 ++++++++++++++ cmd/beastbench/runner.go | 318 ++++++++++++++++++++++++++++++ cmd/beastbench/selfhost.go | 39 ++++ cmd/beastbench/workload.go | 73 +++++++ docs/benchmarking.md | 172 ++++++++++++++++ 9 files changed, 1254 insertions(+) create mode 100644 cmd/beastbench/beastbench_test.go create mode 100644 cmd/beastbench/client.go create mode 100644 cmd/beastbench/dist.go create mode 100644 cmd/beastbench/histogram.go create mode 100644 cmd/beastbench/main.go create mode 100644 cmd/beastbench/runner.go create mode 100644 cmd/beastbench/selfhost.go create mode 100644 cmd/beastbench/workload.go create mode 100644 docs/benchmarking.md diff --git a/cmd/beastbench/beastbench_test.go b/cmd/beastbench/beastbench_test.go new file mode 100644 index 0000000..5bd0323 --- /dev/null +++ b/cmd/beastbench/beastbench_test.go @@ -0,0 +1,185 @@ +package main + +import ( + "bytes" + "encoding/json" + "math/rand" + "sort" + "sync/atomic" + "testing" + "time" +) + +func TestHistogramQuantiles(t *testing.T) { + h := NewHistogram() + var vals []int + for i := 1; i <= 100000; i++ { + d := time.Duration(i) * time.Microsecond + h.Record(d) + vals = append(vals, int(d)) + } + sort.Ints(vals) + for _, q := range []float64{.5, .95, .99} { + want := float64(vals[int(q*float64(len(vals)))-1]) + got := float64(h.Quantile(q)) + if got < want || got > want*1.02 { + t.Errorf("q=%v got %v want ~%v", q, got, want) + } + } + if h.Max() != 100000*time.Microsecond || h.Min() != time.Microsecond { + t.Errorf("min/max wrong: %v %v", h.Min(), h.Max()) + } + if h.Count() != 100000 { + t.Errorf("count %d", h.Count()) + } +} + +func TestHistogramBucketsRoundTrip(t *testing.T) { + for _, v := range []uint64{0, 1, 127, 128, 255, 256, 257, 1000, 1 << 20, 1<<40 + 12345, 1<<63 + 5} { + lo, hi := bucketBounds(bucketIndex(v)) + if v < lo || v > hi { + t.Errorf("v=%d not in [%d,%d]", v, lo, hi) + } + } +} + +func TestHistogramMergeAndEmpty(t *testing.T) { + a, b := NewHistogram(), NewHistogram() + if a.Quantile(.5) != 0 || a.Mean() != 0 || a.Min() != 0 { + t.Error("empty histogram should report zeros") + } + a.Record(time.Millisecond) + b.Record(3 * time.Millisecond) + a.Merge(b) + if a.Count() != 2 || a.Max() != 3*time.Millisecond || a.Min() != time.Millisecond { + t.Errorf("merge wrong: %d %v %v", a.Count(), a.Min(), a.Max()) + } +} + +func TestZipfianSkewAndRange(t *testing.T) { + const n = 1000 + z := NewZipfian(n, DefaultZipfTheta) + r := rand.New(rand.NewSource(1)) + counts := make([]int, n) + for i := 0; i < 200000; i++ { + v := z.Next(r) + if v >= n { + t.Fatalf("out of range: %d", v) + } + counts[v]++ + } + if counts[0] < counts[10] || counts[10] < counts[500] { + t.Errorf("not monotonically skewed: %d %d %d", counts[0], counts[10], counts[500]) + } + if counts[0] < 200000/20 { + t.Errorf("item 0 too cold: %d", counts[0]) + } +} + +func TestUniformAndDeterminism(t *testing.T) { + u := Uniform{N: 10} + r := rand.New(rand.NewSource(7)) + seen := map[uint64]bool{} + for i := 0; i < 1000; i++ { + seen[u.Next(r)] = true + } + if len(seen) != 10 { + t.Errorf("uniform missed buckets: %d", len(seen)) + } + z := NewZipfian(100, .9) + a, b := rand.New(rand.NewSource(3)), rand.New(rand.NewSource(3)) + for i := 0; i < 100; i++ { + if z.Next(a) != z.Next(b) { + t.Fatal("same seed must give same sequence") + } + } +} + +func TestLatestFavoursNewest(t *testing.T) { + f := &atomic.Uint64{} + f.Store(1000) + l := Latest{Zipf: NewZipfian(1000, DefaultZipfTheta), Frontier: f} + r := rand.New(rand.NewSource(2)) + newest := 0 + for i := 0; i < 10000; i++ { + v := l.Next(r) + if v >= 1000 { + t.Fatalf("out of range %d", v) + } + if v >= 990 { + newest++ + } + } + if newest < 2000 { + t.Errorf("latest not skewed to newest: %d", newest) + } +} + +func TestWorkloadMixes(t *testing.T) { + for name, w := range workloads { + var sum float64 + for _, p := range w.Mix { + sum += p + } + if sum < .999 || sum > 1.001 { + t.Errorf("workload %s mix sums to %v", name, sum) + } + } + w := workloads["A"] + if w.pick(0.1) != OpRead || w.pick(0.9) != OpUpdate { + t.Error("A pick wrong") + } + if _, err := lookupWorkload("Z"); err == nil { + t.Error("expected error for unknown workload") + } +} + +func TestSmokeAllWorkloads(t *testing.T) { + for _, wl := range []string{"A", "B", "C", "D", "E", "F"} { + wl := wl + t.Run(wl, func(t *testing.T) { + var out, errb bytes.Buffer + err := realMain([]string{"--self-contained", "--json", "--workload", wl, "--records", "200", + "--ops", "300", "--warmup", "20", "--threads", "4", "--value-size", "64", "--seed", "5", "--max-scan-len", "10"}, &out, &errb) + if err != nil { + t.Fatalf("%v (%s)", err, errb.String()) + } + var rep Report + if err := json.Unmarshal(out.Bytes(), &rep); err != nil { + t.Fatal(err) + } + if len(rep.Phases) != 2 || rep.Phases[1].Ops != 300 { + t.Fatalf("unexpected phases: %+v", rep.Phases) + } + for _, p := range rep.Phases { + for _, o := range p.PerOp { + if o.Errors != 0 { + t.Errorf("%s/%s: %d errors", p.Phase, o.Op, o.Errors) + } + if o.P50Us > o.P99Us || o.P99Us > o.MaxUs { + t.Errorf("%s/%s percentiles not monotonic: %+v", p.Phase, o.Op, o) + } + } + } + if rep.Phases[0].Ops != 200 { + t.Errorf("load ops %d", rep.Phases[0].Ops) + } + }) + } +} + +func TestTextOutputAndBadFlags(t *testing.T) { + var out, errb bytes.Buffer + if err := realMain([]string{"--self-contained", "--records", "50", "--ops", "50", "--threads", "2"}, &out, &errb); err != nil { + t.Fatal(err) + } + if !bytes.Contains(out.Bytes(), []byte("not a competitive benchmark")) || !bytes.Contains(out.Bytes(), []byte("p99(us)")) { + t.Errorf("text output missing pieces:\n%s", out.String()) + } + if err := realMain([]string{"--workload", "Q"}, &out, &errb); err == nil { + t.Error("expected error") + } + if err := realMain([]string{"--value-size", "1"}, &out, &errb); err == nil { + t.Error("expected error") + } +} diff --git a/cmd/beastbench/client.go b/cmd/beastbench/client.go new file mode 100644 index 0000000..2812911 --- /dev/null +++ b/cmd/beastbench/client.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "github.com/ChromaBeast/beastdb/internal/api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// tokenAuth mirrors cmd/beastctl: Bearer token in the authorization header. +type tokenAuth struct{ token string } + +func (t tokenAuth) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + if t.token == "" { + return nil, nil + } + return map[string]string{"authorization": "Bearer " + t.token}, nil +} + +// RequireTransportSecurity is false to match beastctl, so tokens work on plain +// loopback test servers; use the TLS flags for anything non-local. +func (t tokenAuth) RequireTransportSecurity() bool { return false } + +// ConnOpts describes how to reach the server. +type ConnOpts struct { + Addr, Token string + TLSCert, TLSKey, TLSCA string + TLSServerName string + Conns int +} + +func (o ConnOpts) creds() (credentials.TransportCredentials, error) { + if o.TLSCert == "" && o.TLSCA == "" { + return insecure.NewCredentials(), nil + } + cfg := &tls.Config{MinVersion: tls.VersionTLS13, ServerName: o.TLSServerName} + if o.TLSCA != "" { + pem, err := os.ReadFile(o.TLSCA) + if err != nil { + return nil, fmt.Errorf("read CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("no certificates in %s", o.TLSCA) + } + cfg.RootCAs = pool + } + if o.TLSCert != "" || o.TLSKey != "" { + cert, err := tls.LoadX509KeyPair(o.TLSCert, o.TLSKey) + if err != nil { + return nil, fmt.Errorf("load client cert: %w", err) + } + cfg.Certificates = []tls.Certificate{cert} + } + return credentials.NewTLS(cfg), nil +} + +// Dial opens o.Conns connections and verifies the server answers. +func Dial(o ConnOpts) ([]beastv1.BeastDBServiceClient, func(), error) { + creds, err := o.creds() + if err != nil { + return nil, nil, err + } + n := o.Conns + if n < 1 { + n = 1 + } + var conns []*grpc.ClientConn + closeAll := func() { + for _, c := range conns { + c.Close() + } + } + var out []beastv1.BeastDBServiceClient + for i := 0; i < n; i++ { + opts := []grpc.DialOption{grpc.WithTransportCredentials(creds), api.MessageSizeDialOption()} + if o.Token != "" { + opts = append(opts, grpc.WithPerRPCCredentials(tokenAuth{o.Token})) + } + conn, err := grpc.NewClient(o.Addr, opts...) + if err != nil { + closeAll() + return nil, nil, fmt.Errorf("dial %s: %w", o.Addr, err) + } + conns = append(conns, conn) + out = append(out, beastv1.NewBeastDBServiceClient(conn)) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := out[0].Get(ctx, &beastv1.GetRequest{Key: 0}); err != nil { + closeAll() + return nil, nil, fmt.Errorf("could not reach BeastDB at %s: %w", o.Addr, err) + } + return out, closeAll, nil +} diff --git a/cmd/beastbench/dist.go b/cmd/beastbench/dist.go new file mode 100644 index 0000000..c5eb65a --- /dev/null +++ b/cmd/beastbench/dist.go @@ -0,0 +1,82 @@ +package main + +import ( + "math" + "math/rand" + "sync/atomic" +) + +// KeyChooser picks a record index in [0, n). +type KeyChooser interface { + Next(r *rand.Rand) uint64 +} + +// Uniform picks indexes uniformly. +type Uniform struct{ N uint64 } + +func (u Uniform) Next(r *rand.Rand) uint64 { return uint64(r.Int63n(int64(u.N))) } + +// Zipfian implements the YCSB / Gray et al. generator. Index 0 is the most +// popular item. Parameters are precomputed so Next is O(1) and the value is +// safe for concurrent use (each caller supplies its own *rand.Rand). +type Zipfian struct { + n uint64 + theta, alpha, zeta float64 + zeta2, eta float64 +} + +// DefaultZipfTheta is YCSB's default skew constant. +const DefaultZipfTheta = 0.99 + +func zetaN(n uint64, theta float64) float64 { + var s float64 + for i := uint64(1); i <= n; i++ { + s += 1 / math.Pow(float64(i), theta) + } + return s +} + +// NewZipfian builds a generator over n items. +func NewZipfian(n uint64, theta float64) *Zipfian { + z := &Zipfian{n: n, theta: theta} + z.zeta2 = zetaN(2, theta) + z.zeta = zetaN(n, theta) + z.alpha = 1 / (1 - theta) + z.eta = (1 - math.Pow(2/float64(n), 1-theta)) / (1 - z.zeta2/z.zeta) + return z +} + +func (z *Zipfian) Next(r *rand.Rand) uint64 { + u := r.Float64() + uz := u * z.zeta + if uz < 1 { + return 0 + } + if uz < 1+math.Pow(0.5, z.theta) { + return 1 % z.n + } + v := uint64(float64(z.n) * math.Pow(z.eta*u-z.eta+1, z.alpha)) + if v >= z.n { + v = z.n - 1 + } + return v +} + +// Latest favours the most recently inserted records (workload D): it draws a +// zipfian offset back from the current insert frontier. +type Latest struct { + Zipf *Zipfian + Frontier *atomic.Uint64 // number of records currently present +} + +func (l Latest) Next(r *rand.Rand) uint64 { + n := l.Frontier.Load() + if n == 0 { + return 0 + } + off := l.Zipf.Next(r) + if off >= n { + off = n - 1 + } + return n - 1 - off +} diff --git a/cmd/beastbench/histogram.go b/cmd/beastbench/histogram.go new file mode 100644 index 0000000..e368129 --- /dev/null +++ b/cmd/beastbench/histogram.go @@ -0,0 +1,131 @@ +package main + +import ( + "math" + "math/bits" + "time" +) + +// Histogram is a small HDR-style log-linear histogram of nanosecond values. +// Each power-of-two range is split into 128 linear sub-buckets, bounding the +// relative quantile error to under 1%. It is not safe for concurrent use; +// give each worker its own and Merge afterwards. +type Histogram struct { + counts [histBuckets]uint64 + total uint64 + sum float64 + min uint64 + max uint64 +} + +const ( + histSubBits = 7 + histSub = 1 << histSubBits + histBuckets = (64-histSubBits)*histSub + 2*histSub +) + +// NewHistogram returns an empty histogram. +func NewHistogram() *Histogram { return &Histogram{min: math.MaxUint64} } + +func bucketIndex(v uint64) int { + shift := bits.Len64(v) - histSubBits - 1 + if shift < 0 { + shift = 0 + } + return shift*histSub + int(v>>uint(shift)) +} + +// bucketBounds returns the inclusive value range covered by bucket idx. +func bucketBounds(idx int) (lo, hi uint64) { + if idx < 2*histSub { + return uint64(idx), uint64(idx) + } + shift := uint(idx/histSub - 1) + m := uint64(idx%histSub + histSub) + lo = m << shift + return lo, lo + (uint64(1) << shift) - 1 +} + +// Record adds one observation. +func (h *Histogram) Record(d time.Duration) { + if d < 0 { + d = 0 + } + v := uint64(d) + h.counts[bucketIndex(v)]++ + h.total++ + h.sum += float64(v) + if v < h.min { + h.min = v + } + if v > h.max { + h.max = v + } +} + +// Count returns the number of observations. +func (h *Histogram) Count() uint64 { return h.total } + +// Max returns the largest observation (exact). +func (h *Histogram) Max() time.Duration { return time.Duration(h.max) } + +// Min returns the smallest observation (exact), or 0 when empty. +func (h *Histogram) Min() time.Duration { + if h.total == 0 { + return 0 + } + return time.Duration(h.min) +} + +// Mean returns the arithmetic mean. +func (h *Histogram) Mean() time.Duration { + if h.total == 0 { + return 0 + } + return time.Duration(h.sum / float64(h.total)) +} + +// Quantile returns the value at quantile q in [0,1] (bucket upper bound, +// clamped to the exact max). +func (h *Histogram) Quantile(q float64) time.Duration { + if h.total == 0 { + return 0 + } + if q >= 1 { + return h.Max() + } + rank := uint64(math.Ceil(q * float64(h.total))) + if rank < 1 { + rank = 1 + } + var seen uint64 + for i, c := range h.counts { + if c == 0 { + continue + } + seen += c + if seen >= rank { + _, hi := bucketBounds(i) + if hi > h.max { + hi = h.max + } + return time.Duration(hi) + } + } + return h.Max() +} + +// Merge adds all observations of o into h. +func (h *Histogram) Merge(o *Histogram) { + for i, c := range o.counts { + h.counts[i] += c + } + h.total += o.total + h.sum += o.sum + if o.total > 0 && o.min < h.min { + h.min = o.min + } + if o.max > h.max { + h.max = o.max + } +} diff --git a/cmd/beastbench/main.go b/cmd/beastbench/main.go new file mode 100644 index 0000000..561dd38 --- /dev/null +++ b/cmd/beastbench/main.go @@ -0,0 +1,151 @@ +// Command beastbench is a reproducible YCSB-style load generator for BeastDB. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/signal" + "runtime" + "strings" + "time" +) + +// Report is the full machine-readable result. +type Report struct { + Tool string `json:"tool"` + GoVersion string `json:"go_version"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + NumCPU int `json:"num_cpu"` + SelfHosted bool `json:"self_hosted"` + Addr string `json:"addr"` + Config Config `json:"config"` + Phases []PhaseResult `json:"phases"` +} + +func main() { + if err := realMain(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "beastbench:", err) + os.Exit(1) + } +} + +func realMain(args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("beastbench", flag.ContinueOnError) + fs.SetOutput(stderr) + var ( + cfg Config + co ConnOpts + self = fs.Bool("self-contained", false, "start an in-process engine+gRPC server on a temp dir (smoke runs only)") + poolSize = fs.Int("pool-size", 128, "buffer pool frames for --self-contained") + asJSON = fs.Bool("json", false, "emit JSON instead of text") + skipLoad = fs.Bool("skip-load", false, "skip the load phase (dataset already loaded with the same --records/--value-size/--seed)") + loadOnly = fs.Bool("load-only", false, "only run the load phase") + timeout = fs.Duration("op-timeout", 30*time.Second, "per-operation timeout") + ) + fs.StringVar(&co.Addr, "addr", "127.0.0.1:50051", "BeastDB gRPC address") + fs.StringVar(&co.Token, "token", strings.TrimSpace(os.Getenv("BEASTDB_API_TOKEN")), "Bearer token (default $BEASTDB_API_TOKEN)") + fs.StringVar(&co.TLSCert, "tls-cert", os.Getenv("BEASTDB_TLS_CERT"), "client TLS certificate (PEM)") + fs.StringVar(&co.TLSKey, "tls-key", os.Getenv("BEASTDB_TLS_KEY"), "client TLS key (PEM)") + fs.StringVar(&co.TLSCA, "tls-ca", os.Getenv("BEASTDB_TLS_CA"), "CA certificate to verify the server (PEM)") + fs.StringVar(&co.TLSServerName, "tls-server-name", "", "override TLS server name") + fs.IntVar(&co.Conns, "conns", 1, "gRPC connections (threads are spread round-robin)") + fs.StringVar(&cfg.Workload, "workload", "A", "workload A-F") + fs.StringVar(&cfg.Dist, "dist", "zipfian", "key distribution: zipfian or uniform (workload D always uses read-latest)") + fs.Float64Var(&cfg.ZipfTheta, "zipf-theta", DefaultZipfTheta, "zipfian skew constant in (0,1)") + fs.Uint64Var(&cfg.Records, "records", 10000, "records loaded before the run") + fs.Uint64Var(&cfg.Ops, "ops", 50000, "operations in the measured run phase") + fs.Uint64Var(&cfg.WarmupOps, "warmup", 0, "unmeasured warmup operations before the run phase") + fs.IntVar(&cfg.Threads, "threads", 8, "concurrent client workers") + fs.IntVar(&cfg.ValueSize, "value-size", 100, "value size in bytes (8..2097152)") + fs.IntVar(&cfg.MaxScanLen, "max-scan-len", 100, "workload E scan length is uniform in [1,max-scan-len]") + fs.Int64Var(&cfg.Seed, "seed", 1, "RNG seed; same seed gives the same operation sequence per thread") + if err := fs.Parse(args); err != nil { + return err + } + cfg.Workload = strings.ToUpper(cfg.Workload) + cfg.Dist = strings.ToLower(cfg.Dist) + cfg.Timeout = *timeout + if _, err := lookupWorkload(cfg.Workload); err != nil { + return err + } + switch { + case cfg.Dist != "zipfian" && cfg.Dist != "uniform": + return fmt.Errorf("--dist must be zipfian or uniform") + case cfg.Records < 2, cfg.Threads < 1, cfg.MaxScanLen < 1: + return fmt.Errorf("need --records >= 2, --threads >= 1, --max-scan-len >= 1") + case cfg.ValueSize < 8 || cfg.ValueSize > 2<<20: + return fmt.Errorf("--value-size must be in [8, 2097152]") + case cfg.ZipfTheta <= 0 || cfg.ZipfTheta >= 1: + return fmt.Errorf("--zipf-theta must be in (0,1)") + } + if uint64(cfg.Threads) > cfg.Records { + cfg.Threads = int(cfg.Records) + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + if *self { + addr, stopSrv, err := StartSelfHosted(*poolSize) + if err != nil { + return err + } + defer stopSrv() + co.Addr = addr + co.TLSCert, co.TLSKey, co.TLSCA, co.Token = "", "", "", "" + } + cl, closeConns, err := Dial(co) + if err != nil { + return err + } + defer closeConns() + + rep := Report{Tool: "beastbench", GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, + NumCPU: runtime.NumCPU(), SelfHosted: *self, Addr: co.Addr, Config: cfg} + + if !*skipLoad { + res, err := Load(ctx, cl, cfg) + rep.Phases = append(rep.Phases, res) + if err != nil { + return err + } + } + if !*loadOnly { + res, err := Run(ctx, cl, cfg) + rep.Phases = append(rep.Phases, res) + if err != nil { + return err + } + } + if *asJSON { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + return enc.Encode(rep) + } + printText(stdout, rep) + return nil +} + +func printText(w io.Writer, r Report) { + c := r.Config + fmt.Fprintf(w, "beastbench workload=%s dist=%s records=%d ops=%d warmup=%d threads=%d value=%dB seed=%d\n", + c.Workload, c.Dist, c.Records, c.Ops, c.WarmupOps, c.Threads, c.ValueSize, c.Seed) + fmt.Fprintf(w, "%s/%s go=%s cpus=%d self-contained=%v addr=%s\n", r.GOOS, r.GOARCH, r.GoVersion, r.NumCPU, r.SelfHosted, r.Addr) + if r.SelfHosted { + fmt.Fprintln(w, "NOTE: self-contained loopback smoke run; not a competitive benchmark.") + } + for _, p := range r.Phases { + fmt.Fprintf(w, "\n[%s] %d ops in %.2fs = %.0f ops/s\n", p.Phase, p.Ops, p.Seconds, p.Throughput) + fmt.Fprintf(w, " %-8s %10s %10s %10s %10s %10s %10s %7s %7s %7s\n", + "op", "count", "ops/s", "p50(us)", "p95(us)", "p99(us)", "max(us)", "errors", "misses", "confl") + for _, o := range p.PerOp { + fmt.Fprintf(w, " %-8s %10d %10.0f %10.1f %10.1f %10.1f %10.1f %7d %7d %7d\n", + o.Op, o.Count, o.Throughput, o.P50Us, o.P95Us, o.P99Us, o.MaxUs, o.Errors, o.Misses, o.Conflicts) + } + } +} diff --git a/cmd/beastbench/runner.go b/cmd/beastbench/runner.go new file mode 100644 index 0000000..69cd245 --- /dev/null +++ b/cmd/beastbench/runner.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "math" + "math/rand" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Config holds everything that determines a run. +type Config struct { + Workload string `json:"workload"` + Dist string `json:"distribution"` + ZipfTheta float64 `json:"zipf_theta,omitempty"` + Records uint64 `json:"records"` + Ops uint64 `json:"ops"` + WarmupOps uint64 `json:"warmup_ops"` + Threads int `json:"threads"` + ValueSize int `json:"value_size"` + MaxScanLen int `json:"max_scan_len"` + Seed int64 `json:"seed"` + Timeout time.Duration `json:"-"` +} + +// OpStats is the per-operation result. +type OpStats struct { + Op string `json:"op"` + Count uint64 `json:"count"` + Errors uint64 `json:"errors"` + Misses uint64 `json:"misses,omitempty"` // reads/scans that found nothing + Conflicts uint64 `json:"conflicts,omitempty"` // conditional writes rejected (rmw) + Throughput float64 `json:"ops_per_sec"` + MeanUs float64 `json:"mean_us"` + P50Us float64 `json:"p50_us"` + P95Us float64 `json:"p95_us"` + P99Us float64 `json:"p99_us"` + MaxUs float64 `json:"max_us"` +} + +// PhaseResult summarises one phase. +type PhaseResult struct { + Phase string `json:"phase"` + Seconds float64 `json:"seconds"` + Ops uint64 `json:"ops"` + Throughput float64 `json:"ops_per_sec"` + PerOp []OpStats `json:"per_op"` +} + +func us(d time.Duration) float64 { return float64(d) / float64(time.Microsecond) } + +func recordKey(idx uint64) uint64 { return idx + 1 } + +// makeValue deterministically fills a value for (idx, version). +func makeValue(buf []byte, idx uint64, version uint64) { + x := idx*0x9E3779B97F4A7C15 + version + 1 + for i := 0; i < len(buf); i += 8 { + x ^= x << 13 + x ^= x >> 7 + x ^= x << 17 + var tmp [8]byte + binary.LittleEndian.PutUint64(tmp[:], x) + copy(buf[i:], tmp[:]) + } +} + +type counters struct { + hist [numOps]*Histogram + errs [numOps]uint64 + misses [numOps]uint64 + conflicts [numOps]uint64 +} + +func newCounters() *counters { + c := &counters{} + for i := range c.hist { + c.hist[i] = NewHistogram() + } + return c +} + +// Load inserts cfg.Records records with the given parallelism. +func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + var firstErr atomic.Value + start := time.Now() + per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + wg.Add(1) + go func(t int) { + defer wg.Done() + c := cs[t] + client := cl[t%len(cl)] + buf := make([]byte, cfg.ValueSize) + lo := uint64(t) * per + hi := lo + per + if hi > cfg.Records { + hi = cfg.Records + } + for i := lo; i < hi; i++ { + if ctx.Err() != nil { + return + } + makeValue(buf, i, 0) + t0 := time.Now() + _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) + c.hist[OpInsert].Record(time.Since(t0)) + if err != nil { + c.errs[OpInsert]++ + firstErr.CompareAndSwap(nil, err) + return + } + } + }(t) + } + wg.Wait() + el := time.Since(start) + res := summarize("load", el, cs) + if e, ok := firstErr.Load().(error); ok { + return res, fmt.Errorf("load failed: %w", e) + } + return res, ctx.Err() +} + +// callPut returns conflict=true when a conditional write was rejected (ABORTED). +func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) + if err != nil { + if status.Code(err) == codes.Aborted { + return true, nil + } + return false, err + } + return false, nil +} + +// Run executes warmup (unrecorded) then the measured run phase. +func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + w, err := lookupWorkload(cfg.Workload) + if err != nil { + return PhaseResult{}, err + } + frontier := &atomic.Uint64{} + frontier.Store(cfg.Records) + + var chooser KeyChooser + switch { + case w.Latest: + chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} + case cfg.Dist == "uniform": + chooser = Uniform{N: cfg.Records} + default: + chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) + } + + phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + start := time.Now() + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + n := total / uint64(cfg.Threads) + if uint64(t) < total%uint64(cfg.Threads) { + n++ + } + wg.Add(1) + go func(t int, n uint64) { + defer wg.Done() + r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) + worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) + }(t, n) + } + wg.Wait() + return cs, time.Since(start) + } + + if cfg.WarmupOps > 0 { + phase(cfg.WarmupOps, 1_000_003, false) + } + cs, el := phase(cfg.Ops, 0, true) + return summarize("run", el, cs), ctx.Err() +} + +func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, + ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { + buf := make([]byte, cfg.ValueSize) + for i := uint64(0); i < n && ctx.Err() == nil; i++ { + op := w.pick(r.Float64()) + t0 := time.Now() + var err error + var miss, conflict bool + switch op { + case OpRead: + idx := ch.Next(r) + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) + miss = err == nil && !found + case OpUpdate: + idx := ch.Next(r) + r.Read(buf) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpInsert: + idx := frontier.Add(1) - 1 + makeValue(buf, idx, 0) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpScan: + idx := ch.Next(r) + ln := 1 + r.Intn(cfg.MaxScanLen) + var got int + got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) + miss = err == nil && got == 0 + case OpRMW: + idx := ch.Next(r) + var old []byte + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) + if err == nil && !found { + miss = true + break + } + if err == nil { + nv := append([]byte(nil), old...) + if len(nv) > 0 { + nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) + } + conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ + Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) + } + } + if ctx.Err() != nil { + return + } + if !record { + continue + } + c.hist[op].Record(time.Since(t0)) + if err != nil { + c.errs[op]++ + } + if miss { + c.misses[op]++ + } + if conflict { + c.conflicts[op]++ + } + } +} + +func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) + if err != nil { + return false, err + } + if out != nil { + *out = resp.Value + } + return resp.Found, nil +} + +func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) + if err != nil { + return 0, err + } + n := 0 + for { + _, err := st.Recv() + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + n++ + } +} + +func summarize(name string, el time.Duration, cs []*counters) PhaseResult { + res := PhaseResult{Phase: name, Seconds: el.Seconds()} + for op := OpType(0); op < numOps; op++ { + h := NewHistogram() + var errs, miss, conf uint64 + for _, c := range cs { + h.Merge(c.hist[op]) + errs += c.errs[op] + miss += c.misses[op] + conf += c.conflicts[op] + } + if h.Count() == 0 { + continue + } + res.Ops += h.Count() + res.PerOp = append(res.PerOp, OpStats{ + Op: op.String(), Count: h.Count(), Errors: errs, Misses: miss, Conflicts: conf, + Throughput: float64(h.Count()) / el.Seconds(), + MeanUs: us(h.Mean()), P50Us: us(h.Quantile(.50)), P95Us: us(h.Quantile(.95)), + P99Us: us(h.Quantile(.99)), MaxUs: us(h.Max()), + }) + } + res.Throughput = float64(res.Ops) / el.Seconds() + return res +} diff --git a/cmd/beastbench/selfhost.go b/cmd/beastbench/selfhost.go new file mode 100644 index 0000000..55e35fd --- /dev/null +++ b/cmd/beastbench/selfhost.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "net" + "os" + "path/filepath" + + "github.com/ChromaBeast/beastdb/internal/api" +) + +// StartSelfHosted runs an in-process engine and gRPC server on a temp dir +// listening on loopback. The returned stop func removes everything. +func StartSelfHosted(poolSize int) (addr string, stop func(), err error) { + dir, err := os.MkdirTemp("", "beastbench-*") + if err != nil { + return "", nil, err + } + eng, err := api.NewEngine(filepath.Join(dir, "beast.bin"), filepath.Join(dir, "beast.wal"), poolSize) + if err != nil { + os.RemoveAll(dir) + return "", nil, fmt.Errorf("start engine: %w", err) + } + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + eng.Close() + os.RemoveAll(dir) + return "", nil, err + } + srv := api.NewGRPCServer(eng, api.MessageSizeServerOptions()...) + done := make(chan struct{}) + go func() { _ = srv.Serve(lis); close(done) }() + return lis.Addr().String(), func() { + srv.Stop() + <-done + eng.Close() + os.RemoveAll(dir) + }, nil +} diff --git a/cmd/beastbench/workload.go b/cmd/beastbench/workload.go new file mode 100644 index 0000000..26ec9ed --- /dev/null +++ b/cmd/beastbench/workload.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" +) + +// OpType is a benchmarked operation kind. +type OpType int + +const ( + OpRead OpType = iota + OpUpdate + OpInsert + OpScan + OpRMW + numOps +) + +var opNames = [numOps]string{"read", "update", "insert", "scan", "rmw"} + +func (o OpType) String() string { return opNames[o] } + +// Workload is a YCSB-style operation mix; proportions sum to 1. +type Workload struct { + Name string + Desc string + Mix [numOps]float64 + Latest bool // read-latest key distribution (D) + Uniform bool // workload forces a distribution regardless of flag (none today) +} + +var workloads = map[string]Workload{ + "A": {Name: "A", Desc: "50% read / 50% update", Mix: mix(OpRead, .5, OpUpdate, .5)}, + "B": {Name: "B", Desc: "95% read / 5% update", Mix: mix(OpRead, .95, OpUpdate, .05)}, + "C": {Name: "C", Desc: "100% read", Mix: mix(OpRead, 1.0)}, + "D": {Name: "D", Desc: "95% read-latest / 5% insert", Mix: mix(OpRead, .95, OpInsert, .05), Latest: true}, + "E": {Name: "E", Desc: "95% short scan / 5% insert", Mix: mix(OpScan, .95, OpInsert, .05)}, + "F": {Name: "F", Desc: "50% read / 50% read-modify-write (conditional put)", Mix: mix(OpRead, .5, OpRMW, .5)}, +} + +func mix(kv ...any) [numOps]float64 { + var m [numOps]float64 + for i := 0; i < len(kv); i += 2 { + m[kv[i].(OpType)] = kv[i+1].(float64) + } + return m +} + +func lookupWorkload(name string) (Workload, error) { + w, ok := workloads[strings.ToUpper(name)] + if !ok { + return w, fmt.Errorf("unknown workload %q (want A-F)", name) + } + return w, nil +} + +// pick maps u in [0,1) to an operation. +func (w Workload) pick(u float64) OpType { + var acc float64 + last := OpRead + for op := OpType(0); op < numOps; op++ { + if w.Mix[op] == 0 { + continue + } + last = op + acc += w.Mix[op] + if u < acc { + return op + } + } + return last +} diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 0000000..231bf37 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,172 @@ +# Benchmarking BeastDB with `beastbench` + +`cmd/beastbench` is a reproducible, YCSB-style load generator that drives a +BeastDB server over gRPC. This document explains how to run it and how to run +*equivalent* workloads against Redis, Pebble and Badger so that a comparison is +fair. It deliberately contains no results: numbers depend on your hardware, +settings and dataset, and only numbers you measured yourself are meaningful. + +## Running beastbench + +``` +go build -o beastbench ./cmd/beastbench + +# against a running server (flags follow cmd/beastctl: token and TLS) +./beastbench --addr 127.0.0.1:50051 --token "$BEASTDB_API_TOKEN" \ + --workload A --dist zipfian --records 10000000 --ops 5000000 \ + --threads 64 --value-size 1024 --warmup 200000 --seed 42 --json > a.json + +# TLS / mTLS +./beastbench --addr db:50051 --tls-ca ca.pem --tls-cert client.pem --tls-key client-key.pem ... + +# quick smoke run: in-process engine + gRPC on a temp dir (loopback, no auth) +./beastbench --self-contained --workload B --records 5000 --ops 5000 --threads 4 +``` + +Flags: `--addr --token --tls-ca --tls-cert --tls-key --tls-server-name --conns` +(connection), `--workload A-F`, `--dist zipfian|uniform`, `--zipf-theta`, +`--records --ops --warmup --threads --value-size --max-scan-len --seed`, +`--json`, `--skip-load`, `--load-only`, `--self-contained --pool-size`. +`--token` defaults to `$BEASTDB_API_TOKEN`, the TLS flags to `$BEASTDB_TLS_*`. + +### Phases + +1. **Load**: inserts `--records` records (keys 1..N, deterministic values of + `--value-size` bytes) in parallel. Reported separately. +2. **Warmup** (optional): `--warmup` unmeasured operations of the chosen mix. +3. **Run**: `--ops` measured operations split across `--threads` workers. + +Each worker has its own RNG seeded from `--seed`, so a given +(seed, threads, ops) produces the same operation and key sequence. Timing and +server behaviour still vary run to run; repeat runs and report the spread. + +### Workloads + +| Name | Mix | Keys | +|---|---|---| +| A | 50% read, 50% update | zipfian / uniform | +| B | 95% read, 5% update | zipfian / uniform | +| C | 100% read | zipfian / uniform | +| D | 95% read, 5% insert | read-latest (zipfian offset from newest insert) | +| E | 95% scan (length uniform 1..`--max-scan-len`), 5% insert | zipfian / uniform start key | +| F | 50% read, 50% read-modify-write | zipfian / uniform | + +Workload F's read-modify-write is `Get` followed by `Put` with a +`VALUE_EQUALS` condition on the value read. A rejected condition (ABORTED) is +counted as a conflict, not an error, and is included in the `rmw` latency. +Latency for `rmw` covers both RPCs. + +Notes on semantics: + +- Zipfian uses the YCSB generator (theta 0.99 by default). Keys are *not* + hash-scrambled, so the hottest keys are the lowest ones; this keeps scans + contiguous but concentrates the hot set in one region of the B+ tree. +- Workloads D/E allocate new keys above the loaded range. A D read may hit a key + whose insert is still in flight; that is reported as a `miss`. +- Percentiles come from a log-linear histogram (128 sub-buckets per power of + two, under 1% error); max is exact. Latency is client-observed per call, + including gRPC and network time. There is no coordinated-omission correction: + workers are closed-loop, so a stalled server lowers offered load. Say so when + you report tail latencies. + +### Output + +Text by default; `--json` emits config, host info (Go version, OS, CPUs, +self-contained flag) and per-phase, per-operation count, errors, misses, +conflicts, ops/s, mean, p50, p95, p99 and max in microseconds. Keep the JSON +with any number you quote. + +## Fair comparison against Redis, Pebble and Badger + +A comparison is only as honest as its weakest control. Fix all of the following +and record them next to results. + +### Environment + +- **Same machine, same disk, run sequentially**, never concurrently. Record CPU + model, core count, RAM, disk model and filesystem, OS and kernel, Go version + and every system's version/commit. +- Disable CPU frequency scaling surprises (performance governor), keep the + machine otherwise idle, and use a fresh data directory per run on the same + device. +- **Dataset larger than RAM.** If the dataset fits in the page cache you are + benchmarking memory, not the storage engine. Choose `records x (value size + + per-key overhead)` at least 2-4x physical RAM (or constrain RAM with a + cgroup/container memory limit and state it). Check with `free`/`vmstat` that + the page cache did not absorb the working set, and drop caches between + phases if you intend a cold start (`sync; echo 3 > /proc/sys/vm/drop_caches`). +- State the **cache budgets** explicitly and make them comparable: BeastDB + `--pool-size` (4 KiB frames), Pebble block cache, Badger block/index cache, + Redis `maxmemory`. Do not leave one system on defaults while tuning another. +- Same key count, key size (BeastDB keys are `uint64`; use 8-byte keys + elsewhere, big-endian so ordering matches), value size, and value contents. + Values from `beastbench` are incompressible-ish pseudo-random bytes; disable + or equalise compression (Pebble/Badger default compression off or on must + match; Redis stores raw). + +### Durability: the biggest source of unfair numbers + +Compare like with like. State what each system guarantees when an +acknowledged write is followed by a power loss. + +| System | Per-write durable (fsync on ack) | Relaxed | +|---|---|---| +| BeastDB | Each write is acknowledged only after the WAL is synced (`docs/v1-semantics.md`); benchmark as shipped | no relaxed mode is documented; document any change you make | +| Redis | `appendonly yes`, `appendfsync always` | `appendfsync everysec`, or no AOF (RDB only) | +| Pebble | `pebble.Sync` on every write (`WriteOptions{Sync: true}`) | `pebble.NoSync` | +| Badger | `SyncWrites: true` | `SyncWrites: false` (default) | + +Run the per-write-durable configuration for all systems as the primary +comparison, and report any relaxed configuration as a separate, clearly +labelled table. A durable system compared against a non-durable one tells you +nothing. Also note that group commit/batching behaviour differs and is part of +what is being measured when many threads write concurrently. + +### Matching the access path + +- `beastbench` measures a *networked* client-server round trip over gRPC. Redis + has a network protocol too (use `redis-benchmark` or a small RESP client, + no pipelining unless you also pipeline everywhere). Pebble and Badger are + embedded libraries with no network hop; comparing them to a gRPC call + mostly measures the RPC. Either (a) wrap them in an equivalent gRPC/TCP + service and run the same `--threads`, or (b) benchmark BeastDB's engine + in-process (`go test ./internal/api -bench`) and compare to the embedded + libraries in-process. Do not mix the two. +- Use the same concurrency (`--threads`) and report results at several + concurrency levels, not one. +- Same operation mix, key distribution, warmup, run length, and seed-derived + key sequence. Implement the equivalent of each workload in the other + system's harness: A/B/C/D as get/put, E as an iterator seeking to the start + key and reading up to N entries (Redis: a sorted-set or key range scheme and + say which), F as get then a conditional/transactional write (Redis + `WATCH`/`MULTI` or a Lua CAS; Pebble/Badger: a transaction or CAS loop). + Document any place the semantics differ. +- Load the dataset the same way (same ordering and parallelism) and let + background work finish (compaction in Pebble/Badger, checkpoint in BeastDB, + `BGSAVE`/AOF rewrite in Redis) before the measured phase, or measure it + deliberately and say so. + +### Existing harnesses + +You may use YCSB itself (Redis binding exists; Pebble/Badger need a custom or +community binding) or `redis-benchmark`. If you do, map its parameters to the +above (recordcount, operationcount, fieldcount=1, fieldlength, request +distribution, threadcount) and note that YCSB's latency measurement differs +from `beastbench`'s. + +### Reporting + +- Publish the exact command lines, config files, versions, hardware and the raw + JSON/CSV outputs. +- Report several repetitions (at least 5) with median and range, not a single + best run. +- Report p50/p95/p99/max latency, not only throughput, plus error and conflict + counts. +- State what you did *not* control. Do not generalise from one workload, + dataset size or machine; do not claim one system is faster in general. + +### Smoke runs are not benchmarks + +`--self-contained` runs client and server in one process on loopback, with +whatever disk the temp dir lands on and no auth or TLS. Use it to validate the +harness and catch regressions in your own environment only. From c9d5113fa0dbe157de8df47bafe967d5edcc7aa6 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:23:00 +0530 Subject: [PATCH 03/10] feat(obs): Prometheus /metrics endpoint with gRPC/HTTP request metrics Hand-written text exposition (no new deps). Authenticated by admin session or bearer token (--api-token / BEASTDB_METRICS_TOKEN); no loopback bypass. Adds buffer-pool hit/miss counters and checkpoint timestamps, gRPC metrics interceptors, runbook docs with scrape config and alert rules. Co-Authored-By: Claude Sonnet 5.5 --- cmd/server/main.go | 6 +- cmd/server/web.go | 4 +- docs/runbooks/operator-guide.md | 74 +++++++++++ internal/api/engine.go | 4 + internal/api/engine_checkpoint.go | 8 +- internal/api/engine_stats.go | 32 +++-- internal/api/grpc_metrics.go | 35 ++++++ internal/api/grpc_metrics_test.go | 42 +++++++ internal/metrics/diskfree_other.go | 6 + internal/metrics/diskfree_unix.go | 15 +++ internal/metrics/metrics.go | 175 +++++++++++++++++++++++++++ internal/storage/buffer_pool.go | 11 ++ internal/web/handler/metrics.go | 155 ++++++++++++++++++++++++ internal/web/handler/metrics_test.go | 164 +++++++++++++++++++++++++ internal/web/server.go | 22 +++- 15 files changed, 738 insertions(+), 15 deletions(-) create mode 100644 internal/api/grpc_metrics.go create mode 100644 internal/api/grpc_metrics_test.go create mode 100644 internal/metrics/diskfree_other.go create mode 100644 internal/metrics/diskfree_unix.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/web/handler/metrics.go create mode 100644 internal/web/handler/metrics_test.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 624c994..8e34914 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -74,6 +74,8 @@ func main() { tokenStore := initTokenStore(engine) grpcOpts := api.MessageSizeServerOptions() + // Metrics interceptors run first so they also observe auth rejections. + grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(api.MetricsUnaryInterceptor()), grpc.ChainStreamInterceptor(api.MetricsStreamInterceptor())) if *tlsCert != "" && *tlsKey != "" { creds, err := loadServerTLS(*tlsCert, *tlsKey, *tlsCA) if err != nil { @@ -84,7 +86,7 @@ func main() { } if *apiToken != "" || tokenStore != nil { authInterceptor := api.NewAuthInterceptor(*apiToken, tokenStore) - grpcOpts = append(grpcOpts, grpc.UnaryInterceptor(authInterceptor.Unary()), grpc.StreamInterceptor(authInterceptor.Stream())) + grpcOpts = append(grpcOpts, grpc.ChainUnaryInterceptor(authInterceptor.Unary()), grpc.ChainStreamInterceptor(authInterceptor.Stream())) if *apiToken != "" { log.Printf("๐Ÿ” gRPC service protected with Bearer token authentication.") } @@ -145,7 +147,7 @@ func main() { } } - stopWeb := startWebServer(*webAddr, *dataDir, *role, *adminPassword, engine, partitions, tokenStore) + stopWeb := startWebServer(*webAddr, *dataDir, *role, *adminPassword, engine, partitions, tokenStore, *apiToken, os.Getenv("BEASTDB_METRICS_TOKEN")) defer stopWeb() lis, err := net.Listen("tcp", grpcBind) diff --git a/cmd/server/web.go b/cmd/server/web.go index c15d202..62fdb8d 100644 --- a/cmd/server/web.go +++ b/cmd/server/web.go @@ -10,7 +10,7 @@ import ( ) // startWebServer boots the HTTP admin console and seeds or synchronizes the admin credentials. -func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.Engine, partitions []handler.PartitionEntry, tokenStore *auth.TokenStore) func() { +func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.Engine, partitions []handler.PartitionEntry, tokenStore *auth.TokenStore, metricsTokens ...string) func() { if webAddr == "" { return func() {} } @@ -20,7 +20,7 @@ func startWebServer(webAddr, dataDir, role, adminPassword string, engine *api.En log.Fatalf("Failed to initialize session secret: %v", secretErr) } - webSrv, webErr := web.NewServer(webAddr, engine, secret, role, Version, partitions, tokenStore) + webSrv, webErr := web.NewServerWithOptions(webAddr, engine, secret, role, Version, partitions, web.Options{DataDir: dataDir, MetricsTokens: metricsTokens}, tokenStore) if webErr != nil { log.Fatalf("Failed to create web server: %v", webErr) } diff --git a/docs/runbooks/operator-guide.md b/docs/runbooks/operator-guide.md index afabf9e..61da4e7 100644 --- a/docs/runbooks/operator-guide.md +++ b/docs/runbooks/operator-guide.md @@ -64,6 +64,80 @@ The embedded HTTP server exposes unauthenticated health endpoints for Kubernetes --- +### Prometheus Metrics (`GET /metrics`) + +The web server exposes Prometheus text exposition (format 0.0.4) at `/metrics` on `--web-addr`. It is **never anonymous**: there is no loopback exemption, because a reverse proxy or sidecar on the same host makes every remote peer look like `127.0.0.1`. A request is accepted only with one of: + +- `Authorization: Bearer ` equal to the gRPC `--api-token`, or to `BEASTDB_METRICS_TOKEN` (preferred: a dedicated scrape-only secret), or +- an admin web session cookie. + +With neither token configured, only admin sessions work (Prometheus cannot scrape). Set `BEASTDB_METRICS_TOKEN` and keep the web listener on loopback or a private network/TLS-terminating proxy, since the bearer token travels in cleartext over plain HTTP. + +| Metric | Meaning | +|---|---| +| `beastdb_requests_total{protocol,method,code}` | gRPC (code name) and HTTP (status, route pattern) request counts; errors are non-`OK` / 4xx-5xx | +| `beastdb_request_duration_seconds` (histogram) | latency per protocol/method (streams: total stream duration) | +| `beastdb_grpc_requests_in_flight` | executing gRPC calls | +| `beastdb_wal_size_bytes`, `beastdb_wal_current_lsn` | WAL file size and highest LSN | +| `beastdb_checkpoint_last_lsn`, `beastdb_checkpoint_age_seconds` | last checkpoint by this process (series absent until the first checkpoint after start) | +| `beastdb_buffer_pool_{size,cached,pinned,dirty}_pages`, `beastdb_buffer_pool_hits_total` / `_misses_total` / `_hit_ratio` | buffer pool state; ratio is lifetime, use `rate()` of the counters for recent behavior | +| `beastdb_data_file_size_bytes`, `beastdb_disk_free_bytes` | data file size, free space on the data dir filesystem (unix only) | +| `beastdb_read_only` | 1 when writes are rejected (follower or fail-stop after a failed apply) | +| `beastdb_replica_applied_lsn` | followers only: last applied LSN | +| `beastdb_build_info{version,role}` | always 1 | + +Replica lag is computed in PromQL: `max(beastdb_wal_current_lsn{role="leader"}) - beastdb_replica_applied_lsn` (add a `role` label via relabeling, or key on `instance`). + +Sample scrape config (`prometheus.yml`): + +```yaml +scrape_configs: + - job_name: beastdb + metrics_path: /metrics + scrape_interval: 15s + authorization: + type: Bearer + credentials_file: /etc/prometheus/beastdb.token # contains BEASTDB_METRICS_TOKEN + static_configs: + - targets: ["leader.internal:8080"] + labels: { role: leader } + - targets: ["replica1.internal:8080", "replica2.internal:8080"] + labels: { role: follower } +``` + +Example alert rules: + +```yaml +groups: + - name: beastdb + rules: + - alert: BeastDBDown + expr: up{job="beastdb"} == 0 + for: 2m + - alert: BeastDBHighErrorRate + expr: | + sum by (instance) (rate(beastdb_requests_total{protocol="grpc",code!~"OK|NotFound|AlreadyExists|FailedPrecondition"}[5m])) + / sum by (instance) (rate(beastdb_requests_total{protocol="grpc"}[5m])) > 0.05 + for: 5m + - alert: BeastDBHighP99Latency + expr: histogram_quantile(0.99, sum by (instance, le) (rate(beastdb_request_duration_seconds_bucket{protocol="grpc",method!~".*Sync.*|.*Scan.*"}[5m]))) > 0.25 + for: 10m + - alert: BeastDBWALGrowing + expr: beastdb_wal_size_bytes > 1073741824 or beastdb_checkpoint_age_seconds > 600 + for: 10m + - alert: BeastDBDiskLow + expr: beastdb_disk_free_bytes < 5 * 1024^3 + for: 5m + - alert: BeastDBReplicaLagging + expr: max(beastdb_wal_current_lsn{role="leader"}) - on() group_right() beastdb_replica_applied_lsn > 10000 + for: 5m + - alert: BeastDBFailStop + expr: beastdb_read_only{role="leader"} == 1 + for: 1m +``` + +--- + ## 5. Replica Promotion & Failover BeastDB V1 does not employ automatic distributed consensus. In the event of leader loss: diff --git a/internal/api/engine.go b/internal/api/engine.go index 19c13dd..decd4a5 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sync" + "sync/atomic" "github.com/ChromaBeast/beastdb/internal/index" "github.com/ChromaBeast/beastdb/internal/storage" @@ -24,6 +25,9 @@ type Engine struct { failed error // set by halt; blocks writes until restart reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree mu sync.RWMutex + + lastCkptLSN atomic.Uint64 // LSN at the last successful checkpoint (metrics) + lastCkptUnix atomic.Int64 // unix seconds of the last successful checkpoint } // NewEngine initializes the database engine components with recovery support. diff --git a/internal/api/engine_checkpoint.go b/internal/api/engine_checkpoint.go index 91036ba..8a2e085 100644 --- a/internal/api/engine_checkpoint.go +++ b/internal/api/engine_checkpoint.go @@ -54,7 +54,13 @@ func (e *Engine) Checkpoint() error { if err := e.disk.Sync(); err != nil { return fmt.Errorf("checkpoint sync data file: %w", err) } - return e.wal.Checkpoint() + ckptLSN := e.wal.CurrentLSN() + if err := e.wal.Checkpoint(); err != nil { + return err + } + e.lastCkptLSN.Store(ckptLSN) + e.lastCkptUnix.Store(time.Now().Unix()) + return nil } // StartAutoCheckpoint checkpoints whenever the WAL exceeds maxWALBytes, polling every diff --git a/internal/api/engine_stats.go b/internal/api/engine_stats.go index c4aa1f2..f776a03 100644 --- a/internal/api/engine_stats.go +++ b/internal/api/engine_stats.go @@ -15,6 +15,13 @@ type StorageMetrics struct { WALSizeBytes int64 `json:"walSizeBytes"` CurrentLSN uint64 `json:"currentLSN"` ActiveDataPage uint64 `json:"activeDataPage"` + + BufferHits uint64 `json:"bufferHits"` + BufferMisses uint64 `json:"bufferMisses"` + LastCheckpointLSN uint64 `json:"lastCheckpointLSN"` + // LastCheckpointUnix is 0 until a checkpoint completes in this process. + LastCheckpointUnix int64 `json:"lastCheckpointUnix"` + ReadOnly bool `json:"readOnly"` } // StorageMetrics collects live runtime storage indicators. @@ -25,15 +32,22 @@ func (e *Engine) StorageMetrics() StorageMetrics { numPages := e.disk.NumPages() poolSize, cached, pinned, dirty := e.bpm.Stats() + hits, misses := e.bpm.HitMiss() + return StorageMetrics{ - TotalPages: numPages, - DiskSizeBytes: numPages * storage.PageSize, - PoolSize: poolSize, - CachedPages: cached, - PinnedPages: pinned, - DirtyPages: dirty, - WALSizeBytes: e.wal.FileSize(), - CurrentLSN: e.wal.CurrentLSN(), - ActiveDataPage: e.activeDataPage, + BufferHits: hits, + BufferMisses: misses, + LastCheckpointLSN: e.lastCkptLSN.Load(), + LastCheckpointUnix: e.lastCkptUnix.Load(), + ReadOnly: e.readOnly, + TotalPages: numPages, + DiskSizeBytes: numPages * storage.PageSize, + PoolSize: poolSize, + CachedPages: cached, + PinnedPages: pinned, + DirtyPages: dirty, + WALSizeBytes: e.wal.FileSize(), + CurrentLSN: e.wal.CurrentLSN(), + ActiveDataPage: e.activeDataPage, } } diff --git a/internal/api/grpc_metrics.go b/internal/api/grpc_metrics.go new file mode 100644 index 0000000..b501bc5 --- /dev/null +++ b/internal/api/grpc_metrics.go @@ -0,0 +1,35 @@ +package api + +import ( + "context" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/status" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +// MetricsUnaryInterceptor records count, status code and latency of unary RPCs. +func MetricsUnaryInterceptor() grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, h grpc.UnaryHandler) (any, error) { + metrics.InFlight.Add(1) + start := time.Now() + resp, err := h(ctx, req) + metrics.InFlight.Add(-1) + metrics.Default.Observe("grpc", info.FullMethod, status.Code(err).String(), time.Since(start)) + return resp, err + } +} + +// MetricsStreamInterceptor records count, status code and total duration of streaming RPCs. +func MetricsStreamInterceptor() grpc.StreamServerInterceptor { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, h grpc.StreamHandler) error { + metrics.InFlight.Add(1) + start := time.Now() + err := h(srv, ss) + metrics.InFlight.Add(-1) + metrics.Default.Observe("grpc", info.FullMethod, status.Code(err).String(), time.Since(start)) + return err + } +} diff --git a/internal/api/grpc_metrics_test.go b/internal/api/grpc_metrics_test.go new file mode 100644 index 0000000..a582b97 --- /dev/null +++ b/internal/api/grpc_metrics_test.go @@ -0,0 +1,42 @@ +package api + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +func TestMetricsInterceptors(t *testing.T) { + u := MetricsUnaryInterceptor() + _, err := u(context.Background(), nil, &grpc.UnaryServerInfo{FullMethod: "/t.S/Unary"}, + func(context.Context, any) (any, error) { return nil, status.Error(codes.PermissionDenied, "no") }) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("error not propagated: %v", err) + } + s := MetricsStreamInterceptor() + if err := s(nil, nil, &grpc.StreamServerInfo{FullMethod: "/t.S/Stream"}, + func(any, grpc.ServerStream) error { return nil }); err != nil { + t.Fatal(err) + } + var b strings.Builder + metrics.Default.WriteRequests(metrics.NewWriter(&b)) + out := b.String() + for _, want := range []string{ + `beastdb_requests_total{protocol="grpc",method="/t.S/Unary",code="PermissionDenied"} 1`, + `beastdb_requests_total{protocol="grpc",method="/t.S/Stream",code="OK"} 1`, + `beastdb_request_duration_seconds_count{protocol="grpc",method="/t.S/Unary"} 1`, + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q", want) + } + } + if metrics.InFlight.Load() != 0 { + t.Fatalf("in-flight leaked: %d", metrics.InFlight.Load()) + } +} diff --git a/internal/metrics/diskfree_other.go b/internal/metrics/diskfree_other.go new file mode 100644 index 0000000..5e22402 --- /dev/null +++ b/internal/metrics/diskfree_other.go @@ -0,0 +1,6 @@ +//go:build !unix + +package metrics + +// DiskFreeBytes is unsupported on this platform. +func DiskFreeBytes(string) (uint64, bool) { return 0, false } diff --git a/internal/metrics/diskfree_unix.go b/internal/metrics/diskfree_unix.go new file mode 100644 index 0000000..17c4433 --- /dev/null +++ b/internal/metrics/diskfree_unix.go @@ -0,0 +1,15 @@ +//go:build unix + +package metrics + +import "syscall" + +// DiskFreeBytes returns bytes available to unprivileged users on the +// filesystem holding path. +func DiskFreeBytes(path string) (uint64, bool) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, false + } + return uint64(st.Bavail) * uint64(st.Bsize), true +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..77646f4 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,175 @@ +// Package metrics is a tiny dependency-free Prometheus text-format (0.0.4) +// exposition layer: request counters, latency histograms and gauge helpers. +package metrics + +import ( + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// DefaultBuckets are latency histogram upper bounds in seconds. +var DefaultBuckets = []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10} + +type hist struct { + counts []uint64 // per-bucket (non-cumulative), len = len(bounds)+1 (+Inf) + sum float64 + n uint64 +} + +type reqKey struct{ proto, method, code string } + +// RequestMetrics records request counts (by protocol, method, status code) +// and latency histograms (by protocol, method). +type RequestMetrics struct { + mu sync.Mutex + count map[reqKey]uint64 + hists map[[2]string]*hist +} + +// NewRequestMetrics creates an empty collector. +func NewRequestMetrics() *RequestMetrics { + return &RequestMetrics{count: map[reqKey]uint64{}, hists: map[[2]string]*hist{}} +} + +// Observe records one finished request. code is a short status string +// (gRPC code name or HTTP status number). +func (r *RequestMetrics) Observe(proto, method, code string, d time.Duration) { + r.mu.Lock() + defer r.mu.Unlock() + r.count[reqKey{proto, method, code}]++ + hk := [2]string{proto, method} + h := r.hists[hk] + if h == nil { + h = &hist{counts: make([]uint64, len(DefaultBuckets)+1)} + r.hists[hk] = h + } + s := d.Seconds() + i := sort.SearchFloat64s(DefaultBuckets, s) // first bound >= s + h.counts[i]++ + h.sum += s + h.n++ +} + +// Default is the process-wide request collector shared by the gRPC +// interceptors and the HTTP middleware. +var Default = NewRequestMetrics() + +// InFlight tracks currently executing gRPC requests. +var InFlight atomic.Int64 + +// Writer emits Prometheus text exposition. +type Writer struct { + w io.Writer + seen map[string]bool +} + +// NewWriter wraps w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w, seen: map[string]bool{}} } + +func (w *Writer) header(name, help, typ string) { + if w.seen[name] { + return + } + w.seen[name] = true + fmt.Fprintf(w.w, "# HELP %s %s\n# TYPE %s %s\n", name, help, name, typ) +} + +// Escape escapes a label value. +func Escape(v string) string { + return strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`).Replace(v) +} + +func labels(kv []string) string { + if len(kv) == 0 { + return "" + } + parts := make([]string, 0, len(kv)/2) + for i := 0; i+1 < len(kv); i += 2 { + parts = append(parts, kv[i]+`="`+Escape(kv[i+1])+`"`) + } + return "{" + strings.Join(parts, ",") + "}" +} + +func fmtFloat(v float64) string { + switch { + case math.IsInf(v, 1): + return "+Inf" + case math.IsInf(v, -1): + return "-Inf" + case math.IsNaN(v): + return "NaN" + } + return strconv.FormatFloat(v, 'g', -1, 64) +} + +// Gauge writes a gauge sample. kv are alternating label names/values. +func (w *Writer) Gauge(name, help string, v float64, kv ...string) { + w.header(name, help, "gauge") + fmt.Fprintf(w.w, "%s%s %s\n", name, labels(kv), fmtFloat(v)) +} + +// Counter writes a counter sample. +func (w *Writer) Counter(name, help string, v float64, kv ...string) { + w.header(name, help, "counter") + fmt.Fprintf(w.w, "%s%s %s\n", name, labels(kv), fmtFloat(v)) +} + +// WriteRequests emits the request counter and latency histogram families. +func (r *RequestMetrics) WriteRequests(w *Writer) { + r.mu.Lock() + keys := make([]reqKey, 0, len(r.count)) + counts := make(map[reqKey]uint64, len(r.count)) + for k, v := range r.count { + keys = append(keys, k) + counts[k] = v + } + hk := make([][2]string, 0, len(r.hists)) + hs := make(map[[2]string]hist, len(r.hists)) + for k, h := range r.hists { + hk = append(hk, k) + hs[k] = hist{counts: append([]uint64(nil), h.counts...), sum: h.sum, n: h.n} + } + r.mu.Unlock() + + sort.Slice(keys, func(i, j int) bool { + a, b := keys[i], keys[j] + if a.proto != b.proto { + return a.proto < b.proto + } + if a.method != b.method { + return a.method < b.method + } + return a.code < b.code + }) + for _, k := range keys { + w.Counter("beastdb_requests_total", "Requests handled, by protocol, method and status code.", + float64(counts[k]), "protocol", k.proto, "method", k.method, "code", k.code) + } + sort.Slice(hk, func(i, j int) bool { + if hk[i][0] != hk[j][0] { + return hk[i][0] < hk[j][0] + } + return hk[i][1] < hk[j][1] + }) + const name = "beastdb_request_duration_seconds" + for _, k := range hk { + h := hs[k] + w.header(name, "Request latency in seconds.", "histogram") + var cum uint64 + for i, b := range DefaultBuckets { + cum += h.counts[i] + fmt.Fprintf(w.w, "%s_bucket%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1], "le", fmtFloat(b)}), cum) + } + cum += h.counts[len(DefaultBuckets)] + fmt.Fprintf(w.w, "%s_bucket%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1], "le", "+Inf"}), cum) + fmt.Fprintf(w.w, "%s_sum%s %s\n", name, labels([]string{"protocol", k[0], "method", k[1]}), fmtFloat(h.sum)) + fmt.Fprintf(w.w, "%s_count%s %d\n", name, labels([]string{"protocol", k[0], "method", k[1]}), h.n) + } +} diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index 19d4a96..6e14a97 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -3,6 +3,7 @@ package storage import ( "errors" "sync" + "sync/atomic" ) var ( @@ -22,6 +23,8 @@ type BufferPoolManager struct { clockHand int mu sync.RWMutex pagePool sync.Pool // Fix #6: reusable 4KB page buffers + + hits, misses atomic.Uint64 // FetchPage cache hit/miss counters } // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. @@ -48,6 +51,7 @@ func (bpm *BufferPoolManager) FetchPage(pageID uint64) (*SlottedPage, error) { frame.Pin() page := frame.Page bpm.mu.RUnlock() + bpm.hits.Add(1) return page, nil } bpm.mu.RUnlock() @@ -59,9 +63,11 @@ func (bpm *BufferPoolManager) FetchPage(pageID uint64) (*SlottedPage, error) { // Double-check: another goroutine may have loaded the page while we waited. if idx, exists := bpm.pageTable[pageID]; exists { bpm.frames[idx].Pin() + bpm.hits.Add(1) return bpm.frames[idx].Page, nil } + bpm.misses.Add(1) return bpm.loadPage(pageID) } @@ -151,6 +157,11 @@ func (bpm *BufferPoolManager) FlushAll() error { return bpm.disk.Sync() } +// HitMiss returns cumulative FetchPage cache hits and misses. +func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { + return bpm.hits.Load(), bpm.misses.Load() +} + // Stats returns operational metrics for the buffer pool. func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { bpm.mu.RLock() diff --git a/internal/web/handler/metrics.go b/internal/web/handler/metrics.go new file mode 100644 index 0000000..af15807 --- /dev/null +++ b/internal/web/handler/metrics.go @@ -0,0 +1,155 @@ +package handler + +import ( + "bytes" + "crypto/subtle" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ChromaBeast/beastdb/internal/metrics" + "github.com/ChromaBeast/beastdb/internal/web/auth" +) + +// MetricsConfig configures the /metrics endpoint. +type MetricsConfig struct { + Role string + Version string + // DataDir is used for the disk-free gauge (omitted when empty). + DataDir string + // Tokens are static bearer tokens accepted for scraping (e.g. the gRPC + // --api-token and BEASTDB_METRICS_TOKEN). Empty entries are ignored. + Tokens []string + // Sessions authenticates admin browser sessions. May be nil. + Sessions *auth.SessionManager +} + +var processStart = time.Now() + +// authorized reports whether r carries an admin session or a configured +// bearer token. There is deliberately no unauthenticated/loopback bypass: a +// reverse proxy on the same host would make every remote peer look like +// loopback. +func (c MetricsConfig) authorized(r *http.Request) bool { + if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") { + got := []byte(strings.TrimSpace(strings.TrimPrefix(h, "Bearer "))) + for _, t := range c.Tokens { + t = strings.TrimSpace(t) + if t != "" && subtle.ConstantTimeCompare(got, []byte(t)) == 1 { + return true + } + } + return false + } + if c.Sessions != nil { + if _, role, err := c.Sessions.ValidateSession(r); err == nil && role == auth.RoleAdmin { + return true + } + } + return false +} + +// MetricsHandler serves Prometheus text exposition (version 0.0.4). +func MetricsHandler(engine EngineReader, cfg MetricsConfig) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + if !cfg.authorized(r) { + w.Header().Set("WWW-Authenticate", `Bearer realm="beastdb-metrics"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + var buf bytes.Buffer + WriteMetrics(&buf, engine, cfg) + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + _, _ = w.Write(buf.Bytes()) + }) +} + +// WriteMetrics renders all metrics. +func WriteMetrics(buf *bytes.Buffer, engine EngineReader, cfg MetricsConfig) { + w := metrics.NewWriter(buf) + w.Gauge("beastdb_build_info", "Build information; value is always 1.", 1, + "version", cfg.Version, "role", cfg.Role) + w.Gauge("beastdb_process_start_time_seconds", "Process start time, unix seconds.", float64(processStart.Unix())) + w.Gauge("beastdb_uptime_seconds", "Seconds since process start.", time.Since(processStart).Seconds()) + + if engine != nil { + m := engine.StorageMetrics() + w.Gauge("beastdb_wal_size_bytes", "Current WAL file size in bytes.", float64(m.WALSizeBytes)) + w.Gauge("beastdb_wal_current_lsn", "Highest LSN written to the WAL.", float64(m.CurrentLSN)) + w.Gauge("beastdb_checkpoint_last_lsn", "LSN of the last checkpoint completed by this process (0 if none).", float64(m.LastCheckpointLSN)) + if m.LastCheckpointUnix > 0 { + w.Gauge("beastdb_checkpoint_last_timestamp_seconds", "Unix time of the last checkpoint.", float64(m.LastCheckpointUnix)) + age := time.Since(time.Unix(m.LastCheckpointUnix, 0)).Seconds() + if age < 0 { + age = 0 + } + w.Gauge("beastdb_checkpoint_age_seconds", "Seconds since the last checkpoint.", age) + } + w.Gauge("beastdb_buffer_pool_size_pages", "Buffer pool capacity in pages.", float64(m.PoolSize)) + w.Gauge("beastdb_buffer_pool_cached_pages", "Pages resident in the buffer pool.", float64(m.CachedPages)) + w.Gauge("beastdb_buffer_pool_pinned_pages", "Pinned buffer pool pages.", float64(m.PinnedPages)) + w.Gauge("beastdb_buffer_pool_dirty_pages", "Dirty buffer pool pages.", float64(m.DirtyPages)) + w.Counter("beastdb_buffer_pool_hits_total", "Buffer pool fetches served from memory.", float64(m.BufferHits)) + w.Counter("beastdb_buffer_pool_misses_total", "Buffer pool fetches that read from disk.", float64(m.BufferMisses)) + if total := m.BufferHits + m.BufferMisses; total > 0 { + w.Gauge("beastdb_buffer_pool_hit_ratio", "Lifetime buffer pool hit ratio (0..1).", float64(m.BufferHits)/float64(total)) + } + w.Gauge("beastdb_data_file_size_bytes", "Data file size in bytes.", float64(m.DiskSizeBytes)) + w.Gauge("beastdb_data_pages", "Total pages in the data file.", float64(m.TotalPages)) + ro := 0.0 + if m.ReadOnly { + ro = 1 + } + w.Gauge("beastdb_read_only", "1 if the engine rejects writes (follower or fail-stop).", ro) + if cfg.Role != "leader" { + w.Gauge("beastdb_replica_applied_lsn", + "Last LSN applied by this follower. Lag = leader beastdb_wal_current_lsn minus this.", float64(engine.CurrentLSN())) + } + } + if cfg.DataDir != "" { + if free, ok := metrics.DiskFreeBytes(filepath.Clean(cfg.DataDir)); ok { + w.Gauge("beastdb_disk_free_bytes", "Free bytes on the data directory filesystem.", float64(free)) + } + } + w.Gauge("beastdb_grpc_requests_in_flight", "gRPC requests currently executing.", float64(metrics.InFlight.Load())) + metrics.Default.WriteRequests(w) +} + +type statusRecorder struct { + http.ResponseWriter + code int +} + +func (s *statusRecorder) WriteHeader(c int) { + if s.code == 0 { + s.code = c + } + s.ResponseWriter.WriteHeader(c) +} + +func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter } + +// HTTPMetrics records request count, status and latency for every HTTP +// request. The route label is the matched mux pattern (bounded cardinality). +func HTTPMetrics(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w} + next.ServeHTTP(rec, r) + code := rec.code + if code == 0 { + code = http.StatusOK + } + route := r.Pattern + if route == "" { + route = "unmatched" + } + metrics.Default.Observe("http", route, strconv.Itoa(code), time.Since(start)) + }) +} diff --git a/internal/web/handler/metrics_test.go b/internal/web/handler/metrics_test.go new file mode 100644 index 0000000..7b04a64 --- /dev/null +++ b/internal/web/handler/metrics_test.go @@ -0,0 +1,164 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/ChromaBeast/beastdb/internal/metrics" +) + +var sampleRe = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*(\{([a-zA-Z_][a-zA-Z0-9_]*="([^"\\]|\\.)*"(,|))*\})? (-?[0-9][0-9eE.+-]*|\+Inf|-Inf|NaN)$`) + +// validateExposition checks Prometheus text format basics and histogram +// consistency (cumulative buckets, +Inf bucket == _count). +func validateExposition(t *testing.T, body string) { + t.Helper() + typed := map[string]string{} + infBucket := map[string]string{} + count := map[string]string{} + for _, line := range strings.Split(strings.TrimSuffix(body, "\n"), "\n") { + if line == "" { + t.Fatalf("blank line in exposition") + } + if strings.HasPrefix(line, "# TYPE ") { + f := strings.Fields(line) + if len(f) != 4 { + t.Fatalf("bad TYPE line %q", line) + } + if _, dup := typed[f[2]]; dup { + t.Fatalf("duplicate TYPE for %s", f[2]) + } + typed[f[2]] = f[3] + continue + } + if strings.HasPrefix(line, "#") { + continue + } + if !sampleRe.MatchString(line) { + t.Fatalf("invalid sample line %q", line) + } + if strings.Contains(line, `le="+Inf"`) { + i := strings.LastIndex(line, " ") + key := strings.Replace(line[:i], `,le="+Inf"`, "", 1) + infBucket[strings.Replace(key, "_bucket", "", 1)] = line[i+1:] + } else if strings.Contains(line, "_count") { + i := strings.LastIndex(line, " ") + count[strings.Replace(line[:i], "_count", "", 1)] = line[i+1:] + } + } + for k, v := range infBucket { + if count[k] != v { + t.Fatalf("histogram %s: +Inf bucket %s != count %s", k, v, count[k]) + } + } + if len(infBucket) == 0 { + t.Fatal("no histogram series found") + } + // Bucket monotonicity. + var prevKey string + var prev float64 + for _, line := range strings.Split(body, "\n") { + if !strings.Contains(line, "_bucket{") { + continue + } + i := strings.LastIndex(line, " ") + v, _ := strconv.ParseFloat(line[i+1:], 64) + j := strings.Index(line, ",le=") + key := line[:j] + if key == prevKey && v < prev { + t.Fatalf("non-cumulative buckets at %q", line) + } + prevKey, prev = key, v + } +} + +func TestMetricsAuthAndFormat(t *testing.T) { + metrics.Default.Observe("grpc", "/beastdb.v1.KV/Get", "OK", 3*time.Millisecond) + metrics.Default.Observe("grpc", "/beastdb.v1.KV/Get", "NotFound", 30*time.Second) + metrics.Default.Observe("http", "/healthz", "200", time.Millisecond) + + h := MetricsHandler(&mockEngineReader{lsn: 7}, MetricsConfig{ + Role: "follower", Version: `v"1`, DataDir: t.TempDir(), Tokens: []string{"s3cret", ""}, + }) + + // No credentials, and an empty-bearer attempt, must be rejected. + for _, hdr := range []string{"", "Bearer ", "Bearer wrong", "Basic s3cret"} { + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + if hdr != "" { + req.Header.Set("Authorization", hdr) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("auth header %q: got %d want 401", hdr, rec.Code) + } + } + // Loopback peers get no special treatment. + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.RemoteAddr = "127.0.0.1:1234" + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("loopback anonymous: got %d want 401", rec.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.Header.Set("Authorization", "Bearer s3cret") + rec = httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("authorized: got %d", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain; version=0.0.4") { + t.Fatalf("content type %q", ct) + } + body := rec.Body.String() + validateExposition(t, body) + for _, want := range []string{ + `beastdb_build_info{version="v\"1",role="follower"} 1`, + "beastdb_wal_size_bytes ", "beastdb_wal_current_lsn ", "beastdb_buffer_pool_dirty_pages ", + "beastdb_buffer_pool_pinned_pages ", "beastdb_replica_applied_lsn 7", + `beastdb_requests_total{protocol="grpc",method="/beastdb.v1.KV/Get",code="NotFound"} 1`, + `beastdb_request_duration_seconds_bucket{protocol="grpc",method="/beastdb.v1.KV/Get",le="+Inf"} 2`, + "beastdb_disk_free_bytes ", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in output", want) + } + } + + if rec := httptest.NewRecorder(); func() int { + h.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/metrics", nil)) + return rec.Code + }() != http.StatusMethodNotAllowed { + t.Fatal("POST should be 405") + } +} + +func TestMetricsNoTokensRejectsBearer(t *testing.T) { + h := MetricsHandler(&mockEngineReader{}, MetricsConfig{Role: "leader", Version: "x"}) + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.Header.Set("Authorization", "Bearer ") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("got %d want 401", rec.Code) + } +} + +func TestHTTPMetricsMiddleware(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/teapot", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusTeapot) }) + h := HTTPMetrics(mux) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/teapot", nil)) + var b strings.Builder + metrics.Default.WriteRequests(metrics.NewWriter(&b)) + if !strings.Contains(b.String(), `beastdb_requests_total{protocol="http",method="/teapot",code="418"}`) { + t.Fatalf("http request not recorded:\n%s", b.String()) + } +} diff --git a/internal/web/server.go b/internal/web/server.go index 1426ffe..7d35e84 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -30,6 +30,20 @@ type EngineBackend interface { // partitions is the optional project-defined partition registry served to the Studio. // Call Serve() to start accepting connections. func NewServer(addr string, engine EngineBackend, secret []byte, role, version string, partitions []handler.PartitionEntry, tokenStore ...*auth.TokenStore) (*Server, error) { + return NewServerWithOptions(addr, engine, secret, role, version, partitions, Options{}, tokenStore...) +} + +// Options carries optional settings for NewServerWithOptions. +type Options struct { + // DataDir enables the disk-free metric. + DataDir string + // MetricsTokens are bearer tokens accepted by /metrics in addition to + // admin sessions. With none configured only admin sessions may scrape. + MetricsTokens []string +} + +// NewServerWithOptions is NewServer plus optional settings. +func NewServerWithOptions(addr string, engine EngineBackend, secret []byte, role, version string, partitions []handler.PartitionEntry, opts Options, tokenStore ...*auth.TokenStore) (*Server, error) { staticFS, err := fs.Sub(staticFiles, "static") if err != nil { return nil, err @@ -63,6 +77,12 @@ func NewServer(addr string, engine EngineBackend, secret []byte, role, version s mux.HandleFunc("/healthz", handler.HealthzHandler(version)) mux.HandleFunc("/readyz", handler.ReadyzHandler(engine, role)) + // Prometheus metrics: admin session or bearer token required (never anonymous). + mux.Handle("/metrics", handler.MetricsHandler(engine, handler.MetricsConfig{ + Role: role, Version: version, DataDir: opts.DataDir, + Tokens: opts.MetricsTokens, Sessions: sessions, + })) + // Public auth routes mux.HandleFunc("/login", handler.LoginHandler(deps)) mux.HandleFunc("/logout", handler.LogoutHandler(deps)) @@ -114,7 +134,7 @@ func NewServer(addr string, engine EngineBackend, secret []byte, role, version s srv := &http.Server{ Addr: addr, - Handler: securityHeaders(mux), + Handler: securityHeaders(handler.HTTPMetrics(mux)), } return &Server{httpServer: srv}, nil From 7ccd824c54ff69f8a66cc5088e508d06259a5cc0 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:23:27 +0530 Subject: [PATCH 04/10] perf(wal,engine): group commit - concurrent writers share one fsync WAL gains Append (assign LSN, no fsync), WaitDurable/Sync with a shared syncedLSN so one fsync covers all appended records, and a sticky fsync error. Write keeps its signature and durability but now group-commits too. Engine writes (Put/PutIfAbsent/PutIf/Delete/DeleteIf/BatchWrite[If] and replicated apply) go through a commit queue: a committer goroutine runs a group of writes under the engine lock (append + apply), issues one fsync, notifies the observer in LSN order, then releases the lock and acks writers. Reads and replication never see a write before it is durable. A failed fsync halts the engine and fails every write in the group. Concurrent Put benchmark: 7.82 ms/op -> 0.51 ms/op (~15x). Co-Authored-By: Claude Sonnet 5.5 --- internal/api/engine.go | 15 +-- internal/api/engine_batch.go | 13 +- internal/api/engine_checkpoint.go | 8 ++ internal/api/engine_commit.go | 154 +++++++++++++++++++++++ internal/api/engine_conditional.go | 36 +++--- internal/api/engine_delete.go | 8 +- internal/api/engine_groupcommit_test.go | 161 ++++++++++++++++++++++++ internal/api/engine_ops.go | 30 +++-- internal/wal/group_commit_test.go | 111 ++++++++++++++++ internal/wal/wal.go | 112 +++++++++++++++-- 10 files changed, 584 insertions(+), 64 deletions(-) create mode 100644 internal/api/engine_commit.go create mode 100644 internal/api/engine_groupcommit_test.go create mode 100644 internal/wal/group_commit_test.go diff --git a/internal/api/engine.go b/internal/api/engine.go index 19c13dd..07f454c 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -21,9 +21,11 @@ type Engine struct { observer CommitObserver secIndex *index.SecondaryIndex readOnly bool - failed error // set by halt; blocks writes until restart - reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree + failed error // set by halt; blocks writes until restart + reclaim map[uint64]int // data page -> free bytes after compaction, or unknownFree mu sync.RWMutex + cq committer // group-commit queue (engine_commit.go) + pending []pendingNotify // commit events awaiting the group fsync; guarded by mu } // NewEngine initializes the database engine components with recovery support. @@ -114,18 +116,14 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { return nil } keyUint := binary.LittleEndian.Uint64(key) - e.mu.Lock() - defer e.mu.Unlock() - return e.putLocked(keyUint, value) + return e.commit(func() error { return e.putLocked(keyUint, value) }) case wal.OpDelete: if len(key) < 8 { return nil } keyUint := binary.LittleEndian.Uint64(key) - e.mu.Lock() - defer e.mu.Unlock() - return e.deleteLocked(keyUint) + return e.commit(func() error { return e.deleteLocked(keyUint) }) case wal.OpBatch: batchOps, err := wal.DecodeBatchPayload(value) @@ -160,4 +158,3 @@ func (e *Engine) ApplyReplicatedRecord(opType byte, key, value []byte) error { return nil } } - diff --git a/internal/api/engine_batch.go b/internal/api/engine_batch.go index 1fff44e..382869d 100644 --- a/internal/api/engine_batch.go +++ b/internal/api/engine_batch.go @@ -50,8 +50,11 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error return err } - e.mu.Lock() - defer e.mu.Unlock() + return e.commit(func() error { return e.batchWriteLocked(ops, conds) }) +} + +// batchWriteLocked runs under e.mu inside the group-commit pipeline. +func (e *Engine) batchWriteLocked(ops []BatchOperation, conds []KeyCondition) error { if e.failed != nil { return e.failed } @@ -87,7 +90,7 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error } // 2. Commit the entire batch atomically to the WAL - lsn, err := e.wal.Write(wal.OpBatch, nil, payload) + lsn, err := e.wal.Append(wal.OpBatch, nil, payload) if err != nil { return e.halt(err) } @@ -108,9 +111,7 @@ func (e *Engine) BatchWriteIf(ops []BatchOperation, conds []KeyCondition) error } // 4. Notify replication observer of the atomic batch commit as a single unit - if e.observer != nil { - e.observer.OnCommit(lsn, wal.OpBatch, nil, payload) - } + e.queueNotify(lsn, wal.OpBatch, nil, payload) return nil } diff --git a/internal/api/engine_checkpoint.go b/internal/api/engine_checkpoint.go index 91036ba..c13cef3 100644 --- a/internal/api/engine_checkpoint.go +++ b/internal/api/engine_checkpoint.go @@ -11,6 +11,7 @@ import ( // Close flushes all dirty pages to physical disk and releases file handles. func (e *Engine) Close() error { + e.stopCommitter() // drain queued writes before tearing down files e.mu.Lock() defer e.mu.Unlock() @@ -39,6 +40,13 @@ func (e *Engine) Checkpoint() error { e.mu.Lock() defer e.mu.Unlock() + // Writes are applied in memory only inside a commit group that fsyncs before + // releasing e.mu, so nothing un-synced can exist here; sync anyway so data + // pages are never flushed ahead of their WAL records. + if err := e.wal.SyncIfEnabled(); err != nil { + return fmt.Errorf("checkpoint sync wal: %w", err) + } + if err := e.updateMeta(func(m *storage.MetaData) { m.LastCheckpointLSN = e.wal.CurrentLSN() m.ActiveDataPageID = e.activeDataPage diff --git a/internal/api/engine_commit.go b/internal/api/engine_commit.go new file mode 100644 index 0000000..77b5c55 --- /dev/null +++ b/internal/api/engine_commit.go @@ -0,0 +1,154 @@ +package api + +import ( + "errors" + "sync" +) + +// ErrEngineClosed is returned for writes submitted after Close. +var ErrEngineClosed = errors.New("engine: closed") + +// pendingNotify is a commit event held back until its WAL record is fsynced. +type pendingNotify struct { + lsn uint64 + opType byte + key []byte + value []byte +} + +type writeReq struct { + run func() error // executes under e.mu: check, WAL append (no fsync), apply + err error + done chan struct{} + ok bool // run appended a WAL record (so it depends on the batch fsync) +} + +// committer is the group-commit queue. A single goroutine drains it: each pass +// takes every queued write, runs them back to back under the exclusive engine +// lock (appending to the WAL without fsync), issues ONE fsync for the whole +// group, notifies the replication observer in LSN order, and only then +// releases the lock and acknowledges the writers. Because the lock is held +// across the fsync, readers and the observer never see a write before it is +// durable, exactly as with fsync-per-write; the gain is that N queued writers +// pay for one fsync instead of N. +type committer struct { + mu sync.Mutex + queue []*writeReq + started bool + closed bool + wake chan struct{} + done chan struct{} +} + +// commit runs fn through the group-commit pipeline and returns once its WAL +// record (if it wrote one) is durable. fn runs with e.mu held exclusively and +// must record observer events via e.queueNotify rather than calling the +// observer directly. +func (e *Engine) commit(fn func() error) error { + c := &e.cq + req := &writeReq{run: fn, done: make(chan struct{})} + + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return ErrEngineClosed + } + if !c.started { + c.started = true + c.wake = make(chan struct{}, 1) + c.done = make(chan struct{}) + go e.commitLoop() + } + c.queue = append(c.queue, req) + c.mu.Unlock() + + select { + case c.wake <- struct{}{}: + default: + } + <-req.done + return req.err +} + +func (e *Engine) commitLoop() { + c := &e.cq + defer close(c.done) + for { + <-c.wake + for { + c.mu.Lock() + batch := c.queue + c.queue = nil + closed := c.closed + c.mu.Unlock() + if len(batch) == 0 { + if closed { + return + } + break + } + e.runBatch(batch) + } + } +} + +// stopCommitter drains outstanding writes and stops the commit goroutine. +func (e *Engine) stopCommitter() { + c := &e.cq + c.mu.Lock() + c.closed = true + started := c.started + c.mu.Unlock() + if started { + select { + case c.wake <- struct{}{}: + default: + } + <-c.done + } +} + +func (e *Engine) runBatch(batch []*writeReq) { + e.mu.Lock() + e.pending = e.pending[:0] + wrote := false + for _, r := range batch { + before := e.wal.CurrentLSN() + r.err = r.run() + // A request that appended a WAL record must wait for the group fsync, + // even if applying it then failed (the record is still in the log). + r.ok = e.wal.CurrentLSN() != before + wrote = wrote || r.ok + } + + if wrote { + if syncErr := e.wal.SyncIfEnabled(); syncErr != nil { + // The records may or may not be on disk and memory already reflects + // them: fail-stop. Nothing in this group is acknowledged or replicated. + herr := e.halt(syncErr) + for _, r := range batch { + if r.ok { + r.err = herr + } + } + e.pending = e.pending[:0] + } + } + if e.observer != nil { + for _, n := range e.pending { + e.observer.OnCommit(n.lsn, n.opType, n.key, n.value) + } + } + e.pending = e.pending[:0] + e.mu.Unlock() + + for _, r := range batch { + close(r.done) + } +} + +// queueNotify records a commit event to be delivered after the group fsync. +// Caller holds e.mu. +func (e *Engine) queueNotify(lsn uint64, opType byte, key, value []byte) { + e.pending = append(e.pending, pendingNotify{lsn: lsn, opType: opType, key: key, value: value}) +} diff --git a/internal/api/engine_conditional.go b/internal/api/engine_conditional.go index 51e0445..3e97f2c 100644 --- a/internal/api/engine_conditional.go +++ b/internal/api/engine_conditional.go @@ -78,15 +78,15 @@ func (e *Engine) PutIf(key uint64, value []byte, cond Condition) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - if e.failed != nil { - return e.failed - } - if err := e.checkConditionLocked(key, cond); err != nil { - return err - } - return e.putLocked(key, value) + return e.commit(func() error { + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.putLocked(key, value) + }) } // DeleteIf removes key only if cond holds, atomically with the check. @@ -94,13 +94,13 @@ func (e *Engine) DeleteIf(key uint64, cond Condition) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - e.mu.Lock() - defer e.mu.Unlock() - if e.failed != nil { - return e.failed - } - if err := e.checkConditionLocked(key, cond); err != nil { - return err - } - return e.deleteLocked(key) + return e.commit(func() error { + if e.failed != nil { + return e.failed + } + if err := e.checkConditionLocked(key, cond); err != nil { + return err + } + return e.deleteLocked(key) + }) } diff --git a/internal/api/engine_delete.go b/internal/api/engine_delete.go index 93c08aa..ff529cb 100644 --- a/internal/api/engine_delete.go +++ b/internal/api/engine_delete.go @@ -11,9 +11,7 @@ func (e *Engine) Delete(key uint64) error { if e.IsReadOnly() { return ErrReadOnlyReplica } - e.mu.Lock() - defer e.mu.Unlock() - return e.deleteLocked(key) + return e.commit(func() error { return e.deleteLocked(key) }) } func (e *Engine) deleteLocked(key uint64) error { @@ -22,7 +20,7 @@ func (e *Engine) deleteLocked(key uint64) error { } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) - lsn, err := e.wal.Write(wal.OpDelete, keyBytes[:], nil) + lsn, err := e.wal.Append(wal.OpDelete, keyBytes[:], nil) if err != nil { return e.halt(err) } @@ -32,6 +30,6 @@ func (e *Engine) deleteLocked(key uint64) error { } e.unindexValueLocked(key) - e.notifyCommit(lsn, wal.OpDelete, keyBytes[:], nil) + e.queueNotify(lsn, wal.OpDelete, keyBytes[:], nil) return nil } diff --git a/internal/api/engine_groupcommit_test.go b/internal/api/engine_groupcommit_test.go new file mode 100644 index 0000000..305b965 --- /dev/null +++ b/internal/api/engine_groupcommit_test.go @@ -0,0 +1,161 @@ +package api + +import ( + "fmt" + "path/filepath" + "sync" + "testing" +) + +type recObs struct { + mu sync.Mutex + lsns []uint64 + eng *Engine +} + +func (o *recObs) OnCommit(lsn uint64, _ byte, _, _ []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // The observer must only ever see records that are already durable. + if o.eng.wal.DurableLSN() < lsn { + panic(fmt.Sprintf("observer saw LSN %d before durable (%d)", lsn, o.eng.wal.DurableLSN())) + } + o.lsns = append(o.lsns, lsn) +} + +// Many concurrent writers of mixed kinds; observer sees strictly increasing, +// durable LSNs; after an abrupt crash (no Close/flush) every acknowledged +// write is recovered. +func TestGroupCommitConcurrentWritersCrashRecovery(t *testing.T) { + dir := t.TempDir() + dbPath, walPath := filepath.Join(dir, "g.bin"), filepath.Join(dir, "g.wal") + e, err := NewEngine(dbPath, walPath, 256) + if err != nil { + t.Fatal(err) + } + obs := &recObs{eng: e} + e.SetCommitObserver(obs) + + const writers, per = 16, 40 + var wg sync.WaitGroup + for g := 0; g < writers; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < per; i++ { + k := uint64(g*1000 + i + 1) + val := []byte(fmt.Sprintf("v-%d", k)) + var err error + switch i % 4 { + case 0: + err = e.Put(k, val) + case 1: + err = e.PutIfAbsent(k, val) + case 2: + err = e.BatchWrite([]BatchOperation{{Type: BatchOpPut, Key: k, Value: val}}) + case 3: + err = e.PutIf(k, val, Condition{Kind: CondAbsent}) + } + if err != nil { + t.Errorf("write %d: %v", k, err) + } + } + }(g) + } + wg.Wait() + + obs.mu.Lock() + if len(obs.lsns) != writers*per { + t.Fatalf("observer got %d events, want %d", len(obs.lsns), writers*per) + } + for i := 1; i < len(obs.lsns); i++ { + if obs.lsns[i] != obs.lsns[i-1]+1 { + t.Fatalf("observer order broken at %d: %d then %d", i, obs.lsns[i-1], obs.lsns[i]) + } + } + obs.mu.Unlock() + + // Simulate crash: stop the committer, drop files without flushing pages. + e.stopCommitter() + _ = e.wal.Close() + _ = e.disk.Close() + + r, err := NewEngine(dbPath, walPath, 256) + if err != nil { + t.Fatal(err) + } + defer r.Close() + for g := 0; g < writers; g++ { + for i := 0; i < per; i++ { + k := uint64(g*1000 + i + 1) + v, ok, err := r.Get(k) + if err != nil || !ok || string(v) != fmt.Sprintf("v-%d", k) { + t.Fatalf("key %d lost after crash: ok=%v err=%v v=%q", k, ok, err, v) + } + } + } +} + +// Conditional writes stay atomic under contention: exactly one PutIfAbsent wins. +func TestGroupCommitConditionalAtomicity(t *testing.T) { + dir := t.TempDir() + e, err := NewEngine(filepath.Join(dir, "a.bin"), filepath.Join(dir, "a.wal"), 64) + if err != nil { + t.Fatal(err) + } + defer e.Close() + var wg sync.WaitGroup + var mu sync.Mutex + wins := 0 + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if e.PutIfAbsent(7, []byte("x")) == nil { + mu.Lock() + wins++ + mu.Unlock() + } + }() + } + wg.Wait() + if wins != 1 { + t.Fatalf("wins=%d", wins) + } +} + +func TestWriteAfterCloseFails(t *testing.T) { + dir := t.TempDir() + e, _ := NewEngine(filepath.Join(dir, "c.bin"), filepath.Join(dir, "c.wal"), 64) + _ = e.Put(1, []byte("a")) + _ = e.Close() + if err := e.Put(2, []byte("b")); err != ErrEngineClosed { + t.Fatalf("got %v", err) + } +} + +func BenchmarkEngineConcurrentWrite(b *testing.B) { + dir := b.TempDir() + e, err := NewEngine(filepath.Join(dir, "b.bin"), filepath.Join(dir, "b.wal"), 1024) + if err != nil { + b.Fatal(err) + } + defer e.Close() + payload := []byte(`{"name":"benchmark-record","val":12345}`) + var mu sync.Mutex + var next uint64 + b.SetParallelism(8) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + mu.Lock() + next++ + k := next + mu.Unlock() + if err := e.Put(k, payload); err != nil { + b.Error(err) + return + } + } + }) +} diff --git a/internal/api/engine_ops.go b/internal/api/engine_ops.go index a164e58..5d457ea 100644 --- a/internal/api/engine_ops.go +++ b/internal/api/engine_ops.go @@ -24,7 +24,7 @@ var ( ErrReadOnlyReplica = errors.New("writes are not permitted on read-only replica") ) -// Put logs mutation to WAL, stores tuple in slotted page, and indexes key in B+ Tree. +// Put logs mutation (group-committed: returns once the WAL record is fsynced) to WAL, stores tuple in slotted page, and indexes key in B+ Tree. func (e *Engine) Put(key uint64, value []byte) error { if e.IsReadOnly() { return ErrReadOnlyReplica @@ -32,9 +32,7 @@ func (e *Engine) Put(key uint64, value []byte) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - return e.putLocked(key, value) + return e.commit(func() error { return e.putLocked(key, value) }) } // PutIfAbsent inserts only when key is not already present. @@ -45,16 +43,16 @@ func (e *Engine) PutIfAbsent(key uint64, value []byte) error { if len(value) > MaxValueSize { return ErrRecordTooLarge } - e.mu.Lock() - defer e.mu.Unlock() - _, err := e.tree.Find(key) - if err == nil { - return ErrRecordExists - } - if err != index.ErrKeyNotFound { - return err - } - return e.putLocked(key, value) + return e.commit(func() error { + _, err := e.tree.Find(key) + if err == nil { + return ErrRecordExists + } + if err != index.ErrKeyNotFound { + return err + } + return e.putLocked(key, value) + }) } func (e *Engine) putLocked(key uint64, value []byte) error { @@ -63,7 +61,7 @@ func (e *Engine) putLocked(key uint64, value []byte) error { } var keyBytes [8]byte binary.LittleEndian.PutUint64(keyBytes[:], key) - lsn, err := e.wal.Write(wal.OpPut, keyBytes[:], value) + lsn, err := e.wal.Append(wal.OpPut, keyBytes[:], value) if err != nil { return e.halt(err) } @@ -72,7 +70,7 @@ func (e *Engine) putLocked(key uint64, value []byte) error { } e.indexValueLocked(key, value) - e.notifyCommit(lsn, wal.OpPut, keyBytes[:], value) + e.queueNotify(lsn, wal.OpPut, keyBytes[:], value) return nil } diff --git a/internal/wal/group_commit_test.go b/internal/wal/group_commit_test.go new file mode 100644 index 0000000..b5724ba --- /dev/null +++ b/internal/wal/group_commit_test.go @@ -0,0 +1,111 @@ +package wal + +import ( + "path/filepath" + "sync" + "testing" +) + +// Concurrent Write calls must all be durable on return, get unique monotonic +// LSNs, and replay completely after reopen. +func TestWALGroupCommitConcurrentWriters(t *testing.T) { + path := filepath.Join(t.TempDir(), "gc.wal") + w, err := OpenWAL(path, true) + if err != nil { + t.Fatal(err) + } + const writers, per = 32, 25 + var wg sync.WaitGroup + lsns := make(chan uint64, writers*per) + for g := 0; g < writers; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < per; i++ { + lsn, err := w.Write(OpPut, []byte("k"), []byte("v")) + if err != nil { + t.Error(err) + return + } + if w.DurableLSN() < lsn { + t.Errorf("acknowledged LSN %d not durable (durable=%d)", lsn, w.DurableLSN()) + } + lsns <- lsn + } + }() + } + wg.Wait() + close(lsns) + seen := map[uint64]bool{} + for l := range lsns { + if seen[l] { + t.Fatalf("duplicate LSN %d", l) + } + seen[l] = true + } + if len(seen) != writers*per { + t.Fatalf("got %d LSNs", len(seen)) + } + _ = w.Close() + + var last uint64 + rep, err := Replay(path, func(r *Record) error { + if r.LSN != last+1 { + t.Fatalf("non-monotonic replay: %d after %d", r.LSN, last) + } + last = r.LSN + return nil + }) + if err != nil || rep.TotalRecords != writers*per { + t.Fatalf("replay: %v total=%d", err, rep.TotalRecords) + } +} + +func TestWALAppendNotDurableUntilWait(t *testing.T) { + w, err := OpenWAL(filepath.Join(t.TempDir(), "a.wal"), true) + if err != nil { + t.Fatal(err) + } + defer w.Close() + lsn, _ := w.Append(OpPut, []byte("k"), []byte("v")) + if w.DurableLSN() >= lsn { + t.Fatal("Append must not mark record durable") + } + if err := w.WaitDurable(lsn); err != nil || w.DurableLSN() < lsn { + t.Fatalf("WaitDurable: %v durable=%d", err, w.DurableLSN()) + } +} + +func TestWALCheckpointMarksDurableAndLSNContinues(t *testing.T) { + w, _ := OpenWAL(filepath.Join(t.TempDir(), "c.wal"), true) + defer w.Close() + l1, _ := w.Append(OpPut, []byte("k"), []byte("v")) + if err := w.Checkpoint(); err != nil { + t.Fatal(err) + } + if w.DurableLSN() < l1 { + t.Fatal("checkpoint must cover appended records") + } + l2, _ := w.Write(OpPut, []byte("k"), []byte("v")) + if l2 != l1+1 { + t.Fatalf("LSN after checkpoint: %d", l2) + } +} + +func BenchmarkWALConcurrentSyncedWrite(b *testing.B) { + w, err := OpenWAL(filepath.Join(b.TempDir(), "b.wal"), true) + if err != nil { + b.Fatal(err) + } + defer w.Close() + b.SetParallelism(8) + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := w.Write(OpPut, []byte("key"), []byte("value")); err != nil { + b.Error(err) + return + } + } + }) +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 2d038cc..42cc68f 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -8,12 +8,22 @@ import ( ) // WAL manages the sequential append-only write-ahead log file. +// +// Durability is group-committed: Append assigns an LSN and writes the frame +// under mu (cheap), while the fsync is performed by whichever caller reaches +// syncMu first and covers every record appended up to that moment, so +// concurrent writers share one fsync. Lock order is always syncMu, then mu. type WAL struct { mu sync.Mutex file *os.File path string currentLSN uint64 syncOnWrite bool + + syncMu sync.Mutex // serializes fsyncs; guards syncedLSN + syncedLSN uint64 // highest LSN known to be fsynced + errMu sync.Mutex // guards syncErr (read by Append without syncMu) + syncErr error // sticky: after a failed fsync the log is untrustworthy } // OpenWAL opens an existing WAL file or creates a new one at the specified path. @@ -49,17 +59,22 @@ func OpenWAL(path string, syncOnWrite bool) (*WAL, error) { } w.currentLSN = lastLSN + w.syncedLSN = lastLSN // everything already on disk at open is treated as durable return w, nil } -// Write appends a state mutation record to the WAL, returning the assigned LSN. -func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { +// Append writes a record to the log file and assigns its LSN but does NOT +// fsync. The record is not durable until WaitDurable(lsn) or Sync returns. +func (w *WAL) Append(opType byte, key, value []byte) (uint64, error) { w.mu.Lock() defer w.mu.Unlock() - w.currentLSN++ + if err := w.loadSyncErr(); err != nil { + return 0, err + } + rec := &Record{ - LSN: w.currentLSN, + LSN: w.currentLSN + 1, Type: opType, Key: key, Value: value, @@ -67,27 +82,98 @@ func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { encoded := EncodeRecord(rec) if len(encoded) > MaxFrameSize { - w.currentLSN-- return 0, ErrFrameTooLarge } if _, err := w.file.Write(encoded); err != nil { return 0, err } + w.currentLSN = rec.LSN + return rec.LSN, nil +} +// Write appends a state mutation record to the WAL, returning the assigned LSN. +// When syncOnWrite is set it returns only after the record is fsynced; concurrent +// callers share fsyncs (group commit). +func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { + lsn, err := w.Append(opType, key, value) + if err != nil { + return 0, err + } if w.syncOnWrite { - if err := w.file.Sync(); err != nil { + if err := w.WaitDurable(lsn); err != nil { return 0, err } } + return lsn, nil +} + +// SyncOnWrite reports whether this log was opened with fsync-per-commit semantics. +func (w *WAL) SyncOnWrite() bool { return w.syncOnWrite } + +func (w *WAL) loadSyncErr() error { + w.errMu.Lock() + defer w.errMu.Unlock() + return w.syncErr +} + +// WaitDurable blocks until the record with the given LSN has been fsynced. +// If another caller's fsync already covered it, no further fsync is issued. +func (w *WAL) WaitDurable(lsn uint64) error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + if w.syncedLSN >= lsn { + return nil + } + return w.syncLocked() +} + +// syncLocked fsyncs everything appended so far. Caller holds syncMu. +func (w *WAL) syncLocked() error { + w.mu.Lock() + target := w.currentLSN + f := w.file + w.mu.Unlock() - return w.currentLSN, nil + // Appends may proceed while fsync runs; they simply miss this sync and are + // covered by the next one. target only claims what was written before it. + if err := f.Sync(); err != nil { + w.errMu.Lock() + w.syncErr = err + w.errMu.Unlock() + return err + } + if target > w.syncedLSN { + w.syncedLSN = target + } + return nil } // Sync forces all buffered kernel writes to physical non-volatile storage. func (w *WAL) Sync() error { - w.mu.Lock() - defer w.mu.Unlock() - return w.file.Sync() + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + return w.syncLocked() +} + +// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. +func (w *WAL) SyncIfEnabled() error { + if !w.syncOnWrite { + return nil + } + return w.Sync() +} + +// DurableLSN returns the highest LSN known to be fsynced. +func (w *WAL) DurableLSN() uint64 { + w.syncMu.Lock() + defer w.syncMu.Unlock() + return w.syncedLSN } // CurrentLSN returns the latest assigned Log Sequence Number. @@ -115,6 +201,8 @@ func (w *WAL) Path() string { // Close flushes buffered writes and closes the log file. func (w *WAL) Close() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() w.mu.Lock() defer w.mu.Unlock() _ = w.file.Sync() @@ -126,12 +214,16 @@ func (w *WAL) Close() error { // The caller MUST have flushed all dirty buffer pool pages to disk first so the // archived WAL records are no longer needed for recovery. func (w *WAL) Checkpoint() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() w.mu.Lock() defer w.mu.Unlock() if err := w.file.Sync(); err != nil { return err } + // Everything appended so far is now durable (and about to be archived). + w.syncedLSN = w.currentLSN if err := w.file.Close(); err != nil { return err } From 65051a7880134a7644c7a4dd011805fd624443cd Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:36:15 +0530 Subject: [PATCH 05/10] fix(storage): enforce write-ahead barrier before dirty page writes Group commit applies writes to pages before the group's fsync, so the buffer pool could flush a page ahead of its WAL records. Add a WAL barrier (WAL.EnsureDurable) called before eviction, FlushPage and FlushAll; a failing barrier blocks the page write. Adds regression tests and the upgrade spec. Co-Authored-By: Claude Sonnet 5.5 --- docs/upgrade-spec.md | 30 ++++++ internal/api/engine.go | 4 + internal/storage/buffer_pool.go | 28 +++++- internal/storage/buffer_pool_eviction.go | 2 +- internal/storage/wal_barrier_test.go | 112 +++++++++++++++++++++++ internal/wal/wal.go | 12 +++ 6 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 docs/upgrade-spec.md create mode 100644 internal/storage/wal_barrier_test.go diff --git a/docs/upgrade-spec.md b/docs/upgrade-spec.md new file mode 100644 index 0000000..ea293a1 --- /dev/null +++ b/docs/upgrade-spec.md @@ -0,0 +1,30 @@ +# BeastDB upgrade spec (security, usability, performance) + +Source: review of 2026-11-12. Complements [ROADMAP.md](../ROADMAP.md) and [v1-semantics.md](v1-semantics.md). Claims here are goals, not shipped features. + +## Findings that drive the work + +| Area | Finding | Evidence | +|---|---|---| +| Perf | WAL fsyncs inside its mutex on every write; ~4 ms/write | `internal/wal/wal.go` Write | +| Perf | One global `sync.RWMutex`; scans hold it for the cursor lifetime | `internal/api/engine_ops.go` Scan | +| Perf | Benchmarks are in-process, 1 MB pool, 2k records; no competitor comparison | `docs/benchmarks.md` | +| Sec | gRPC auth checks token validity only; roles ignored | `internal/api/grpc_auth.go` | +| Sec | No encryption at rest (data, WAL, backups) | repo-wide search | +| Sec | Static token has no expiry/rotation | `grpc_auth.go` | +| UX | uint64 keys only, no namespaces, one Python example, no /metrics | proto, `examples/` | + +## Work packages (parallel, disjoint files) + +1. **WP1 WAL group commit** (`internal/wal`, write path in `internal/api`): batch concurrent writers into one fsync, same durability. Target: >=10x concurrent write throughput. Must keep crash/fault tests green. +2. **WP2 gRPC role authorization** (`internal/api/grpc_auth*.go`, token store): per-method roles (viewer = Get/Scan; admin/writer = mutations; replication role for StreamWAL/Ack), token expiry. Deny by default. +3. **WP3 YCSB-style harness** (new `cmd/beastbench`): workloads A-F over gRPC, reports throughput and p50/p95/p99, reproducible flags, doc update. +4. **WP4 Prometheus /metrics** (`internal/web/handler`): request/error/latency, WAL size and sync latency, dirty pages, checkpoint age, LSN. + +## Next wave (not started) + +Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log. + +## Studio dashboard design + +Figma design for a Firestore/Supabase-style console: project overview, collections/table editor, SQL-free query bar, API keys and roles, metrics, backups. Aimed at non-experts. Tracked in Figma; implementation in `studio/`. diff --git a/internal/api/engine.go b/internal/api/engine.go index 07f454c..b3bc26a 100644 --- a/internal/api/engine.go +++ b/internal/api/engine.go @@ -41,6 +41,10 @@ func NewEngine(dbPath, walPath string, poolSize int) (*Engine, error) { _ = disk.Close() return nil, err } + // Write-ahead barrier: no dirty data page may reach the data file before + // the WAL records that describe it are durable (group commit applies pages + // before the group's fsync). + bpm.SetWALBarrier(w.EnsureDurable) if disk.NumPages() == 0 { return initFreshV1(disk, bpm, w) diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index 19d4a96..53e4a67 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -22,6 +22,30 @@ type BufferPoolManager struct { clockHand int mu sync.RWMutex pagePool sync.Pool // Fix #6: reusable 4KB page buffers + + // walBarrier, when set, is called before any dirty page is written to the + // data file. It must return only once every WAL record appended so far is + // durable, which preserves the write-ahead invariant even though writes are + // applied to pages before their group's fsync completes. + walBarrier func() error +} + +// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. +func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { + bpm.mu.Lock() + defer bpm.mu.Unlock() + bpm.walBarrier = fn +} + +// writeBack writes one dirty frame to the data file after the WAL barrier. +// Caller must hold bpm.mu (write lock). +func (bpm *BufferPoolManager) writeBack(f *Frame) error { + if bpm.walBarrier != nil { + if err := bpm.walBarrier(); err != nil { + return err + } + } + return bpm.disk.WritePage(f.PageID, f.Page.Data()) } // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. @@ -127,7 +151,7 @@ func (bpm *BufferPoolManager) FlushPage(pageID uint64) error { frame := bpm.frames[idx] if frame.Page != nil && frame.IsDirty { - if err := bpm.disk.WritePage(frame.PageID, frame.Page.Data()); err != nil { + if err := bpm.writeBack(frame); err != nil { return err } frame.IsDirty = false @@ -142,7 +166,7 @@ func (bpm *BufferPoolManager) FlushAll() error { for _, frame := range bpm.frames { if frame.Page != nil && frame.IsDirty { - if err := bpm.disk.WritePage(frame.PageID, frame.Page.Data()); err != nil { + if err := bpm.writeBack(frame); err != nil { return err } frame.IsDirty = false diff --git a/internal/storage/buffer_pool_eviction.go b/internal/storage/buffer_pool_eviction.go index 2c2cb6e..8695808 100644 --- a/internal/storage/buffer_pool_eviction.go +++ b/internal/storage/buffer_pool_eviction.go @@ -45,7 +45,7 @@ func (bpm *BufferPoolManager) evictFrame(idx int) error { return nil } if victim.IsDirty { - if err := bpm.disk.WritePage(victim.PageID, victim.Page.Data()); err != nil { + if err := bpm.writeBack(victim); err != nil { return err } victim.IsDirty = false diff --git a/internal/storage/wal_barrier_test.go b/internal/storage/wal_barrier_test.go new file mode 100644 index 0000000..1e03faa --- /dev/null +++ b/internal/storage/wal_barrier_test.go @@ -0,0 +1,112 @@ +package storage + +import ( + "bytes" + "errors" + "path/filepath" + "testing" +) + +func newBarrierPool(t *testing.T, size int) (*BufferPoolManager, *DiskManager) { + t.Helper() + disk, err := OpenDiskManager(filepath.Join(t.TempDir(), "barrier.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = disk.Close() }) + return NewBufferPoolManager(disk, size), disk +} + +// A dirty page must not reach the data file when the WAL barrier fails: +// otherwise a page could be persisted ahead of the log records that describe it. +func TestWALBarrierFailureBlocksPageWrite(t *testing.T) { + bpm, disk := newBarrierPool(t, 1) + errBarrier := errors.New("wal not durable") + bpm.SetWALBarrier(func() error { return errBarrier }) + + page, id, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + copy(page.Data()[100:], []byte("uncommitted")) + if err := bpm.UnpinPage(id, true); err != nil { + t.Fatal(err) + } + + // Eviction (pool of one) must fail because the barrier fails. + if _, _, err := bpm.NewPage(); !errors.Is(err, errBarrier) { + t.Fatalf("eviction err = %v, want barrier error", err) + } + if err := bpm.FlushPage(id); !errors.Is(err, errBarrier) { + t.Fatalf("FlushPage err = %v, want barrier error", err) + } + if err := bpm.FlushAll(); !errors.Is(err, errBarrier) { + t.Fatalf("FlushAll err = %v, want barrier error", err) + } + + onDisk := make([]byte, PageSize) + if err := disk.ReadPage(id, onDisk); err != nil { + t.Fatal(err) + } + if bytes.Contains(onDisk, []byte("uncommitted")) { + t.Fatal("dirty page reached the data file although the WAL barrier failed") + } +} + +// The barrier runs before every page write path, and pages are written after it. +func TestWALBarrierRunsBeforeEveryPageWrite(t *testing.T) { + bpm, disk := newBarrierPool(t, 1) + calls := 0 + bpm.SetWALBarrier(func() error { calls++; return nil }) + + page, id, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + copy(page.Data()[100:], []byte("committed")) + if err := bpm.UnpinPage(id, true); err != nil { + t.Fatal(err) + } + + // Evict via a second page (pool size 1). + _, id2, err := bpm.NewPage() + if err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("barrier calls after eviction = %d, want 1", calls) + } + onDisk := make([]byte, PageSize) + if err := disk.ReadPage(id, onDisk); err != nil { + t.Fatal(err) + } + if !bytes.Contains(onDisk, []byte("committed")) { + t.Fatal("evicted page was not written after the barrier succeeded") + } + + if err := bpm.UnpinPage(id2, true); err != nil { + t.Fatal(err) + } + if err := bpm.FlushPage(id2); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("barrier calls after FlushPage = %d, want 2", calls) + } + + // Dirty again, then FlushAll. + p, err := bpm.FetchPage(id2) + if err != nil { + t.Fatal(err) + } + copy(p.Data()[200:], []byte("more")) + if err := bpm.UnpinPage(id2, true); err != nil { + t.Fatal(err) + } + if err := bpm.FlushAll(); err != nil { + t.Fatal(err) + } + if calls != 3 { + t.Fatalf("barrier calls after FlushAll = %d, want 3", calls) + } +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 42cc68f..7261209 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -169,6 +169,18 @@ func (w *WAL) SyncIfEnabled() error { return w.Sync() } +// EnsureDurable returns once every record appended so far is fsynced. It is a +// no-op when the log was not opened with syncOnWrite, and costs no fsync when +// a previous sync already covers the current LSN. The buffer pool calls it +// before writing a dirty page so a data page never reaches disk ahead of the +// WAL records that describe it. +func (w *WAL) EnsureDurable() error { + if !w.syncOnWrite { + return nil + } + return w.WaitDurable(w.CurrentLSN()) +} + // DurableLSN returns the highest LSN known to be fsynced. func (w *WAL) DurableLSN() uint64 { w.syncMu.Lock() From cf7ed6c80dbcafaa3ec451d7113bc42fd037266b Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:37:45 +0530 Subject: [PATCH 06/10] docs: document group commit, WAL barrier, gRPC roles, metrics, beastbench and Studio design Co-Authored-By: Claude Sonnet 5.5 --- README.md | 2 ++ ROADMAP.md | 2 +- docs/architecture.md | 4 ++-- docs/benchmarks.md | 6 ++++++ docs/runbooks/operator-guide.md | 4 ++++ docs/studio.md | 24 ++++++++++++++++++++++++ docs/upgrade-spec.md | 24 +++++++++++++++++------- docs/v1-semantics.md | 25 +++++++++++++++++++++++-- 8 files changed, 79 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2a431de..169640d 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,8 @@ BeastDB is optimized for **read-heavy OLTP** โ€” deterministic point reads and o - [**V1 Semantics & Guarantees**](docs/v1-semantics.md) โ€” Durability acknowledgment, limits, and replica reads. - [**System Benchmarks**](docs/benchmarks.md) โ€” Reproducible throughput, tail latencies, and cache-eviction profiles. +- [**Benchmarking Guide**](docs/benchmarking.md) โ€” `beastbench` YCSB-style workloads and how to compare fairly with other databases. +- [**Upgrade Spec**](docs/upgrade-spec.md) โ€” Security, usability and performance work packages and what is still planned. - [**Operator Guide**](docs/runbooks/operator-guide.md) โ€” TLS configuration, cluster roles, manual failover, and `/healthz` probes. - [**Disaster Recovery**](docs/runbooks/disaster-recovery.md) โ€” Physical backup drills, checksum verification, and RTO/RPO SLA. - [**Client Integration**](docs/clients.md) โ€” Protobuf contracts for Go, Python, TypeScript, and Rust. diff --git a/ROADMAP.md b/ROADMAP.md index 15ddabb..3d88fe0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -122,7 +122,7 @@ Order is dependency-based, not a calendar promise. Initial planning range for on ## Beyond V1 gates -See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite), limit-based scan pagination, and values up to 2 MiB with 8 MiB batches ([ADR 0001](docs/adr/0001-large-values.md)). +See the [roadmap plan](docs/roadmap-plan.md) for post-gate phases and the [upgrade spec](docs/upgrade-spec.md) for the current security, usability and performance work. Shipped so far: conditional writes (`ABSENT`/`PRESENT`/`VALUE_EQUALS` on Put, Delete and BatchWrite), limit-based scan pagination, values up to 2 MiB with 8 MiB batches ([ADR 0001](docs/adr/0001-large-values.md)), WAL group commit with a write-ahead buffer-pool barrier, per-method gRPC role authorization with token expiry, a Prometheus `/metrics` endpoint, and the `beastbench` YCSB-style load generator ([benchmarking guide](docs/benchmarking.md)). ## References diff --git a/docs/architecture.md b/docs/architecture.md index 59dceb7..96d531e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ A modular, zero-allocation database architecture built from first principles in 3. The RESP parser processes byte streams in-place into command frames without heap allocations. ### Write Execution Flow -1. **WAL Append:** Before modifying any page in memory, a log record is written to the WAL and synced (`fdatasync`). +1. **Group commit:** Writers enqueue their operation. One committer goroutine drains the queue, takes the engine lock exclusively, and for each queued write checks its condition, appends the record to the WAL (no `fsync` yet) and applies it in memory. It then issues **one** `fsync` for the whole group, notifies replication in LSN order, releases the lock, and only then acknowledges the writers. The lock is held across the `fsync` so readers and replicas never see a write before it is durable. 2. **Buffer Pool Pin:** The targeted 4KB disk page is fetched into a memory frame and pinned. 3. **Slotted Page Update:** The tuple is inserted into the slotted page, updating the slot array and free space pointer. 4. **Index Update:** If indexed, the B+ tree is traversed; keys and record pointers are updated, splitting nodes if page capacity is exceeded. @@ -66,6 +66,6 @@ A modular, zero-allocation database architecture built from first principles in --- ## 3. Storage Invariants -- **Write-Ahead Invariant:** Log records must hit disk before dirty database pages containing those modifications are written. +- **Write-Ahead Invariant:** Log records must hit disk before dirty database pages containing those modifications are written. Because group commit applies a write to its page before the group's `fsync`, the buffer pool enforces this with a **WAL barrier** (`WAL.EnsureDurable`) called before every dirty-page write (eviction, `FlushPage`, `FlushAll`). If the barrier fails, the page is not written. The barrier costs no extra `fsync` when the WAL is already durable up to the current LSN. - **Page Isolation:** Every 4KB page maintains its own self-describing header, slot array, and CRC checksum. - **Pin Safety:** A page frame with `pin_count > 0` cannot be evicted from the buffer pool under any circumstances. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 711814e..9e520bc 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -51,6 +51,12 @@ Measured on 12th Gen Intel(R) Core(TM) i5-12450HX: --- +### Concurrent writes and group commit + +Writers now share `fsync`s (group commit, see [architecture](architecture.md)). On one Apple-silicon laptop the engine benchmark `BenchmarkEngineConcurrentWrite` (96 concurrent writer goroutines) measured about 7.8 ms/op before and 0.5 ms/op after; `BenchmarkEnginePointWrite` (one sequential writer) stays `fsync`-bound at about 9 ms/op. These are single-machine, in-process numbers and not a comparison with other databases. For end-to-end gRPC workloads use `beastbench` and follow the [benchmarking guide](benchmarking.md). + +--- + ## 4. Crash Recovery & Disaster Recovery RTO Measurements taken using `cmd/beastctl/backup_drill_test.go`: diff --git a/docs/runbooks/operator-guide.md b/docs/runbooks/operator-guide.md index 61da4e7..e80cff6 100644 --- a/docs/runbooks/operator-guide.md +++ b/docs/runbooks/operator-guide.md @@ -23,6 +23,10 @@ When bound to non-loopback interfaces or when `BEASTDB_ENV=production`, the serv ### Session Secret & API Tokens - Web sessions are authenticated via HMAC-SHA256 signatures backed by `BEASTDB_SESSION_SECRET` or persisted to `/session.key` (0600 file permissions). - gRPC requests require Bearer token authorization when `BEASTDB_API_TOKEN` is configured or API tokens exist in the token store. +- **Roles are enforced per gRPC method** (see [V1 Semantics ยง4](../v1-semantics.md)): `viewer` (read), `writer` (read and write), `admin` (everything), `replication` (replica streaming only). The static `--api-token` acts as `admin`. Give each application its own token with the smallest role it needs. +- **Followers must authenticate with a `replication` (or `admin`) token.** A viewer or writer token is rejected with `PERMISSION_DENIED` on `StreamWAL`/`Ack`. +- **Token expiry:** `POST /api/tokens` accepts an optional `expires_in_seconds` (0 or absent = never, maximum 10 years) and rejects an unknown `role` with HTTP 400. Studio does not yet expose an expiry field. +- **Upgrade note:** tokens created earlier default to `admin` and keep working. Tokens whose stored role is not one of the four roles now get `PERMISSION_DENIED` on every call; recreate them. --- diff --git a/docs/studio.md b/docs/studio.md index 22aaf8a..a4aa7de 100644 --- a/docs/studio.md +++ b/docs/studio.md @@ -86,6 +86,30 @@ Open `http://localhost:8088` in your browser and sign in with username `admin`. --- +## Design system and planned screens + +A redesign of Studio is specified in Figma (design only; `studio/` is unchanged). It targets people who are not database experts and follows the familiar Firebase/Supabase console layout: sidebar, collection list, records table, detail drawer. + +**Planned screens (10):** Table Editor, Overview, Query (visual builder that shows the equivalent `beastctl`/SDK call), Indexes, Replication, Backups, API Keys & Roles, Metrics, Logs & Audit, Settings. + +**Design tokens** (dark theme; Light mode is not designed): + +| Token | Value | Use | +|---|---|---| +| `accent/default` | `#A8F21A` (logo neon green; matches `brand.lime` in `tailwind.config.ts`) | primary actions, selection, lead chart line | +| `text/on-accent` | `#0A1002` | text on neon (contrast 14.1:1) | +| `bg/base`, `bg/panel`, `bg/raised` | `#0E120E`, `#151A15`, `#1C231B` | green-tinted dark surfaces | +| `border/default` | `#2A342A` | borders | +| `text/primary`, `text/muted` | `#F2F5F0`, `#93A08E` | text (16:1 and 6.4:1 on panel) | +| `status/success` | `#2DD4BF` (teal, so it is not confused with the accent) | healthy, active | +| `status/warning`, `status/danger`, `status/info` | `#FBBF24`, `#F87171`, `#60A5FA` | warnings, errors, info | + +Typography is Inter with Roboto Mono for code. Components: Button (Primary/Secondary/Danger), Status pill, Input, Nav item, Stat card, Toggle, Table row, Card. + +**Not yet backed by the engine:** encryption at rest, two-factor, collections/namespaces and the visual query builder, index suggestions, and a restore-to-new-database flow are shown in the design but are roadmap items (see [upgrade spec](upgrade-spec.md)). The mocks use placeholder data. + +--- + ## ๐Ÿ”ฅ Firebase-Style UI & Universal Capabilities - **3-Pane Collection Explorer**: Firebase Console style navigation: Left Partition Rail (with color-coded indicators) โ†’ Document List (with cover art/avatar badges) โ†’ Full-height Document Viewer. diff --git a/docs/upgrade-spec.md b/docs/upgrade-spec.md index ea293a1..2aace34 100644 --- a/docs/upgrade-spec.md +++ b/docs/upgrade-spec.md @@ -14,17 +14,27 @@ Source: review of 2026-11-12. Complements [ROADMAP.md](../ROADMAP.md) and [v1-se | Sec | Static token has no expiry/rotation | `grpc_auth.go` | | UX | uint64 keys only, no namespaces, one Python example, no /metrics | proto, `examples/` | -## Work packages (parallel, disjoint files) +## Work packages -1. **WP1 WAL group commit** (`internal/wal`, write path in `internal/api`): batch concurrent writers into one fsync, same durability. Target: >=10x concurrent write throughput. Must keep crash/fault tests green. -2. **WP2 gRPC role authorization** (`internal/api/grpc_auth*.go`, token store): per-method roles (viewer = Get/Scan; admin/writer = mutations; replication role for StreamWAL/Ack), token expiry. Deny by default. -3. **WP3 YCSB-style harness** (new `cmd/beastbench`): workloads A-F over gRPC, reports throughput and p50/p95/p99, reproducible flags, doc update. -4. **WP4 Prometheus /metrics** (`internal/web/handler`): request/error/latency, WAL size and sync latency, dirty pages, checkpoint age, LSN. +| WP | Scope | Status | +|---|---|---| +| WP1 | **WAL group commit** (`internal/wal`, `internal/api/engine_commit.go`): concurrent writers share one fsync with unchanged durability. Reported ~15x on a concurrent-write benchmark; a single sequential writer is unchanged. Includes a **WAL barrier** in the buffer pool so a data page never reaches disk ahead of its log record. | Merged | +| WP2 | **gRPC role authorization** (`internal/api/grpc_auth.go`, token store): deny-by-default per-method roles `viewer`/`writer`/`admin`/`replication`, optional token expiry. | Merged. No handler test yet for `expires_in_seconds`; Studio has no expiry field. | +| WP3 | **`beastbench`** (`cmd/beastbench`): YCSB-style workloads Aโ€“F over gRPC, latency histogram, JSON output, self-contained smoke mode, [benchmarking guide](benchmarking.md). | Merged. No competitor comparison has been run. | +| WP4 | **Prometheus `/metrics`**: request counts and latency, WAL size and LSN, checkpoint age, buffer pool, disk free. Never anonymous; accepts an admin session or a metrics/static bearer token. See the [operator guide](runbooks/operator-guide.md). | Merged | + +## Known limits of what shipped + +- Readers and replication are still blocked for the duration of each group's fsync (the engine keeps one global lock). Lifting this needs MVCC or latch coupling. +- The barrier can add an extra fsync when a dirty page is evicted under memory pressure. +- `LeaderServer.StreamWAL` replays the WAL file from disk and can briefly read a record that is appended but not yet fsynced. +- `TestEngineSystemWorkloadBenchmark` has a fixed 1000 ops/s floor and is timing-sensitive on a loaded machine. +- Checkpoint LSN/age metrics only appear after the first checkpoint since process start; the buffer pool hit ratio is cumulative. ## Next wave (not started) -Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log. +Encryption at rest, string keys + namespaces, MVCC/latch-coupling to remove the global lock, SDKs (Go/TS/Python), PITR via WAL archive, CDC feed, tamper-evident audit log, Studio expiry field and token-API handler tests. ## Studio dashboard design -Figma design for a Firestore/Supabase-style console: project overview, collections/table editor, SQL-free query bar, API keys and roles, metrics, backups. Aimed at non-experts. Tracked in Figma; implementation in `studio/`. +A Firestore/Supabase-style console for non-experts is designed in Figma; the implementation in `studio/` has not been changed. See [Studio design system](studio.md#design-system-and-planned-screens). diff --git a/docs/v1-semantics.md b/docs/v1-semantics.md index 366356c..b9615c7 100644 --- a/docs/v1-semantics.md +++ b/docs/v1-semantics.md @@ -17,7 +17,9 @@ ## 2. Operations & Execution Contracts ### `Put(key uint64, value []byte)` -* **Durability Guarantee:** Synchronous write-ahead log (`WAL`) persistence. A write is acknowledged only after `file.Sync()` (`fdatasync`) completes successfully on disk. +* **Durability Guarantee:** Synchronous write-ahead log (`WAL`) persistence. A write is acknowledged only after an `fsync` covering its WAL record completes successfully on disk. +* **Group commit:** Concurrent writers are queued and share one `fsync` per group. The durability guarantee is unchanged (no write is acknowledged before its record is durable, and readers and replication never observe a write before it is durable), but a group of N writers costs one `fsync` instead of N. A single sequential writer sees no speedup. +* **Failed fsync:** If the group `fsync` fails, the engine halts (fail-stop): every write in the group returns the error, later writes are rejected until restart, and the failure is sticky because retrying `fsync` after an error can report false success. * **Overwrite Semantics:** If `key` exists, the B+ Tree record pointer (`RID`) is atomically updated to the new slotted page location. * **Atomicity:** Single-key atomic mutation. @@ -61,7 +63,26 @@ --- -## 4. Supported Platform & Deployment Matrix +## 4. Authorization (gRPC) + +Authentication is a Bearer token: the static `--api-token` (treated as `admin`) or an API token from the token store. Authorization is **deny by default** per method: + +| Role | Allowed methods | +|---|---| +| `viewer` | `Get`, `Scan` | +| `writer` | `Get`, `Scan`, `Put`, `Delete`, `BatchWrite` | +| `admin` | all client methods and `ReplicationService` | +| `replication` | `ReplicationService.StreamWAL`, `ReplicationService.Ack` only | + +- A missing, malformed, unknown or expired token returns `UNAUTHENTICATED`. A valid token whose role may not call the method returns `PERMISSION_DENIED`. +- Any gRPC method not in the role map is denied for every role, including `admin` and the static token. Register new methods in `methodRoles` (`internal/api/grpc_auth.go`). +- A stored token with a role outside the four above can call nothing. +- Tokens may carry an optional expiry (`expires_at`); a token is rejected once the current time reaches it. Tokens created before expiry support never expire. +- With no static token and no token store, the interceptor is open (development only). + +--- + +## 5. Supported Platform & Deployment Matrix | Component | Supported Version / Target | Notes | |---|---|---| From 06b662e9bfda0245ec06f948298a4e8b282b3a5c Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:43:14 +0530 Subject: [PATCH 07/10] docs: add SECURITY.md with private vulnerability reporting policy Co-Authored-By: Claude Sonnet 5.5 --- CONTRIBUTING.md | 6 +++++ README.md | 1 + SECURITY.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 SECURITY.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b54c642..6e83d08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,12 @@ Thank you for your interest in contributing to BeastDB! We welcome contributions --- +## ๐Ÿ”’ Security Issues + +Do not report vulnerabilities in public issues or pull requests. Follow [SECURITY.md](SECURITY.md). + +--- + ## ๐Ÿงช Verification & Testing Before submitting a pull request, ensure all validations pass: diff --git a/README.md b/README.md index 169640d..c5ad977 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ BeastDB is optimized for **read-heavy OLTP** โ€” deterministic point reads and o - [**V1 Semantics & Guarantees**](docs/v1-semantics.md) โ€” Durability acknowledgment, limits, and replica reads. - [**System Benchmarks**](docs/benchmarks.md) โ€” Reproducible throughput, tail latencies, and cache-eviction profiles. - [**Benchmarking Guide**](docs/benchmarking.md) โ€” `beastbench` YCSB-style workloads and how to compare fairly with other databases. +- [**Security Policy**](SECURITY.md) โ€” How to report a vulnerability privately, scope, and safe deployment checklist. - [**Upgrade Spec**](docs/upgrade-spec.md) โ€” Security, usability and performance work packages and what is still planned. - [**Operator Guide**](docs/runbooks/operator-guide.md) โ€” TLS configuration, cluster roles, manual failover, and `/healthz` probes. - [**Disaster Recovery**](docs/runbooks/disaster-recovery.md) โ€” Physical backup drills, checksum verification, and RTO/RPO SLA. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..ebf6904 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,62 @@ +# Security Policy + +## Reporting a vulnerability + +**Please do not open a public issue, pull request or discussion for a security problem.** + +Report it privately through GitHub: + +1. Go to the repository's **Security** tab. +2. Choose **Report a vulnerability** (GitHub private vulnerability reporting). + +Include what you can of: + +- the affected version or commit and how you run BeastDB (flags, TLS, roles, replication); +- steps to reproduce, or a proof of concept; +- the impact you observed (for example unauthorised read or write, data corruption, denial of service); +- any suggested fix. + +If the **Report a vulnerability** button is missing, the maintainers have not enabled it yet. Open a public issue that says only "security contact needed" and contains no technical details, and a maintainer will arrange a private channel. + +### What to expect + +BeastDB is maintained by a small team, so these are goals, not guarantees: + +- acknowledgement of your report within about 5 working days; +- an assessment and a plan within about 14 days; +- a fix or documented mitigation as quickly as the severity allows, with coordinated public disclosure after a fix is available. We will credit you if you want to be credited. + +## Supported versions + +BeastDB has not reached a tagged V1 release (see [ROADMAP.md](ROADMAP.md)). Until then, security fixes are made on the `main` branch only. Once V1 is released, this section will list the supported release lines. + +## Scope + +In scope: + +- authentication and authorization (gRPC bearer tokens and roles, Studio sessions and cookies, the `/metrics` endpoint); +- transport security (TLS and mTLS for clients and replicas); +- the storage engine, WAL, recovery, backup and restore paths, where a flaw could lose or corrupt acknowledged data or expose data to the wrong caller; +- the replication stream; +- the container image and default configuration shipped in this repository. + +Out of scope: + +- findings that require the attacker to already have admin credentials, local root access, or write access to the data directory; +- running with `--dev` or `--insecure-auth` on a reachable network (these flags are documented as for local, non-production use); +- denial of service that needs a very large volume of traffic; +- missing hardening that is already listed as planned in [docs/upgrade-spec.md](docs/upgrade-spec.md) (for example encryption at rest). Reports on how a planned item could be exploited today are still welcome. + +## Deploying safely + +The [Operator Guide](docs/runbooks/operator-guide.md) is the reference. In short: + +- set a strong `BEASTDB_ADMIN_PASSWORD`; the server refuses the default password on non-loopback listeners and when `BEASTDB_ENV=production`; +- enable TLS (and mTLS for replicas) and keep gRPC and Studio off public interfaces; +- give each application its own API token with the smallest role it needs (`viewer`, `writer`, `admin`, `replication`) and set an expiry where you can; +- protect `/metrics` with a dedicated `BEASTDB_METRICS_TOKEN` and scrape it over TLS; +- keep backups off the host and test restores. + +## Safe harbour + +We will not pursue action against researchers who make a good-faith effort to follow this policy: test only against systems you own or have permission to test, avoid accessing or modifying other people's data, do not degrade service for others, and give us reasonable time to fix an issue before disclosing it. From 2cac2c9d771d5af700ebb51f73268cd3a44a80eb Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:45:59 +0530 Subject: [PATCH 08/10] refactor: split wal.go, buffer_pool.go and beastbench runner.go under the 200-line limit No behaviour change: functions moved verbatim into wal_sync.go, wal_lifecycle.go, buffer_pool_wal.go, buffer_pool_stats.go, runner_load.go and runner_run.go. Co-Authored-By: Claude Sonnet 5.5 --- cmd/beastbench/runner.go | 215 -------------------------- cmd/beastbench/runner_load.go | 72 +++++++++ cmd/beastbench/runner_run.go | 158 +++++++++++++++++++ internal/storage/buffer_pool.go | 41 ----- internal/storage/buffer_pool_stats.go | 24 +++ internal/storage/buffer_pool_wal.go | 19 +++ internal/wal/wal.go | 130 ---------------- internal/wal/wal_lifecycle.go | 57 +++++++ internal/wal/wal_sync.go | 79 ++++++++++ 9 files changed, 409 insertions(+), 386 deletions(-) create mode 100644 cmd/beastbench/runner_load.go create mode 100644 cmd/beastbench/runner_run.go create mode 100644 internal/storage/buffer_pool_stats.go create mode 100644 internal/storage/buffer_pool_wal.go create mode 100644 internal/wal/wal_lifecycle.go create mode 100644 internal/wal/wal_sync.go diff --git a/cmd/beastbench/runner.go b/cmd/beastbench/runner.go index 69cd245..197ee7a 100644 --- a/cmd/beastbench/runner.go +++ b/cmd/beastbench/runner.go @@ -1,19 +1,8 @@ package main import ( - "context" "encoding/binary" - "fmt" - "io" - "math" - "math/rand" - "sync" - "sync/atomic" "time" - - beastv1 "github.com/ChromaBeast/beastdb/api/proto" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) // Config holds everything that determines a run. @@ -87,210 +76,6 @@ func newCounters() *counters { return c } -// Load inserts cfg.Records records with the given parallelism. -func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { - cs := make([]*counters, cfg.Threads) - var wg sync.WaitGroup - var firstErr atomic.Value - start := time.Now() - per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) - for t := 0; t < cfg.Threads; t++ { - cs[t] = newCounters() - wg.Add(1) - go func(t int) { - defer wg.Done() - c := cs[t] - client := cl[t%len(cl)] - buf := make([]byte, cfg.ValueSize) - lo := uint64(t) * per - hi := lo + per - if hi > cfg.Records { - hi = cfg.Records - } - for i := lo; i < hi; i++ { - if ctx.Err() != nil { - return - } - makeValue(buf, i, 0) - t0 := time.Now() - _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) - c.hist[OpInsert].Record(time.Since(t0)) - if err != nil { - c.errs[OpInsert]++ - firstErr.CompareAndSwap(nil, err) - return - } - } - }(t) - } - wg.Wait() - el := time.Since(start) - res := summarize("load", el, cs) - if e, ok := firstErr.Load().(error); ok { - return res, fmt.Errorf("load failed: %w", e) - } - return res, ctx.Err() -} - -// callPut returns conflict=true when a conditional write was rejected (ABORTED). -func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) - if err != nil { - if status.Code(err) == codes.Aborted { - return true, nil - } - return false, err - } - return false, nil -} - -// Run executes warmup (unrecorded) then the measured run phase. -func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { - w, err := lookupWorkload(cfg.Workload) - if err != nil { - return PhaseResult{}, err - } - frontier := &atomic.Uint64{} - frontier.Store(cfg.Records) - - var chooser KeyChooser - switch { - case w.Latest: - chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} - case cfg.Dist == "uniform": - chooser = Uniform{N: cfg.Records} - default: - chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) - } - - phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { - cs := make([]*counters, cfg.Threads) - var wg sync.WaitGroup - start := time.Now() - for t := 0; t < cfg.Threads; t++ { - cs[t] = newCounters() - n := total / uint64(cfg.Threads) - if uint64(t) < total%uint64(cfg.Threads) { - n++ - } - wg.Add(1) - go func(t int, n uint64) { - defer wg.Done() - r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) - worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) - }(t, n) - } - wg.Wait() - return cs, time.Since(start) - } - - if cfg.WarmupOps > 0 { - phase(cfg.WarmupOps, 1_000_003, false) - } - cs, el := phase(cfg.Ops, 0, true) - return summarize("run", el, cs), ctx.Err() -} - -func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, - ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { - buf := make([]byte, cfg.ValueSize) - for i := uint64(0); i < n && ctx.Err() == nil; i++ { - op := w.pick(r.Float64()) - t0 := time.Now() - var err error - var miss, conflict bool - switch op { - case OpRead: - idx := ch.Next(r) - var found bool - found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) - miss = err == nil && !found - case OpUpdate: - idx := ch.Next(r) - r.Read(buf) - conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) - case OpInsert: - idx := frontier.Add(1) - 1 - makeValue(buf, idx, 0) - conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) - case OpScan: - idx := ch.Next(r) - ln := 1 + r.Intn(cfg.MaxScanLen) - var got int - got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) - miss = err == nil && got == 0 - case OpRMW: - idx := ch.Next(r) - var old []byte - var found bool - found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) - if err == nil && !found { - miss = true - break - } - if err == nil { - nv := append([]byte(nil), old...) - if len(nv) > 0 { - nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) - } - conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ - Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) - } - } - if ctx.Err() != nil { - return - } - if !record { - continue - } - c.hist[op].Record(time.Since(t0)) - if err != nil { - c.errs[op]++ - } - if miss { - c.misses[op]++ - } - if conflict { - c.conflicts[op]++ - } - } -} - -func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) - if err != nil { - return false, err - } - if out != nil { - *out = resp.Value - } - return resp.Found, nil -} - -func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { - cctx, cancel := context.WithTimeout(ctx, to) - defer cancel() - st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) - if err != nil { - return 0, err - } - n := 0 - for { - _, err := st.Recv() - if err == io.EOF { - return n, nil - } - if err != nil { - return n, err - } - n++ - } -} - func summarize(name string, el time.Duration, cs []*counters) PhaseResult { res := PhaseResult{Phase: name, Seconds: el.Seconds()} for op := OpType(0); op < numOps; op++ { diff --git a/cmd/beastbench/runner_load.go b/cmd/beastbench/runner_load.go new file mode 100644 index 0000000..0bd5757 --- /dev/null +++ b/cmd/beastbench/runner_load.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Load inserts cfg.Records records with the given parallelism. +func Load(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + var firstErr atomic.Value + start := time.Now() + per := (cfg.Records + uint64(cfg.Threads) - 1) / uint64(cfg.Threads) + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + wg.Add(1) + go func(t int) { + defer wg.Done() + c := cs[t] + client := cl[t%len(cl)] + buf := make([]byte, cfg.ValueSize) + lo := uint64(t) * per + hi := lo + per + if hi > cfg.Records { + hi = cfg.Records + } + for i := lo; i < hi; i++ { + if ctx.Err() != nil { + return + } + makeValue(buf, i, 0) + t0 := time.Now() + _, err := callPut(ctx, client, recordKey(i), buf, nil, cfg.Timeout) + c.hist[OpInsert].Record(time.Since(t0)) + if err != nil { + c.errs[OpInsert]++ + firstErr.CompareAndSwap(nil, err) + return + } + } + }(t) + } + wg.Wait() + el := time.Since(start) + res := summarize("load", el, cs) + if e, ok := firstErr.Load().(error); ok { + return res, fmt.Errorf("load failed: %w", e) + } + return res, ctx.Err() +} + +// callPut returns conflict=true when a conditional write was rejected (ABORTED). +func callPut(ctx context.Context, c beastv1.BeastDBServiceClient, key uint64, val []byte, cond *beastv1.Condition, to time.Duration) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + _, err := c.Put(cctx, &beastv1.PutRequest{Key: key, Value: val, Condition: cond}) + if err != nil { + if status.Code(err) == codes.Aborted { + return true, nil + } + return false, err + } + return false, nil +} diff --git a/cmd/beastbench/runner_run.go b/cmd/beastbench/runner_run.go new file mode 100644 index 0000000..73e0030 --- /dev/null +++ b/cmd/beastbench/runner_run.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "io" + "math" + "math/rand" + "sync" + "sync/atomic" + "time" + + beastv1 "github.com/ChromaBeast/beastdb/api/proto" +) + +// Run executes warmup (unrecorded) then the measured run phase. +func Run(ctx context.Context, cl []beastv1.BeastDBServiceClient, cfg Config) (PhaseResult, error) { + w, err := lookupWorkload(cfg.Workload) + if err != nil { + return PhaseResult{}, err + } + frontier := &atomic.Uint64{} + frontier.Store(cfg.Records) + + var chooser KeyChooser + switch { + case w.Latest: + chooser = Latest{Zipf: NewZipfian(cfg.Records, cfg.ZipfTheta), Frontier: frontier} + case cfg.Dist == "uniform": + chooser = Uniform{N: cfg.Records} + default: + chooser = NewZipfian(cfg.Records, cfg.ZipfTheta) + } + + phase := func(total uint64, seedOff int64, record bool) ([]*counters, time.Duration) { + cs := make([]*counters, cfg.Threads) + var wg sync.WaitGroup + start := time.Now() + for t := 0; t < cfg.Threads; t++ { + cs[t] = newCounters() + n := total / uint64(cfg.Threads) + if uint64(t) < total%uint64(cfg.Threads) { + n++ + } + wg.Add(1) + go func(t int, n uint64) { + defer wg.Done() + r := rand.New(rand.NewSource(cfg.Seed + seedOff + int64(t)*7919)) + worker(ctx, cl[t%len(cl)], cfg, w, chooser, frontier, r, n, cs[t], record) + }(t, n) + } + wg.Wait() + return cs, time.Since(start) + } + + if cfg.WarmupOps > 0 { + phase(cfg.WarmupOps, 1_000_003, false) + } + cs, el := phase(cfg.Ops, 0, true) + return summarize("run", el, cs), ctx.Err() +} + +func worker(ctx context.Context, cl beastv1.BeastDBServiceClient, cfg Config, w Workload, + ch KeyChooser, frontier *atomic.Uint64, r *rand.Rand, n uint64, c *counters, record bool) { + buf := make([]byte, cfg.ValueSize) + for i := uint64(0); i < n && ctx.Err() == nil; i++ { + op := w.pick(r.Float64()) + t0 := time.Now() + var err error + var miss, conflict bool + switch op { + case OpRead: + idx := ch.Next(r) + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, nil) + miss = err == nil && !found + case OpUpdate: + idx := ch.Next(r) + r.Read(buf) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpInsert: + idx := frontier.Add(1) - 1 + makeValue(buf, idx, 0) + conflict, err = callPut(ctx, cl, recordKey(idx), buf, nil, cfg.Timeout) + case OpScan: + idx := ch.Next(r) + ln := 1 + r.Intn(cfg.MaxScanLen) + var got int + got, err = doScan(ctx, cl, recordKey(idx), uint32(ln), cfg.Timeout) + miss = err == nil && got == 0 + case OpRMW: + idx := ch.Next(r) + var old []byte + var found bool + found, err = doGet(ctx, cl, recordKey(idx), cfg.Timeout, &old) + if err == nil && !found { + miss = true + break + } + if err == nil { + nv := append([]byte(nil), old...) + if len(nv) > 0 { + nv[r.Intn(len(nv))] ^= byte(1 + r.Intn(255)) + } + conflict, err = callPut(ctx, cl, recordKey(idx), nv, &beastv1.Condition{ + Kind: beastv1.ConditionKind_CONDITION_KIND_VALUE_EQUALS, Value: old}, cfg.Timeout) + } + } + if ctx.Err() != nil { + return + } + if !record { + continue + } + c.hist[op].Record(time.Since(t0)) + if err != nil { + c.errs[op]++ + } + if miss { + c.misses[op]++ + } + if conflict { + c.conflicts[op]++ + } + } +} + +func doGet(ctx context.Context, cl beastv1.BeastDBServiceClient, key uint64, to time.Duration, out *[]byte) (bool, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + resp, err := cl.Get(cctx, &beastv1.GetRequest{Key: key}) + if err != nil { + return false, err + } + if out != nil { + *out = resp.Value + } + return resp.Found, nil +} + +func doScan(ctx context.Context, cl beastv1.BeastDBServiceClient, start uint64, limit uint32, to time.Duration) (int, error) { + cctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + st, err := cl.Scan(cctx, &beastv1.ScanRequest{StartKey: start, EndKey: math.MaxUint64, Limit: limit}) + if err != nil { + return 0, err + } + n := 0 + for { + _, err := st.Recv() + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + n++ + } +} diff --git a/internal/storage/buffer_pool.go b/internal/storage/buffer_pool.go index f29df84..7d1c176 100644 --- a/internal/storage/buffer_pool.go +++ b/internal/storage/buffer_pool.go @@ -33,24 +33,6 @@ type BufferPoolManager struct { walBarrier func() error } -// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. -func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { - bpm.mu.Lock() - defer bpm.mu.Unlock() - bpm.walBarrier = fn -} - -// writeBack writes one dirty frame to the data file after the WAL barrier. -// Caller must hold bpm.mu (write lock). -func (bpm *BufferPoolManager) writeBack(f *Frame) error { - if bpm.walBarrier != nil { - if err := bpm.walBarrier(); err != nil { - return err - } - } - return bpm.disk.WritePage(f.PageID, f.Page.Data()) -} - // NewBufferPoolManager creates a pool with a fixed number of in-memory frames. func NewBufferPoolManager(disk *DiskManager, poolSize int) *BufferPoolManager { frames := make([]*Frame, poolSize) @@ -180,26 +162,3 @@ func (bpm *BufferPoolManager) FlushAll() error { } return bpm.disk.Sync() } - -// HitMiss returns cumulative FetchPage cache hits and misses. -func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { - return bpm.hits.Load(), bpm.misses.Load() -} - -// Stats returns operational metrics for the buffer pool. -func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { - bpm.mu.RLock() - defer bpm.mu.RUnlock() - - poolSize = bpm.poolSize - cachedPages = len(bpm.pageTable) - for _, frame := range bpm.frames { - if frame.PinCount() > 0 { - pinnedPages++ - } - if frame.IsDirty { - dirtyPages++ - } - } - return -} diff --git a/internal/storage/buffer_pool_stats.go b/internal/storage/buffer_pool_stats.go new file mode 100644 index 0000000..3fae07a --- /dev/null +++ b/internal/storage/buffer_pool_stats.go @@ -0,0 +1,24 @@ +package storage + +// HitMiss returns cumulative FetchPage cache hits and misses. +func (bpm *BufferPoolManager) HitMiss() (hits, misses uint64) { + return bpm.hits.Load(), bpm.misses.Load() +} + +// Stats returns operational metrics for the buffer pool. +func (bpm *BufferPoolManager) Stats() (poolSize, cachedPages, pinnedPages, dirtyPages int) { + bpm.mu.RLock() + defer bpm.mu.RUnlock() + + poolSize = bpm.poolSize + cachedPages = len(bpm.pageTable) + for _, frame := range bpm.frames { + if frame.PinCount() > 0 { + pinnedPages++ + } + if frame.IsDirty { + dirtyPages++ + } + } + return +} diff --git a/internal/storage/buffer_pool_wal.go b/internal/storage/buffer_pool_wal.go new file mode 100644 index 0000000..9ba7820 --- /dev/null +++ b/internal/storage/buffer_pool_wal.go @@ -0,0 +1,19 @@ +package storage + +// SetWALBarrier installs the write-ahead barrier. Call it before the pool is shared. +func (bpm *BufferPoolManager) SetWALBarrier(fn func() error) { + bpm.mu.Lock() + defer bpm.mu.Unlock() + bpm.walBarrier = fn +} + +// writeBack writes one dirty frame to the data file after the WAL barrier. +// Caller must hold bpm.mu (write lock). +func (bpm *BufferPoolManager) writeBack(f *Frame) error { + if bpm.walBarrier != nil { + if err := bpm.walBarrier(); err != nil { + return err + } + } + return bpm.disk.WritePage(f.PageID, f.Page.Data()) +} diff --git a/internal/wal/wal.go b/internal/wal/wal.go index 7261209..57f1860 100644 --- a/internal/wal/wal.go +++ b/internal/wal/wal.go @@ -3,7 +3,6 @@ package wal import ( "io" "os" - "path/filepath" "sync" ) @@ -110,84 +109,6 @@ func (w *WAL) Write(opType byte, key, value []byte) (uint64, error) { // SyncOnWrite reports whether this log was opened with fsync-per-commit semantics. func (w *WAL) SyncOnWrite() bool { return w.syncOnWrite } -func (w *WAL) loadSyncErr() error { - w.errMu.Lock() - defer w.errMu.Unlock() - return w.syncErr -} - -// WaitDurable blocks until the record with the given LSN has been fsynced. -// If another caller's fsync already covered it, no further fsync is issued. -func (w *WAL) WaitDurable(lsn uint64) error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - if err := w.loadSyncErr(); err != nil { - return err - } - if w.syncedLSN >= lsn { - return nil - } - return w.syncLocked() -} - -// syncLocked fsyncs everything appended so far. Caller holds syncMu. -func (w *WAL) syncLocked() error { - w.mu.Lock() - target := w.currentLSN - f := w.file - w.mu.Unlock() - - // Appends may proceed while fsync runs; they simply miss this sync and are - // covered by the next one. target only claims what was written before it. - if err := f.Sync(); err != nil { - w.errMu.Lock() - w.syncErr = err - w.errMu.Unlock() - return err - } - if target > w.syncedLSN { - w.syncedLSN = target - } - return nil -} - -// Sync forces all buffered kernel writes to physical non-volatile storage. -func (w *WAL) Sync() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - if err := w.loadSyncErr(); err != nil { - return err - } - return w.syncLocked() -} - -// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. -func (w *WAL) SyncIfEnabled() error { - if !w.syncOnWrite { - return nil - } - return w.Sync() -} - -// EnsureDurable returns once every record appended so far is fsynced. It is a -// no-op when the log was not opened with syncOnWrite, and costs no fsync when -// a previous sync already covers the current LSN. The buffer pool calls it -// before writing a dirty page so a data page never reaches disk ahead of the -// WAL records that describe it. -func (w *WAL) EnsureDurable() error { - if !w.syncOnWrite { - return nil - } - return w.WaitDurable(w.CurrentLSN()) -} - -// DurableLSN returns the highest LSN known to be fsynced. -func (w *WAL) DurableLSN() uint64 { - w.syncMu.Lock() - defer w.syncMu.Unlock() - return w.syncedLSN -} - // CurrentLSN returns the latest assigned Log Sequence Number. func (w *WAL) CurrentLSN() uint64 { w.mu.Lock() @@ -211,57 +132,6 @@ func (w *WAL) Path() string { return w.path } -// Close flushes buffered writes and closes the log file. -func (w *WAL) Close() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - w.mu.Lock() - defer w.mu.Unlock() - _ = w.file.Sync() - return w.file.Close() -} - -// Checkpoint rotates the WAL: syncs the current file, archives it as .bak, -// then opens a fresh append-only log starting from the next LSN. -// The caller MUST have flushed all dirty buffer pool pages to disk first so the -// archived WAL records are no longer needed for recovery. -func (w *WAL) Checkpoint() error { - w.syncMu.Lock() - defer w.syncMu.Unlock() - w.mu.Lock() - defer w.mu.Unlock() - - if err := w.file.Sync(); err != nil { - return err - } - // Everything appended so far is now durable (and about to be archived). - w.syncedLSN = w.currentLSN - if err := w.file.Close(); err != nil { - return err - } - - // Archive the old WAL; keep one generation for safety. - archivePath := w.path + ".bak" - _ = os.Remove(archivePath) - if err := os.Rename(w.path, archivePath); err != nil { - return err - } - - // Open a fresh log file for subsequent writes. - newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) - if err != nil { - return err - } - w.file = newFile - - // Persist the rename and the new file's directory entry. - if dir, err := os.Open(filepath.Dir(w.path)); err == nil { - _ = dir.Sync() - _ = dir.Close() - } - return nil -} - func findHighestLSN(file *os.File) (uint64, error) { if _, err := file.Seek(0, io.SeekStart); err != nil { return 0, err diff --git a/internal/wal/wal_lifecycle.go b/internal/wal/wal_lifecycle.go new file mode 100644 index 0000000..5de4ae0 --- /dev/null +++ b/internal/wal/wal_lifecycle.go @@ -0,0 +1,57 @@ +package wal + +import ( + "os" + "path/filepath" +) + +// Close flushes buffered writes and closes the log file. +func (w *WAL) Close() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + w.mu.Lock() + defer w.mu.Unlock() + _ = w.file.Sync() + return w.file.Close() +} + +// Checkpoint rotates the WAL: syncs the current file, archives it as .bak, +// then opens a fresh append-only log starting from the next LSN. +// The caller MUST have flushed all dirty buffer pool pages to disk first so the +// archived WAL records are no longer needed for recovery. +func (w *WAL) Checkpoint() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + w.mu.Lock() + defer w.mu.Unlock() + + if err := w.file.Sync(); err != nil { + return err + } + // Everything appended so far is now durable (and about to be archived). + w.syncedLSN = w.currentLSN + if err := w.file.Close(); err != nil { + return err + } + + // Archive the old WAL; keep one generation for safety. + archivePath := w.path + ".bak" + _ = os.Remove(archivePath) + if err := os.Rename(w.path, archivePath); err != nil { + return err + } + + // Open a fresh log file for subsequent writes. + newFile, err := os.OpenFile(w.path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) + if err != nil { + return err + } + w.file = newFile + + // Persist the rename and the new file's directory entry. + if dir, err := os.Open(filepath.Dir(w.path)); err == nil { + _ = dir.Sync() + _ = dir.Close() + } + return nil +} diff --git a/internal/wal/wal_sync.go b/internal/wal/wal_sync.go new file mode 100644 index 0000000..c8f40bc --- /dev/null +++ b/internal/wal/wal_sync.go @@ -0,0 +1,79 @@ +package wal + +func (w *WAL) loadSyncErr() error { + w.errMu.Lock() + defer w.errMu.Unlock() + return w.syncErr +} + +// WaitDurable blocks until the record with the given LSN has been fsynced. +// If another caller's fsync already covered it, no further fsync is issued. +func (w *WAL) WaitDurable(lsn uint64) error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + if w.syncedLSN >= lsn { + return nil + } + return w.syncLocked() +} + +// syncLocked fsyncs everything appended so far. Caller holds syncMu. +func (w *WAL) syncLocked() error { + w.mu.Lock() + target := w.currentLSN + f := w.file + w.mu.Unlock() + + // Appends may proceed while fsync runs; they simply miss this sync and are + // covered by the next one. target only claims what was written before it. + if err := f.Sync(); err != nil { + w.errMu.Lock() + w.syncErr = err + w.errMu.Unlock() + return err + } + if target > w.syncedLSN { + w.syncedLSN = target + } + return nil +} + +// Sync forces all buffered kernel writes to physical non-volatile storage. +func (w *WAL) Sync() error { + w.syncMu.Lock() + defer w.syncMu.Unlock() + if err := w.loadSyncErr(); err != nil { + return err + } + return w.syncLocked() +} + +// SyncIfEnabled fsyncs only when the log was opened with syncOnWrite. +func (w *WAL) SyncIfEnabled() error { + if !w.syncOnWrite { + return nil + } + return w.Sync() +} + +// EnsureDurable returns once every record appended so far is fsynced. It is a +// no-op when the log was not opened with syncOnWrite, and costs no fsync when +// a previous sync already covers the current LSN. The buffer pool calls it +// before writing a dirty page so a data page never reaches disk ahead of the +// WAL records that describe it. +func (w *WAL) EnsureDurable() error { + if !w.syncOnWrite { + return nil + } + return w.WaitDurable(w.CurrentLSN()) +} + +// DurableLSN returns the highest LSN known to be fsynced. +func (w *WAL) DurableLSN() uint64 { + w.syncMu.Lock() + defer w.syncMu.Unlock() + return w.syncedLSN +} From ade5d0ae395b215ebdf0217ade2f3c75ccef011d Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:49:18 +0530 Subject: [PATCH 09/10] docs: replace hard 200-line limit with a one-responsibility-per-file guideline About 400 lines is a prompt to ask whether a file does more than one thing, not a gate. Generated code, tests and data are exempt. Updates GEMINI.md (Antigravity), CONTRIBUTING.md and demo.ps1, and adds AGENTS.md (Codex) and CLAUDE.md (imports AGENTS.md). Co-Authored-By: Claude Sonnet 5.5 --- AGENTS.md | 26 ++++++++++++++++++++++++++ CLAUDE.md | 1 + CONTRIBUTING.md | 14 ++++++++------ GEMINI.md | 9 ++++++--- demo.ps1 | 18 +++++++++--------- 5 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..cb8f16f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# BeastDB: instructions for coding agents + +Read by Codex and other agents that use `AGENTS.md`. Claude Code reads it through `CLAUDE.md`. Antigravity also reads `GEMINI.md`, which carries the same rules; keep the two in step. + +## File modularity (guideline, not a hard limit) + +- One responsibility per file. About **400 lines** is the point to stop and ask "does this file do more than one thing?" If yes, split along that seam (for example sync vs. lifecycle, load vs. run). If no, leave it alone. +- Never split a cohesive file, or merge small files, just to satisfy a line count. +- Exempt: generated code (`*.pb.go`), test files, JSON and raw data. +- Do not add CI or lint checks that fail on line count. A non-exempt file past about 500 lines should say why in the change description. + +## Engine principles + +- **WAL before page:** a data page must never reach the data file before the WAL records that describe it are durable. The buffer pool enforces this with the WAL barrier (`WAL.EnsureDurable`); keep every dirty-page write on that path. +- **Zero-allocation hot paths:** query, hashing, parsing and buffer-pinning paths avoid heap allocations; prefer byte slices and `sync.Pool`. +- **Mechanical sympathy:** align structs to avoid false sharing, prefer flat memory layouts, respect 4 KB page alignment. + +## Scope and honesty + +- BeastDB is a general-purpose, domain-agnostic database. Do not hard-code schema or domain rules from downstream applications; client-specific partition metadata belongs in `partitions.json` or `-partition-config`. +- Do not claim performance relative to other databases without an equivalent-workload benchmark (see `docs/benchmarking.md`). Describe planned features as planned. +- Report vulnerabilities privately; see `SECURITY.md`. + +## Before you finish + +Run `go build ./... && go vet ./... && go test ./... -race`. `TestEngineSystemWorkloadBenchmark` has a fixed 1000 ops/s floor and can fail when the machine is loaded; re-run it alone before treating a failure as a regression. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e83d08..7246279 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,9 +6,11 @@ Thank you for your interest in contributing to BeastDB! We welcome contributions ## ๐Ÿ›๏ธ Engineering Standards -### 1. Strict File Modularity (< 200 LoC) -- **Constraint:** Every source file (excluding auto-generated `.pb.go` code) must strictly remain readable and **under 200 Lines of Code (LoC)**. -- **Decomposition:** Large subsystems (B+ Tree, Buffer Pool, Hash Table, WAL) must be decomposed into focused, single-responsibility files and companion helpers. +### 1. File Modularity (guideline, not a hard limit) +- **Principle:** One responsibility per file. Treat about **400 lines** as a prompt to ask "does this file do more than one thing?" If it does, split along that seam. If it doesn't, leave it as is. Never split, or merge, files only to hit a number. +- **Exempt:** generated code (`*.pb.go`), test files, and data files (JSON and similar). +- **Review, not CI:** there is no automated line-count gate. A non-exempt file past about 500 lines should say in the pull request why it stays one file. +- **Decomposition:** Large subsystems (B+ Tree, Buffer Pool, Hash Table, WAL) should be organized into focused, single-responsibility files and companion helpers. ### 2. Zero-Allocation Hot Paths - Query execution, hash calculation, parsing, and buffer pinning paths must not trigger unnecessary heap allocations. @@ -38,9 +40,9 @@ go test -v -count=1 ./... # 2. Run benchmarks to verify zero allocations on hot paths go test -bench=. -benchmem ./... -# 3. Enforce the < 200 LoC modularity limit -Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.FullName -notmatch "\.pb\.go" } | ForEach-Object { +# 3. (Optional) List large non-test, non-generated files to review. Informational only. +Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.Name -notmatch "\.pb\.go$|_test\.go$" } | ForEach-Object { $lines = (Get-Content $_.FullName).Count - if ($lines -ge 200) { Write-Error "$lines lines in $($_.Name)" } + if ($lines -ge 400) { Write-Host "$lines lines in $($_.FullName): does it do more than one thing?" } } ``` diff --git a/GEMINI.md b/GEMINI.md index 3ffbbe1..6150a7e 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -4,9 +4,12 @@ These rules govern all interactions, coding, and architecture in this project. --- -## 1. File Modularity & Readability (< 200 LoC) -* **Strict File Length Limit:** In any project, if a file is not JSON or raw data, it must strictly remain readable and **< 200 Lines of Code (LoC)**. -* **Component Decomposition:** Decompose complex systems (B+ trees, buffer pools, hash tables, page managers) into small, single-responsibility files, structs, and reusable components. +## 1. File Modularity & Readability (guideline, not a hard limit) +* **One responsibility per file.** Organize files by what they do. File length is a smell, not a rule: roughly **400 lines** is the point to stop and ask "does this file do more than one thing?" If yes, split it along that seam (for example sync vs. lifecycle, load vs. run). If no, leave it alone. +* **Never split just to hit a number.** Do not cut a cohesive file into arbitrary pieces, move code without a clear responsibility boundary, or merge small files back together to satisfy a count. +* **Exemptions:** generated code (`*.pb.go`), test files, and JSON/raw data have no size expectation. +* **No automated gate.** Do not add CI or lint checks that fail on line count. Past about 500 lines, mention the reason in the change description or review. +* **Component Decomposition:** Decompose complex systems (B+ trees, buffer pools, hash tables, page managers) into focused structs and reusable components. --- diff --git a/demo.ps1 b/demo.ps1 index 9dfe871..435fa20 100644 --- a/demo.ps1 +++ b/demo.ps1 @@ -62,21 +62,21 @@ Write-Host ' [OK] Corrupted disk blocks detected via IEEE CRC32 checksum' -Fore Write-Host ' [OK] Clean WAL recovery and state restoration verified' -ForegroundColor Green Start-Sleep -Seconds 3 -# ACT 3: Code Modularity & Quality Audit (< 200 LoC) -Print-Section "ACT 3" "Strict Architectural Modularity (< 200 LoC per file)" -Simulate-Typing 'audit-codebase --enforce-max-loc 200' -Write-Host " -> Scanning all Go source files across all packages..." -ForegroundColor Gray +# ACT 3: Code Modularity Review (guideline: one responsibility per file) +Print-Section "ACT 3" "Architectural Modularity (one responsibility per file)" +Simulate-Typing 'audit-codebase --review-above 400' +Write-Host " -> Scanning Go source files (generated and test files exempt)..." -ForegroundColor Gray Start-Sleep -Milliseconds 400 -$violations = Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.FullName -notmatch "\.pb\.go" } | ForEach-Object { +$large = Get-ChildItem -Recurse -Filter "*.go" | Where-Object { $_.Name -notmatch "\.pb\.go$|_test\.go$" } | ForEach-Object { $lines = (Get-Content $_.FullName).Count - if ($lines -ge 200) { "$lines lines: $($_.Name)" } + if ($lines -ge 400) { "$lines lines: $($_.Name)" } } -if ($violations.Count -eq 0) { - Write-Host ' [SUCCESS] AUDIT PASSED: ZERO files exceed 200 lines of code!' -ForegroundColor Green +if ($large.Count -eq 0) { + Write-Host ' [OK] No source file is large enough to need a modularity review' -ForegroundColor Green } else { - $violations | ForEach-Object { Write-Host " [FAIL] VIOLATION: $_" -ForegroundColor Red } + $large | ForEach-Object { Write-Host " [REVIEW] $_ - does it do more than one thing?" -ForegroundColor Yellow } } Start-Sleep -Seconds 2 From c6b851e68293904897925758c078fe33c13a17a6 Mon Sep 17 00:00:00 2001 From: ChromaBeast Date: Thu, 12 Nov 2026 16:53:16 +0530 Subject: [PATCH 10/10] docs: update phase checklists to the file-modularity guideline Co-Authored-By: Claude Sonnet 5.5 --- docs/phase1_dsa.md | 2 +- docs/phase2_net_cache.md | 2 +- docs/phase3_storage.md | 2 +- docs/phase4_scaling.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/phase1_dsa.md b/docs/phase1_dsa.md index d83f92f..8e47ab8 100644 --- a/docs/phase1_dsa.md +++ b/docs/phase1_dsa.md @@ -67,5 +67,5 @@ Focuses on low-level memory representation, zero-copy byte semantics, and pure G ## Success Verification - [ ] 0 allocations per operation on hot lookup paths. -- [ ] All data structure files adhere to the strict `< 200 LoC` limit. +- [ ] Each data structure file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). - [ ] Comprehensive unit test coverage (>90%) with fuzz tests for edge cases. diff --git a/docs/phase2_net_cache.md b/docs/phase2_net_cache.md index 0ead3d3..228e9aa 100644 --- a/docs/phase2_net_cache.md +++ b/docs/phase2_net_cache.md @@ -73,4 +73,4 @@ Builds the network communication layer, protocol parsing engines, and concurrent - [ ] Connect with standard `redis-cli` and execute `SET`, `GET`, `DEL`, `PING`. - [ ] Handle 10,000 concurrent client connections without deadlocks or goroutine leaks. - [ ] LRU evicts least recently accessed keys accurately under memory pressure. -- [ ] All code files remain strictly under 200 LoC. +- [ ] Each code file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). diff --git a/docs/phase3_storage.md b/docs/phase3_storage.md index ded9acd..9a1dc9c 100644 --- a/docs/phase3_storage.md +++ b/docs/phase3_storage.md @@ -73,4 +73,4 @@ B+ Tree nodes serialized directly inside 4KB slotted pages. - [ ] Process recovery test: Crash during active load; re-open DB and verify 100% data consistency via WAL replay. - [ ] Sequential scan matches exact key ordering across 100,000 inserted keys. - [ ] Zero unpinned frame leaks under heavy concurrent read/write transactions. -- [ ] All source files modularized to `< 200 LoC`. +- [ ] Each source file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)). diff --git a/docs/phase4_scaling.md b/docs/phase4_scaling.md index 464a389..3cd4c2f 100644 --- a/docs/phase4_scaling.md +++ b/docs/phase4_scaling.md @@ -76,4 +76,4 @@ Expands the database engine into a production-ready distributed system with gRPC - [ ] Container image build succeeds under 25MB total footprint. - [ ] Server cleanly flushes and terminates on `SIGTERM` without corrupting active pages. - [ ] Sustained 50k+ QPS with p99 latency under 2ms. -- [ ] All source files modularized to `< 200 LoC`. +- [ ] Each source file has a single responsibility (guideline: review any file over about 400 lines; see [CONTRIBUTING](../CONTRIBUTING.md)).