From 2a7094bac0007523ba92033dcca10ffac6ffea03 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Fri, 4 Sep 2026 22:53:52 +0800 Subject: [PATCH] wal: reduce per-record allocations in the decoder WAL replay at startup (Open + ReadAll) decodes every record since the last snapshot in one burst. Until now decodeRecord allocated a record-sized buffer per record (make([]byte, recBytes+padBytes)), so for a multi-GB WAL, replay produced double the garbage: one frame buffer plus proto's copy of the record for every entry. readInt64 also paid binary.Read's interface conversion and type-switch dispatch for every record. Two changes, mirroring cockroachdb/pebble@8ca7bf3 ("Reduce VersionEdit.Decode allocations"): - Reuse a scratch buffer on the decoder, grown on demand to the largest record seen and never shrunk. Reuse is safe: no reference to the buffer (or a subslice) escapes decodeRecord, because proto.Unmarshal copies the record bytes out of it and this package contains no unsafe unmarshal. isTornEntry and the CRC computation only use the buffer within the current call. - readInt64 reads 8 bytes and decodes via binary.LittleEndian.Uint64. It is equivalent to binary.Read(r, binary.LittleEndian, &n) with identical error semantics (io.EOF when no bytes were read, io.ErrUnexpectedEOF on a partial read), so end-of-file, preallocated-space, and torn-write detection are unchanged. Adds decoder benchmarks: the raw record decode loop over a WAL segment and the full Open + ReadAll + Close replay, each with 64B/512B/4KB entries. go test ./server/storage/wal/... passes, also with -race, including torn-write, CRC-mismatch, repair, and continueOnCrcError coverage. ## Results (Sep 4, 2026) Setup: 10,000-entry single-segment WAL per benchmark; the decode benchmark reuses one open segment file and replays NewDecoder + Decode until EOF; the ReadAll benchmark does a full Open + ReadAll + Close per iteration. go1.26.7, darwin/arm64, Apple M4, benchstat, n=6 (decode) and n=10 (ReadAll). "~" marks differences that are not statistically significant (p>0.05) on this host; all other deltas have p<=0.023. ### Decode only (lower is better) | Benchmark | sec/op | B/op | allocs/op | |---|---|---|---| | DecoderDecodeEntry64B | 1.387m -> 1.176m (**-15.2%**) | 1957.8Ki -> 1020.4Ki (**-47.9%**) | 50.02k -> 40.02k (**-20.0%**) | | DecoderDecodeEntry512B | 3.362m -> 2.700m (**-19.7%**) | 11.220Mi -> 5.727Mi (**-49.0%**) | 50.02k -> 40.02k (**-20.0%**) | | DecoderDecodeEntry4KB | 18.90m -> 15.02m (**-20.5%**) | 93.01Mi -> 46.63Mi (**-49.9%**) | 50.03k -> 40.02k (**-20.0%**) | Throughput: +17.9% / +24.5% / +25.8% (64B / 512B / 4KB). ### Full replay, Open + ReadAll + Close (lower is better) | Benchmark | sec/op | B/op | allocs/op | |---|---|---|---| | WALReadAllEntry64B | 60.99m -> 59.99m (~) | 5.026Mi -> 4.111Mi (**-18.2%**) | 100.12k -> 90.12k (**-10.0%**) | | WALReadAllEntry512B | 62.48m -> 58.66m (~, p=0.063) | 18.61Mi -> 13.11Mi (**-29.5%**) | 100.12k -> 90.12k (**-10.0%**) | | WALReadAllEntry4KB | 64.23m -> 59.00m (**-8.1%**) | 134.57Mi -> 88.19Mi (**-34.5%**) | 100.12k -> 90.12k (**-10.0%**) | Reading the numbers: the raw decode loop is 15-20% faster and its heap allocation is halved, one frame-buffer allocation saved per record. In the full replay the decode step is only one component (entry unmarshal and file open/close dominate), so end-to-end time improves 8.1% for 4KB entries and is within noise for smaller ones, but the allocation burst during recovery drops 18-35%, from about 2.3x to 1.2x the WAL payload size, which lowers GC pressure during the most latency-sensitive phase of a member restart. Signed-off-by: Gyuho Lee --- server/storage/wal/decoder.go | 35 +++++- server/storage/wal/decoder_bench_test.go | 133 +++++++++++++++++++++++ 2 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 server/storage/wal/decoder_bench_test.go diff --git a/server/storage/wal/decoder.go b/server/storage/wal/decoder.go index b23dfb3d48e..32d14d46c01 100644 --- a/server/storage/wal/decoder.go +++ b/server/storage/wal/decoder.go @@ -51,6 +51,14 @@ type decoder struct { lastValidOff int64 crc hash.Hash32 + // scratch is a buffer reused across Decode calls to avoid allocating a + // fresh buffer for every record. Reuse is safe: no reference to it (or a + // subslice) escapes decodeRecord, because proto.Unmarshal copies the + // record bytes out of it (there is no unsafe unmarshal in this package). + // The buffer grows to the largest record decoded and is retained for the + // lifetime of the decoder. + scratch []byte + // continueOnCrcError - causes the decoder to continue working even in case of crc mismatch. // This is a desired mode for tools performing inspection of the corrupted WAL logs. // See comments on 'Decode' method for semantic. @@ -113,7 +121,19 @@ func (d *decoder) decodeRecord(rec *walpb.Record) error { io.ErrUnexpectedEOF, fileBufReader.FileInfo().Name(), recBytes, fileBufReader.FileInfo().Size(), d.lastValidOff, padBytes, maxEntryLimit) } - data := make([]byte, recBytes+padBytes) + // Reuse the decoder's scratch buffer instead of allocating a + // record-sized buffer per record. WAL replay at startup reads every + // record in one burst, so a per-record allocation here doubles the + // garbage produced during recovery (this frame buffer plus proto's + // copy of the record). Reuse is safe because nothing aliases the + // buffer beyond the current call; see the comment on the scratch + // field. The buffer is grown on demand to the largest record seen and + // never shrinks. + bufLen := int(recBytes + padBytes) + if cap(d.scratch) < bufLen { + d.scratch = make([]byte, bufLen) + } + data := d.scratch[:bufLen] if _, err = io.ReadFull(fileBufReader, data); err != nil { // ReadFull returns io.EOF only if no bytes were read // the decoder should treat this as an ErrUnexpectedEOF instead. @@ -224,8 +244,15 @@ func MustUnmarshalState(d []byte) *raftpb.HardState { return &s } +// readInt64 reads a little-endian int64. It is equivalent to +// binary.Read(r, binary.LittleEndian, &n), but avoids the interface +// conversions and type-switch dispatch inside binary.Read — this runs once +// per WAL record during replay. Error semantics are identical: io.EOF when +// no bytes were read, io.ErrUnexpectedEOF on a partial read. func readInt64(r io.Reader) (int64, error) { - var n int64 - err := binary.Read(r, binary.LittleEndian, &n) - return n, err + var b [8]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return 0, err + } + return int64(binary.LittleEndian.Uint64(b[:])), nil } diff --git a/server/storage/wal/decoder_bench_test.go b/server/storage/wal/decoder_bench_test.go new file mode 100644 index 00000000000..a7a55355a3e --- /dev/null +++ b/server/storage/wal/decoder_bench_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wal + +import ( + "io" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "go.etcd.io/etcd/client/pkg/v3/fileutil" + "go.etcd.io/etcd/server/v3/storage/wal/walpb" + "go.etcd.io/raft/v3/raftpb" +) + +const benchEntryCount = 10000 + +// writeBenchWAL creates a WAL with benchEntryCount entries of entrySize bytes each. +func writeBenchWAL(tb testing.TB, dir string, entrySize int) { + w, err := Create(zaptest.NewLogger(tb), dir, []byte("metadata")) + require.NoError(tb, err) + data := make([]byte, entrySize) + for i := range data { + data[i] = byte(i) + } + for i := 0; i < benchEntryCount; i++ { + e := &raftpb.Entry{Term: new(uint64(1)), Index: new(uint64(i + 1)), Type: new(raftpb.EntryNormal), Data: data} + require.NoError(tb, w.saveEntry(e)) + } + require.NoError(tb, w.sync()) + require.NoError(tb, w.Close()) +} + +func benchWALFiles(tb testing.TB, dir string) []string { + ents, err := os.ReadDir(dir) + require.NoError(tb, err) + var files []string + for _, e := range ents { + if filepath.Ext(e.Name()) == ".wal" { + files = append(files, filepath.Join(dir, e.Name())) + } + } + sort.Strings(files) + require.NotEmpty(tb, files) + return files +} + +// benchmarkDecoderDecode measures the raw record decode loop (NewDecoder + Decode +// until EOF) over an existing WAL segment. This is the decode path exercised +// during WAL replay at startup. +func benchmarkDecoderDecode(b *testing.B, entrySize int) { + p := b.TempDir() + writeBenchWAL(b, p, entrySize) + files := benchWALFiles(b, p) + require.Len(b, files, 1) + + f, err := os.Open(files[0]) + require.NoError(b, err) + defer f.Close() + + b.SetBytes(int64(entrySize * benchEntryCount)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := f.Seek(0, io.SeekStart); err != nil { + b.Fatal(err) + } + dec := NewDecoder(fileutil.NewFileReader(f)) + rec := &walpb.Record{} + n := 0 + for { + err := dec.Decode(rec) + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + n++ + } + if n < benchEntryCount { + b.Fatalf("decoded %d records, want at least %d", n, benchEntryCount) + } + } +} + +// benchmarkWALReadAll measures the full replay path used at node startup: +// Open WAL + ReadAll + Close. +func benchmarkWALReadAll(b *testing.B, entrySize int) { + p := b.TempDir() + writeBenchWAL(b, p, entrySize) + lg := zaptest.NewLogger(b) + snap := &walpb.Snapshot{} + + b.SetBytes(int64(entrySize * benchEntryCount)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + w, err := Open(lg, p, snap) + require.NoError(b, err) + _, _, ents, err := w.ReadAll() + if err != nil { + b.Fatal(err) + } + if len(ents) != benchEntryCount { + b.Fatalf("got %d entries, want %d", len(ents), benchEntryCount) + } + require.NoError(b, w.Close()) + } +} + +func BenchmarkDecoderDecodeEntry64B(b *testing.B) { benchmarkDecoderDecode(b, 64) } +func BenchmarkDecoderDecodeEntry512B(b *testing.B) { benchmarkDecoderDecode(b, 512) } +func BenchmarkDecoderDecodeEntry4KB(b *testing.B) { benchmarkDecoderDecode(b, 4096) } +func BenchmarkWALReadAllEntry64B(b *testing.B) { benchmarkWALReadAll(b, 64) } +func BenchmarkWALReadAllEntry512B(b *testing.B) { benchmarkWALReadAll(b, 512) } +func BenchmarkWALReadAllEntry4KB(b *testing.B) { benchmarkWALReadAll(b, 4096) }