Correctness, diagnostics & bench/coverage hardening (Zig-book adoption)

.github/workflows/bench.yml

@@ -0,0 +1,38 @@
+name: Bench
+
+# Benchmarks run nightly (and on demand), off the PR-blocking path: the harness
+# is noisy on shared runners, so we capture numbers for trend/regression review
+# rather than gate merges on them. Each run uploads the BENCH_JSON lines so a
+# committed baseline + diff can be layered on later.
+on:
+  schedule:
+    - cron: "0 7 * * *" # daily at 07:00 UTC
+  workflow_dispatch:
+
+# Least privilege: read the repo, upload artifacts via the Actions runtime token.
+permissions:
+  contents: read
+
+jobs:
+  bench:
+    name: Benchmarks (ReleaseFast)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+      - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
+        with:
+          version: "0.16.0"
+      - name: Run benchmarks
+        run: zig build bench | tee bench-results.txt
+      - name: Extract machine-readable results
+        run: grep '^BENCH_JSON' bench-results.txt > bench-results.jsonl || true
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: bench-results
+          path: |
+            bench-results.txt
+            bench-results.jsonl

.github/workflows/ci.yml

@@ -6,6 +6,11 @@ on:
   pull_request:
     branches: [main]
 
+# Least privilege: these jobs only read the repo (and upload artifacts via the
+# Actions runtime token, which needs no extra GITHUB_TOKEN scope).
+permissions:
+  contents: read
+
 jobs:
   test:
     name: Test / Zig ${{ matrix.zig-version }} / ${{ matrix.os }}
@@ -16,8 +21,10 @@ jobs:
         zig-version: ["0.16.0"]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: mlugg/setup-zig@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+      - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
         with:
           version: ${{ matrix.zig-version }}
       - name: Run unit tests
@@ -27,8 +34,10 @@ jobs:
     name: Format check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: mlugg/setup-zig@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+      - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
         with:
           version: "0.16.0"
       - name: Check formatting
@@ -41,9 +50,47 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: mlugg/setup-zig@v2
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+      - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
         with:
           version: "0.16.0"
       - name: Build library
         run: zig build
+
+  coverage:
+    name: Coverage (kcov)
+    runs-on: ubuntu-latest
+    # Informational: surfaces which branches the unit tests exercise. Not a
+    # required check -- a coverage dip should not block a merge on its own.
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+      - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
+        with:
+          version: "0.16.0"
+      - name: Install kcov
+        # kcov was dropped from Ubuntu's repos (gone in 24.04 noble) and ships
+        # no prebuilt binaries, so build a pinned release from source.
+        env:
+          KCOV_VERSION: v43
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            cmake g++ binutils-dev libcurl4-openssl-dev libdw-dev \
+            libiberty-dev zlib1g-dev libssl-dev
+          git clone --depth 1 --branch "$KCOV_VERSION" https://github.com/SimonKagstrom/kcov.git /tmp/kcov
+          cmake -S /tmp/kcov -B /tmp/kcov/build -DCMAKE_BUILD_TYPE=Release
+          cmake --build /tmp/kcov/build --parallel "$(nproc)"
+          sudo cmake --install /tmp/kcov/build
+      - name: Build unit-test binary
+        run: zig build install-test
+      - name: Run kcov
+        run: kcov --include-pattern=src/ --exclude-pattern=src/crypto/ kcov-out ./zig-out/bin/test
+      - name: Upload coverage report
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: coverage-report
+          path: kcov-out

.gitignore

@@ -11,6 +11,9 @@ zig-pkg/
 *.o
 *.s
 
+# Coverage output (make coverage / kcov)
+kcov-out/
+
 # Docs site build artifacts
 docs/node_modules/
 docs/.next/

Makefile

@@ -1,6 +1,7 @@
 ZIG ?= zig
+KCOV ?= kcov
 
-.PHONY: build test fmt lint ci integration-test bench bench-u256 bench-keccak clean
+.PHONY: build test fmt fmt-fix lint ci integration-test bench bench-u256 bench-keccak coverage docs clean
 
 ## Build the library (default)
 build:
@@ -18,6 +19,18 @@ fmt:
 fmt-fix:
 	$(ZIG) fmt src/ tests/
 
+## Measure unit-test code coverage with kcov (Linux; requires kcov installed).
+## Excludes vendored C/crypto so the report reflects the Zig SDK surface.
+coverage:
+	$(ZIG) build install-test
+	rm -rf kcov-out
+	$(KCOV) --include-pattern=src/ --exclude-pattern=src/crypto/ kcov-out ./zig-out/bin/test
+	@echo "Coverage report: kcov-out/index.html"
+
+## Generate API documentation into zig-out/docs
+docs:
+	$(ZIG) build docs
+
 ## Run fmt + test — everything CI checks locally (no Anvil required)
 lint: fmt test
 

bench/bench.zig

@@ -78,6 +78,9 @@ const Timer = struct {
 
 const BenchResult = struct {
     ns_per_op: u64,
+    /// Sample standard deviation of the per-batch ns/op measurements. Turns the
+    /// "within measurement noise" claims into data callers can actually check.
+    stddev_ns: u64,
     iters: u64,
 };
 
@@ -100,23 +103,53 @@ fn runBench(comptime func: fn () void) BenchResult {
         batch *= 2;
     }
 
-    // Measure: collect samples over BENCH_NS
+    // Measure: collect one ns/op sample per ~100ms batch over BENCH_NS. With a
+    // 2s window and ~100ms batches this is ~20 samples -- enough for a stddev.
+    var samples: [4096]f64 = undefined;
+    var n_samples: usize = 0;
     var total_iters: u64 = 0;
     timer.reset();
-
-    while (timer.read() < BENCH_NS) {
+    var elapsed: u64 = 0;
+    while (elapsed < BENCH_NS) {
+        const t0 = timer.read();
         for (0..batch) |_| func();
+        const t1 = timer.read();
+        elapsed = t1;
+        if (n_samples < samples.len) {
+            const batch_ns: f64 = @floatFromInt(t1 - t0);
+            samples[n_samples] = batch_ns / @as(f64, @floatFromInt(batch));
+            n_samples += 1;
+        }
         total_iters += batch;
     }
 
     const total_ns = timer.read();
     const ns_per_op = if (total_iters > 0) total_ns / total_iters else 0;
-    return .{ .ns_per_op = ns_per_op, .iters = total_iters };
+
+    // Sample standard deviation (Bessel-corrected) of the per-batch ns/op.
+    var stddev_ns: u64 = 0;
+    if (n_samples > 1) {
+        var mean: f64 = 0;
+        for (samples[0..n_samples]) |s| mean += s;
+        mean /= @floatFromInt(n_samples);
+        var acc: f64 = 0;
+        for (samples[0..n_samples]) |s| {
+            const d = s - mean;
+            acc += d * d;
+        }
+        const variance = acc / @as(f64, @floatFromInt(n_samples - 1));
+        stddev_ns = @intFromFloat(@round(@sqrt(variance)));
+    }
+
+    return .{ .ns_per_op = ns_per_op, .stddev_ns = stddev_ns, .iters = total_iters };
 }
 
 fn runAndPrint(comptime name: []const u8, comptime func: fn () void, stdout: anytype) !void {
     const result = runBench(func);
-    try stdout.print("{s:<34} {d:>9} ns {d:>14}\n", .{ name, result.ns_per_op, result.iters });
+    try stdout.print("{s:<34} {d:>9} ns {d:>9} ns {d:>14}\n", .{ name, result.ns_per_op, result.stddev_ns, result.iters });
+    // Machine-readable line for compare.sh and regression tooling. Filtered out
+    // of human views with `grep -v '^BENCH_JSON'`.
+    try stdout.print("BENCH_JSON|{{\"name\":\"{s}\",\"ns_per_op\":{d},\"stddev_ns\":{d},\"iters\":{d}}}\n", .{ name, result.ns_per_op, result.stddev_ns, result.iters });
 }
 
 // ============================================================================
@@ -496,8 +529,8 @@ pub fn main() !void {
     var w = std.Io.File.stdout().writerStreaming(std.Io.Threaded.global_single_threaded.io(), &out_buf);
     const stdout = &w.interface;
 
-    try stdout.print("\n{s:<34} {s:>12} {s:>14}\n", .{ "Benchmark", "ns/op", "iters" });
-    try stdout.print("{s}\n", .{"" ++ @as([64]u8, @splat('-'))});
+    try stdout.print("\n{s:<34} {s:>12} {s:>12} {s:>14}\n", .{ "Benchmark", "ns/op", "stddev", "iters" });
+    try stdout.print("{s}\n", .{"" ++ @as([72]u8, @splat('-'))});
 
     // Keccak256
     try runAndPrint("keccak256_empty", benchKeccakEmpty, stdout);

bench/compare.sh

@@ -19,7 +19,9 @@ echo ""
 # -- Step 1: Run eth-zig benchmarks --
 echo "[1/3] Running eth-zig benchmarks (ReleaseFast)..."
 ZIG_OUTPUT=$(cd "$ROOT_DIR" && zig build bench 2>&1)
-echo "$ZIG_OUTPUT"
+# `|| true`: grep exits 1 if every line were filtered, which would abort the
+# script under `set -o pipefail` even though the bench itself succeeded.
+echo "$ZIG_OUTPUT" | grep -v "^BENCH_JSON" || true
 echo ""
 
 # -- Step 2: Run alloy.rs benchmarks --

build.zig

@@ -39,6 +39,35 @@ pub fn build(b: *std.Build) void {
     const test_step = b.step("test", "Run unit tests");
     test_step.dependOn(&run_unit_tests.step);
 
+    // Install the unit-test binary so coverage tooling (kcov) can run it
+    // out-of-band: `zig build install-test` writes it to zig-out/bin/test.
+    const install_unit_tests = b.addInstallArtifact(unit_tests, .{});
+    const install_test_step = b.step("install-test", "Install the unit-test binary (for coverage tooling)");
+    install_test_step.dependOn(&install_unit_tests.step);
+
+    // Formatting check as a first-class build step (mirrors the CI fmt job and
+    // `make fmt`, so `zig build fmt-check` is self-describing without Make).
+    const fmt_check = b.addFmt(.{
+        .paths = &.{ "src", "tests", "bench", "build.zig" },
+        .check = true,
+    });
+    const fmt_step = b.step("fmt-check", "Check source formatting (zig fmt --check)");
+    fmt_step.dependOn(&fmt_check.step);
+
+    // API documentation: `zig build docs` emits the HTML docs generated from the
+    // public surface (src/root.zig) into zig-out/docs.
+    const docs_obj = b.addObject(.{
+        .name = "eth",
+        .root_module = eth_module,
+    });
+    const install_docs = b.addInstallDirectory(.{
+        .source_dir = docs_obj.getEmittedDocs(),
+        .install_dir = .prefix,
+        .install_subdir = "docs",
+    });
+    const docs_step = b.step("docs", "Generate API documentation into zig-out/docs");
+    docs_step.dependOn(&install_docs.step);
+
     // Integration tests (requires Anvil)
     const integration_tests = b.addTest(.{
         .root_module = b.createModule(.{