benchmark: compile time & serialization vs. popular Scala libraries

.github/workflows/ci.yml

@@ -16,7 +16,9 @@ jobs:
         uses: coursier/cache-action@v8.1
       - uses: VirtusLab/scala-cli-setup@v1
       - name: Check formatting
-        run: scala-cli --power fmt --check .
+        # benchmark/ is a separate scala-cli build (competitor deps, JMH) — its
+        # own CI is .github/workflows/benchmark.yml.
+        run: scala-cli --power fmt --check . --exclude benchmark
 
   test:
     name: Test
@@ -38,7 +40,7 @@ jobs:
       - name: Run tests
         # Coverage is recorded only for production sources; tests were previously
         # measured as 100%-covered and inflated the reported number.
-        run: scala-cli --power test . -O -coverage-out:./.scoverage -O '-coverage-exclude-files:.*test/.*'
+        run: scala-cli --power test . --exclude benchmark -O -coverage-out:./.scoverage -O '-coverage-exclude-files:.*test/.*'
       - name: Generate coverage
         run: scala-cli .scoverage/report.sc
       - uses: codecov/codecov-action@v7
@@ -47,7 +49,7 @@ jobs:
           files: .scoverage/report/cobertura.xml
           token: ${{ secrets.CODECOV_TOKEN }}
       - name: Test documentation
-        run: scala-cli --power doc .
+        run: scala-cli --power doc . --exclude benchmark
 
   compile:
     name: Compile (${{ matrix.platform }})
@@ -75,7 +77,7 @@ jobs:
             scala-build-${{ matrix.platform }}-
       - uses: VirtusLab/scala-cli-setup@v1
       - name: Compile
-        run: scala-cli --power compile . --platform ${{ matrix.platform }} ${{ matrix.args }}
+        run: scala-cli --power compile . --exclude benchmark --platform ${{ matrix.platform }} ${{ matrix.args }}
 
   # JVM is covered by the `test` job above (with coverage/docs); this just runs
   # the same suite on the other targets to catch platform-specific regressions.
@@ -103,4 +105,4 @@ jobs:
             scala-build-${{ matrix.platform }}-
       - uses: VirtusLab/scala-cli-setup@v1
       - name: Run tests
-        run: scala-cli --power test . --platform ${{ matrix.platform }} ${{ matrix.args }}
+        run: scala-cli --power test . --exclude benchmark --platform ${{ matrix.platform }} ${{ matrix.args }}

.github/workflows/docs.yaml

@@ -39,7 +39,7 @@ jobs:
           # name should differ from the repo slug, e.g. made -> "M&DE".
           PROJECT_NAME: ${{ vars.PROJECT_NAME || github.event.repository.name }}
         run: |
-          scala-cli --power doc . -- \
+          scala-cli --power doc . --exclude benchmark -- \
           -project "$PROJECT_NAME" \
           -project-version ${{ github.ref_name }} \
           -project-footer "made with ❤️ and coffee" \

.github/workflows/mima.yml

@@ -45,8 +45,8 @@ jobs:
           org=$(grep -oP '(?<=using publish.organization ).*' project.scala | tr -d '"')
           name=$(grep -oP '(?<=using publish.name ).*' project.scala | tr -d '"')
 
-          scala-cli --power package . --library -o "$RUNNER_TEMP/new.jar" --force
-          shared_cp=$(scala-cli --power compile . --print-class-path | tail -1)
+          scala-cli --power package . --exclude benchmark --library -o "$RUNNER_TEMP/new.jar" --force
+          shared_cp=$(scala-cli --power compile . --exclude benchmark --print-class-path | tail -1)
 
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             base_sha=$(git rev-parse HEAD^1)

.github/workflows/publish.yml

@@ -32,7 +32,7 @@ jobs:
           PGP_PRIVATE_KEY: ${{ secrets.PGP_PRIVATE_KEY }}
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
         run: |
-          scala-cli --power publish . --verbose --platform ${{ matrix.platform }} ${{ matrix.args }} \
+          scala-cli --power publish . --exclude benchmark --verbose --platform ${{ matrix.platform }} ${{ matrix.args }} \
             --user "env:SONATYPE_USERNAME" \
             --password "env:SONATYPE_PASSWORD" \
             --secret-key "env:PGP_PRIVATE_KEY" \
@@ -63,12 +63,12 @@ jobs:
           set -euo pipefail
 
           # 1. Reusable javadoc from JVM scaladoc (platform-agnostic API; JS scaladoc is broken).
-          scala-cli --power publish . --platform jvm --signer none \
+          scala-cli --power publish . --exclude benchmark --platform jvm --signer none \
             --publish-repository "$PWD/.jvmdoc"
           jdoc="$(find "$PWD/.jvmdoc" -name 'mcodec_3-*-javadoc.jar' | head -1)"
 
           # 2. JS artifacts (jar/sources/pom + checksums), unsigned and without a doc JAR.
-          scala-cli --power publish . --platform scala-js --js-version 1.22.0 \
+          scala-cli --power publish . --exclude benchmark --platform scala-js --js-version 1.22.0 \
             --doc=false --signer none --publish-repository "$PWD/.staging"
 
           # 3. Inject the reused javadoc into the mcodec_sjs1_3 coordinate.

.gitignore

@@ -10,3 +10,10 @@ scala-doc
 # coverage (keep the report generator script)
 .scoverage/*
 !.scoverage/report.sc
+
+# benchmark run artifacts — keep the tidy CSVs, drop the bulky raw output
+benchmark/results/*.json
+benchmark/results/compile-raw.csv
+benchmark/results-env.txt
+__pycache__/
+

README.md

@@ -92,11 +92,22 @@ val nullableInt: MCodec[Int | Null] = MCodec[Int].nullable
 ## Build
 
 ```sh
-scala-cli --power compile .
-scala-cli --power test .
+scala-cli --power compile . --exclude benchmark
+scala-cli --power test . --exclude benchmark
 scala-cli --power fmt .
 ```
 
+`--exclude benchmark` keeps the separate benchmark build (competitor deps, JMH)
+out of the library build — `scala-cli` has no directive for this.
+
+## Benchmarks
+
+[`docs/benchmarks.md`](docs/benchmarks.md) compares mcodec's **compile time** and
+**serialization throughput** against circe, jsoniter-scala, uPickle, zio-json,
+borer, play-json and AVSystem GenCodec (the design mcodec is modelled on). The
+suite lives in [`benchmark/`](benchmark/) (a separate scala-cli build) and is
+regenerated with `benchmark/scripts/run_all.sh`.
+
 ## Acknowledgements
 
 mcodec is inspired by the [**AVSystem commons**](https://github.com/AVSystem/scala-commons) by [**ghik

benchmark/README.md

@@ -0,0 +1,66 @@
+# benchmark/
+
+Comparative benchmarks for mcodec: **compile time** and **serialization
+throughput** against circe, jsoniter-scala, uPickle, zio-json, borer, play-json
+and GenCodec. Results and methodology live in
+[`../docs/benchmarks.md`](../docs/benchmarks.md).
+
+This directory is a **separate scala-cli build** — `//> using exclude` does not
+keep it out of `scala-cli .`, so the main CI/publish workflows pass
+`--exclude benchmark`. It is not published, and **there is no CI for it** — run it
+locally with `scripts/run_all.sh` when you want fresh numbers.
+
+`benchmark/gencodec/` is a **second, nested Scala 2.13 build** (AVSystem GenCodec
+has no Scala 3 release). Build the Scala 3 suite with
+`scala-cli --power compile benchmark --exclude gencodec` and that one with
+`scala-cli --power compile benchmark/gencodec`.
+
+## Layout
+
+```
+project.scala          scala 3.9.0 + JMH + every competitor library
+src/models/            shared, library-neutral data models + sample instances
+src/codecs/            per-library codec instances behind a common `JsonCodec` facade
+src/bench/             JMH state classes (library is a @Param)
+gencodec/              nested Scala 2.13 build — AVSystem GenCodec, runtime rows only
+compile/generate.py    emit a standalone single-library project of N derived codecs
+compile/bench_compile.py   clean-compile sweep over N, per library
+scripts/aggregate.py   JMH json + compile csv  ->  tidy csv + refreshed report tables
+scripts/plot.py        charts (uv + matplotlib)
+scripts/run_all.sh     the whole pipeline
+results/               generated csv/json (git-ignored)
+```
+
+## Prerequisites
+
+- `scala-cli` (JMH support is behind `--power`)
+- `python3` for the compile sweep and aggregation
+- `uv` (or a `matplotlib` on `PATH`) for charts — optional
+
+## Running
+
+```sh
+# mcodec is consumed as a normal dependency; publish the working tree first
+# (from the repo root):
+scala-cli --power publish local . --exclude benchmark --project-version 0.0.0-BENCH --doc=false
+
+# everything
+benchmark/scripts/run_all.sh full        # ~1h, the committed dataset
+benchmark/scripts/run_all.sh quick       # a few minutes, for a fast sanity pass
+
+# serialization only, some libraries
+scala-cli --power run benchmark --exclude gencodec --jmh -- 'CompanyBench.*' -p lib=mcodec,circe
+scala-cli --power run benchmark/gencodec --jmh -- 'bench.gencodec.*'   # GenCodec (Scala 2.13)
+
+# compile time only
+python3 benchmark/compile/bench_compile.py --libs mcodec,circe --sizes 0,10,50
+python3 benchmark/compile/bench_compile.py --smoke        # N=0,1 sanity check
+```
+
+## Adding a library
+
+1. `src/codecs/<Lib>Codecs.scala` — provide `JsonCodec[A]` for `Primitives`,
+   `Company`, `FeatureCollection`, `Batch` (skip what the library can't derive).
+2. Register it in `src/codecs/Codecs.scala` and add its id to `JsonCodec.Lib`.
+3. Add it to the `@Param` arrays in `src/bench/SerdeBench.scala`.
+4. Add a dependency + `_derive_line` case in `compile/generate.py`.

benchmark/compile/bench_compile.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Compile-time sweep: for each library and each model count N, measure a clean
+(no compilation server, no incremental) compile of N derived codecs.
+
+    bench_compile.py [--libs a,b,c] [--sizes 0,1,10,25,50,100] [--runs 3]
+                     [--profile-n 50] [--out DIR] [--smoke]
+
+Writes to <out> (default: benchmark/results):
+    compile-raw.csv      library,n,run,seconds,ok
+    compile.csv          library,n,mean_s,stdev_s,min_s,class_bytes,ok
+    compile-phases.csv   library,phase,seconds        (only for --profile-n)
+
+Absolute numbers include a fixed ~JVM+scalac startup cost (measured as the N=0
+row); the meaningful signal is the slope as N grows. Run on a quiet machine.
+"""
+import argparse
+import csv
+import json
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parent.parent
+GENERATE = HERE / "generate.py"
+DEFAULT_LIBS = ["mcodec", "circe", "jsoniter", "upickle", "zio-json", "borer", "play-json", "gencodec"]
+DEFAULT_SIZES = [0, 1, 10, 25, 50, 100]
+# -Yprofile-trace is Scala 3 only; gencodec (Scala 2.13) gets wall-time rows only.
+NO_PROFILE = {"gencodec"}
+
+
+def run(cmd, **kw):
+    return subprocess.run(cmd, capture_output=True, text=True, **kw)
+
+
+def generate(lib: str, n: int, dest: Path) -> None:
+    r = run([sys.executable, str(GENERATE), lib, str(n), str(dest)])
+    if r.returncode != 0:
+        raise RuntimeError(f"generate failed: {r.stderr}")
+
+
+def compile_once(src: Path, extra: list[str] | None = None) -> tuple[float, bool, str]:
+    # --server=false: no Bloop, no incremental compiler -> a clean measurement.
+    cmd = ["scala-cli", "--power", "compile", str(src), "--server=false"]
+    if extra:
+        cmd += extra
+    t0 = time.perf_counter()
+    r = run(cmd, cwd=REPO)
+    dt = time.perf_counter() - t0
+    return dt, r.returncode == 0, r.stdout + r.stderr
+
+
+def class_bytes(src: Path) -> int:
+    total = 0
+    for p in (src / ".scala-build").rglob("*.class"):
+        total += p.stat().st_size
+    return total
+
+
+def parse_trace(trace: Path) -> dict[str, float]:
+    """Sum scalac phase durations (seconds) from a -Yprofile-trace file.
+
+    dotty emits paired B/E events with `cat == "phase"`; ts is microseconds.
+    """
+    data = json.loads(trace.read_text())
+    events = data["traceEvents"] if isinstance(data, dict) else data
+    phases: dict[str, float] = {}
+    stacks: dict[str, list[tuple[str, float]]] = {}
+    for e in events:
+        if e.get("cat") != "phase":
+            continue
+        tid = e.get("tid", "?")
+        if e.get("ph") == "B":
+            stacks.setdefault(tid, []).append((e["name"], e["ts"]))
+        elif e.get("ph") == "E" and stacks.get(tid):
+            name, start = stacks[tid].pop()
+            phases[name] = phases.get(name, 0.0) + (e["ts"] - start) / 1e6
+    return phases
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--libs", default=",".join(DEFAULT_LIBS))
+    ap.add_argument("--sizes", default=",".join(map(str, DEFAULT_SIZES)))
+    ap.add_argument("--runs", type=int, default=3)
+    ap.add_argument("--profile-n", type=int, default=50)
+    ap.add_argument("--out", default=str(REPO / "benchmark" / "results"))
+    ap.add_argument("--smoke", action="store_true", help="sizes=0,1 runs=1, no profile")
+    args = ap.parse_args()
+
+    libs = args.libs.split(",")
+    sizes = [0, 1] if args.smoke else sorted(set(map(int, args.sizes.split(","))))
+    runs = 1 if args.smoke else args.runs
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    raw_rows, summary_rows, phase_rows = [], [], []
+    work = Path(tempfile.mkdtemp(prefix="mcodec-compile-"))
+    print(f"workdir: {work}")
+    try:
+        for lib in libs:
+            for n in sizes:
+                dest = work / f"{lib}-{n}"
+                generate(lib, n, dest)
+                # warm dependency resolution once (not timed)
+                compile_once(dest)
+                shutil.rmtree(dest / ".scala-build", ignore_errors=True)
+                times, ok_all = [], True
+                for k in range(runs):
+                    dt, ok, log = compile_once(dest)
+                    ok_all &= ok
+                    raw_rows.append([lib, n, k, f"{dt:.3f}", ok])
+                    if ok:
+                        times.append(dt)
+                    else:
+                        print(f"  !! {lib} n={n} run={k} FAILED\n{log[-800:]}")
+                    shutil.rmtree(dest / ".scala-build", ignore_errors=True)
+                cb = 0
+                if times:
+                    compile_once(dest)  # rebuild to measure class size
+                    cb = class_bytes(dest)
+                summary_rows.append([
+                    lib, n,
+                    f"{statistics.mean(times):.3f}" if times else "",
+                    f"{statistics.pstdev(times):.3f}" if len(times) > 1 else "0",
+                    f"{min(times):.3f}" if times else "",
+                    cb, ok_all,
+                ])
+                print(f"  {lib:10s} n={n:<4d} "
+                      f"{('%.2fs' % statistics.mean(times)) if times else 'ERR':>8s}  {cb} class bytes")
+
+            if not args.smoke and args.profile_n and lib not in NO_PROFILE:
+                dest = work / f"{lib}-{args.profile_n}-prof"
+                generate(lib, args.profile_n, dest)
+                trace = dest / "trace.json"
+                _, ok, log = compile_once(
+                    dest, ["-O", "-Yprofile-enabled", "-O", f"-Yprofile-trace:{trace}"])
+                if ok and trace.exists():
+                    for phase, secs in sorted(parse_trace(trace).items(), key=lambda x: -x[1]):
+                        phase_rows.append([lib, phase, f"{secs:.4f}"])
+                else:
+                    print(f"  (no profile trace for {lib})")
+
+        _write(out / "compile-raw.csv", ["library", "n", "run", "seconds", "ok"], raw_rows)
+        _write(out / "compile.csv",
+               ["library", "n", "mean_s", "stdev_s", "min_s", "class_bytes", "ok"], summary_rows)
+        if phase_rows:
+            _write(out / "compile-phases.csv", ["library", "phase", "seconds"], phase_rows)
+        print(f"\nwrote {out}/compile.csv"
+              + ("" if not phase_rows else f" + compile-phases.csv"))
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+
+def _write(path: Path, header: list[str], rows: list) -> None:
+    with path.open("w", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        w.writerows(rows)
+
+
+if __name__ == "__main__":
+    main()