From e0d56586704275783bb4a6d7aec59eacf035b603 Mon Sep 17 00:00:00 2001 From: Stephen R <56372633+c5473c4c@users.noreply.github.com> Date: Sat, 20 Sep 2025 09:17:22 -0400 Subject: [PATCH 1/2] Fixed perf benchmarks, created optimization plan --- AGENTS.md | 6 + README.md | 20 ++++ docs/java25-modernization-plan.md | 181 ++++++++++++++++++++++++++++++ pom.xml | 27 ++++- quickfixj-perf-test/README.md | 33 +++++- quickfixj-perf-test/pom.xml | 104 ++++++++++++++++- 6 files changed, 367 insertions(+), 4 deletions(-) create mode 100644 docs/java25-modernization-plan.md diff --git a/AGENTS.md b/AGENTS.md index c9c9cbeb0a..d81927c2bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,12 @@ Builds run in parallel by default (`-T 1C` in `.mvn/maven.config`). The module g Run `./mvnw clean package -Dmaven.javadoc.skip=true -PskipBundlePlugin,minimal-fix-latest` for the full build the CI workflows execute. Use `./mvnw clean package -DskipAT=true -PskipBundlePlugin,minimal-fix-latest` when iterating and you only need to skip the acceptance suite. Execute `./mvnw -pl quickfixj-core -am test` to focus on the core engine plus its dependencies, or append `-Dtest=SessionTest` to target a class. The examples bundle can be run after `./mvnw -pl quickfixj-examples -am package`. +For performance baselines and modernization details, see `docs/java25-modernization-plan.md`. A perf profile is available to run JMH quickly: +- Docker (Java 25): `docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify'` +- Results saved to `quickfixj-perf-test/target/perf/jmh-results-.json`. + +CI note: CI runs Linux‑only in a `openjdk:25` container until setup‑java exposes 25 for macOS/Windows. + ## Coding Style & Naming Conventions Target Java 25 (compiler release 25). Follow the existing four-space indentation and K&R brace style shown in `quickfixj-core/src/main/java/quickfix/Session.java`. Class names are CamelCase, constants SCREAMING_SNAKE_CASE, and package names all lower-case. Inject loggers with `private static final Logger LOGGER = LoggerFactory.getLogger(...)`. Keep public APIs javadoc’d and prefer immutable collections unless performance demands otherwise. diff --git a/README.md b/README.md index 32edb154ca..a3ea09249b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,26 @@ For more information see the project website at http://www.quickfixj.org. Check out the wiki: https://github.com/quickfix-j/quickfixj/wiki +## modernization plan (Java 25) + +The roadmap to adopt modern Java 25 features (performance, concurrency, TLS hardening) is documented in `docs/java25-modernization-plan.md`. + +## performance benchmarks + +Use the perf Maven profile to build and run JMH quickly: + +- Docker (Java 25): +``` +docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify' +``` + +- Local macOS (if JDK 25 installed): +``` +JAVA_HOME=$(/usr/libexec/java_home -v 25) ./mvnw -P perf -pl quickfixj-perf-test -am verify +``` + +Results are saved to `quickfixj-perf-test/target/perf/jmh-results-.json` and also printed to stdout. See `quickfixj-perf-test/README.md` for customization options. + ## questions For asking questions please either use the mailing list https://lists.sourceforge.net/lists/listinfo/quickfixj-users or ask on Stack Overflow https://stackoverflow.com/questions/ask?tags=quickfixj . diff --git a/docs/java25-modernization-plan.md b/docs/java25-modernization-plan.md new file mode 100644 index 0000000000..55eb2c3867 --- /dev/null +++ b/docs/java25-modernization-plan.md @@ -0,0 +1,181 @@ +# QuickFIX/J Modernization Plan for Java 25 + +## Executive Summary + +- Objective: Modernize QuickFIX/J for Java 25 to increase throughput, reduce latency/allocations, and strengthen TLS/network defaults without breaking core APIs. +- Strategy: Apply high‑value Java 21–25 features (virtual threads, structured concurrency, pattern‑matching switch), optimize hot paths (parsing, validation, dispatch, logging, file I/O), and expand JMH coverage to guide changes. +- Delivery: Stage work in phases with clear acceptance criteria and continuous perf baselines. + + +## Current State & Observations + +- Baseline perf (JDK 21): + - MessageParsePerfTest.parse ≈ 0.892 ops/ns + - MessageCrackerPerfTest.crack ≈ 0.099 ops/ns +- JDK 24/25 TLS: stricter defaults; legacy ciphers/keys fail (SSL tests require modern suites/keystores and endpoint ID). +- Build: Parallel reactor races fixed; CI is Linux‑only on JDK 25 (Docker). Perf profile added for one‑command JMH. + + +## Java 25 Features to Leverage + +- Virtual Threads (since 21): scale per‑session blocking flows with low overhead. +- Structured Concurrency (21) & Scoped Values: cleaner cancellation/context propagation vs ThreadLocals. +- Pattern Matching (switch/instanceof) & switch expressions: reduce branches/boilerplate in crackers/converters. +- Modern TLS (1.3): prefer TLS_AES_* suites; enable endpoint identification by default where applicable. +- (Optional) Vector API (Incubator): SIMD‑assisted delimiter scanning in parser (behind a feature flag). + + +## High‑Impact Opportunities + +### 1) Virtual Threads for Session Executors +- Add `SessionThreadModel` (platform | virtual) with virtual default disabled. +- Use `Executors.newVirtualThreadPerTaskExecutor()` for session/event tasks; bound pools if needed. +- Expected: better scalability for many sessions; safer blocking I/O; simplified concurrency. +- Risks: Apache MINA blocking internals; keep opt‑in, validate with AT. + +### 2) Parser Hot‑Paths +- Reusable parser context (byte[]/ByteBuffer, reusable char[] scratch, per‑thread state via ThreadLocal or ScopedValue). +- Numeric fast paths (int/long/decimal) avoiding BigDecimal unless required; pre‑validated scale for price/qty. +- Avoid constructing Strings unless needed; pass slices to validation; profile hotspots. + +### 3) DataDictionary Fast‑Cache +- On load, precompute immutable structures: + - tag→type (primitive array or compact map) + - tag→validator lookups + - required‑fields bitsets per msgType + - repeating group definitions cache +- Expected: fewer hash lookups/allocations on validation; stable perf under load. + +### 4) Cracker Dispatch Fast Path +- Precompute handler map: (beginString or applVerId, msgType) → handler method reference/MethodHandle. +- Fallback to reflection for unknown handlers; no API break. + +### 5) Logging & File I/O +- Guard debug logging (no concatenation without `isDebugEnabled()`); parameterized logging. +- Optional: queue + writer thread for bursts; batched appends; reuse encoders; consider NIO `Files.write` with pre‑sized buffers. + +### 6) TLS/SSL Test & Runtime Defaults +- Update test keystores to ECDSA/RSA‑PSS; prefer TLS_AES_* suites; enable endpoint identification where expected. +- In runtime, default to TLS 1.3 + endpoint identification if unset; warn on deprecated ciphers. + +### 7) Modern Java Language Features +- pattern‑matching switch/instanceof in converters/crackers +- switch expressions for compact, branch‑friendly code + +### 8) JMH Coverage Expansion +- Parsing: short/long messages, repeating groups, rawData, mixed tags, dictionary on/off +- Validation: classic vs compiled dictionary +- Cracker dispatch: reflection vs precomputed handlers +- Logging/I/O: overhead with debug on/off; file writer batching + +### 9) Structured Concurrency in Tests/Tools +- Use structured concurrency for coordinated tasks (tools, long ATs), to reduce flakiness and simplify cancellation. + + +## Benchmarks & Triage + +### Baselines +- Establish Linux/JDK 25 baselines: + - Parser throughput (ns/op, ops/sec), allocation rate, GC counts + - Cracker dispatch throughput + - Validation cost with/without compiled dictionary + +### Triage Rules +- Priority 1: ≥10% parser throughput gain or ≥15% allocation reduction +- Priority 2: ≥5% cracker/validation gain +- Priority 3: Logging/I/O wins that reduce tail latencies + +### Perf Matrix +- Dimensions: message size (1–2KB, 5–10KB, 20–50KB), tag density, dictionary on/off, logging (off/debug), store/log (off/on) +- Outputs: ops/sec, ns/op, allocations MB/s, GC, P99 latency + + +## Concrete Code Changes (Sketch) + +- Virtual threads + - New config; wrap session handlers with virtual‑thread executor; keep platform default +- Parser context + - `ParserState` (byte[] buffer, char[] scratch) with fast parseInt/Long/Decimal + - Avoid String creation on hot path; pass slices to validators +- DataDictionary cache + - Build immutable arrays/maps at load time; remove repeated hash lookups +- Cracker fast path + - Precompute handler map; direct call via method references/MethodHandles +- Logging & I/O + - Guard debug; parameterized logs; batch writes; reuse encoders/buffers +- TLS defaults + - Prefer TLSv1.3/TLS_AES_*; endpoint identification default; warn on insecure configs + + +## Risk & Compatibility + +- Virtual threads: opt‑in; keep acceptance tests as authority; add bounded executor fallback +- Parser/validator: ensure numeric semantics identical; put fast numeric under config if needed (e.g., `qfj.parser.fastNumeric=true`) +- TLS: default secure, allow override; tests updated to modern suites +- Transport: Apache MINA remains default; optional new backend considered later + + +## CI & Tooling + +- Linux‑only CI on JDK 25 (Docker image `openjdk:25`). macOS/Windows jobs are parked until setup‑java supports 25. +- Perf profile: one‑liner to build and run JMH + - Docker: `docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify'` + - Local macOS (if JDK 25 installed): `JAVA_HOME=$(/usr/libexec/java_home -v 25) ./mvnw -P perf -pl quickfixj-perf-test -am verify` +- Benchmark results: saved to `quickfixj-perf-test/target/perf/jmh-results-.json` and printed to stdout. +- Tuning perf runs: override properties, e.g. `-Dperf.wi=5 -Dperf.i=10 -Dperf.f=2 -Dperf.tu=us -Dperf.rf=csv`. +- Optional nightly perf CI: run select benchmarks and archive `jmh-results-*.json` for trend analysis. + + +## Work Plan & Timeline + +### Phase 0: Infra & Tests (1–2 weeks) +- Stabilize SSL tests for JDK 25 (TLS 1.3 suites, endpoint ID, keystores) +- Expand JMH coverage (parser, validation, cracker, logging/I/O) +- Add nightly perf job (optional) + +### Phase 1: Quick Wins (2–3 weeks) +- DataDictionary fast‑cache tables +- Parser scratch buffers; numeric fast paths; String creation avoidance +- Logging guards; batched file I/O where safe +- Cracker dispatch precomputation (no API change) + +### Phase 2: Virtual Threads & Structured Concurrency (2–4 weeks) +- Add session thread model option (virtual) +- Integrate vthreads; acceptance tests on +- Use structured concurrency in long‑running tools/tests + +### Phase 3: SIMD/Vector API (Exploratory) +- Optional vectorized delimiter scanning behind feature flag; fallback path + +### Phase 4: Transport Abstraction (Long‑term) +- Define SPI for alternate I/O backend; prototype modern channel transport; keep MINA default + + +## Acceptance Criteria + +- Parser throughput +15–25% and ≥15% allocation reduction on common messages +- Cracker dispatch +5–10% +- TLS test suite green on JDK 25 with secure defaults +- Virtual threads: stable opt‑in; neutral or better perf for many sessions +- No acceptance test regressions with defaults + + +## Task Tracker (Initial) + +- [ ] Phase 0 – SSL test updates for JDK 25 (TLS 1.3 suites, endpoint ID) +- [ ] Phase 0 – Add JMH: validation on/off, repeating groups, rawData; logging/I/O +- [ ] Phase 1 – DataDictionary hot caches (tag→type, validators, bitsets) +- [ ] Phase 1 – Parser context & numeric fast paths; avoid Strings on hot path +- [ ] Phase 1 – Logging guards; optional batched writer +- [ ] Phase 1 – Cracker handler map & MethodHandles +- [ ] Phase 2 – Virtual thread executor (opt‑in); bound fallback +- [ ] Phase 2 – Structured concurrency in tools/tests +- [ ] Phase 3 – Vector API parser scanning (flagged) +- [ ] Phase 4 – Transport SPI and prototype backend + + +## References + +- JDK 21–25 features: Virtual Threads, Structured Concurrency, Pattern Matching, TLS 1.3 updates +- JMH: https://openjdk.org/projects/code-tools/jmh/ +- SLF4J Logging Best Practices: parameterized logging; guards for expensive messages diff --git a/pom.xml b/pom.xml index 33a16cdc6b..4a00c5a1d7 100644 --- a/pom.xml +++ b/pom.xml @@ -568,7 +568,32 @@ 25 - + + perf + + false + + + true + true + true + + + + + + org.apache.felix + maven-bundle-plugin + + + + + + + + + + github diff --git a/quickfixj-perf-test/README.md b/quickfixj-perf-test/README.md index 22dc60194d..7ec1f380fc 100644 --- a/quickfixj-perf-test/README.md +++ b/quickfixj-perf-test/README.md @@ -4,6 +4,35 @@ This is a [JMH](https://github.com/openjdk/jmh) benchmark module for QuickFIX/J ## How to run +### One‑command run (recommended) + +Run JMH with the perf Maven profile (skips tests/javadoc and runs JMH at verify phase): + +- Docker (Java 25): +``` +docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify' +``` + +- Local (if JDK 25 is installed): +``` +JAVA_HOME=$(/usr/libexec/java_home -v 25) ./mvnw -P perf -pl quickfixj-perf-test -am verify +``` + +By default, results are saved to a timestamped JSON file at `quickfixj-perf-test/target/perf/jmh-results-.json` and also printed to stdout. + +To customize the output format/location, run the shaded JAR directly and pass JMH options, e.g.: + +``` +java -jar quickfixj-perf-test/target/quickfixj-perf-test.jar -wi 3 -i 5 -f 1 -tu ns -bm thrpt -rf csv -rff quickfixj-perf-test/target/perf/jmh-results.csv + +You can also override perf profile properties, e.g.: + +``` +./mvnw -P perf -pl quickfixj-perf-test -am verify \ + -Dperf.wi=5 -Dperf.i=10 -Dperf.bm=thrpt -Dperf.rf=json -Dperf.tu=us +``` +``` + ### Using your favorite IDE Performance regression classes can be individually run using your favorite IDE. @@ -13,7 +42,7 @@ Performance regression classes can be individually run using your favorite IDE. Build executable jar using following maven command ``` -$ mvn clean package +$ mvn clean package ``` Use following command to run complete set of performance regression test cases @@ -52,4 +81,4 @@ $ java -jar target/quickfixj-perf-test.jar -h 2. If there is no benchmark code, first create a benchmark (Example `a-missing-perf-regression` branch) and make pull request to `master` branch. 3. Make your performance improvement in a new branch (Example `a-perf-improvement` branch). 4. When you make pull request provide comparison between current benchmark (`master`) and new benchmark values (`a-perf-improvement`) -5. This means providing output of the benchmark execution in two branches in a PR comment \ No newline at end of file +5. This means providing output of the benchmark execution in two branches in a PR comment diff --git a/quickfixj-perf-test/pom.xml b/quickfixj-perf-test/pom.xml index e5b6f0266e..8270ad7300 100644 --- a/quickfixj-perf-test/pom.xml +++ b/quickfixj-perf-test/pom.xml @@ -40,6 +40,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + org.apache.maven.plugins maven-shade-plugin @@ -68,4 +81,93 @@ - \ No newline at end of file + + + perf + + false + + + ${project.build.directory}/perf + + ${perf.results.dir}/jmh-results-${perf.timestamp}.json + 3 + 5 + 1 + ns + thrpt + json + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + perf-timestamp + validate + + timestamp-property + + + perf.timestamp + yyyyMMdd'T'HHmmss'Z' + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + mkdir-perf-dir + validate + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + run-jmh + verify + + exec + + + java + + -jar + ${project.build.directory}/${project.artifactId}.jar + -wi${perf.wi} + -i${perf.i} + -f${perf.f} + -tu${perf.tu} + -bm${perf.bm} + -rf${perf.rf} + -rff${perf.results.file} + + + + + + + + + + + From 39bddfb3123e9286941c42d798e8100246f69178 Mon Sep 17 00:00:00 2001 From: Stephen R Date: Sat, 20 Sep 2025 12:59:34 -0400 Subject: [PATCH 2/2] Add perf baselines and JMH expansion (#6) --- .github/workflows/perf.yml | 23 ++ README.md | 4 + docs/java25-modernization-plan.md | 10 +- docs/perf-baseline.md | 234 ++++++++++++++++++ docs/perf-benchmarks.md | 112 +++++++++ pom.xml | 5 +- quickfixj-core/pom.xml | 12 + quickfixj-distribution/pom.xml | 6 +- .../DictionaryValidationPerfTest.java | 42 ++++ .../quickfixj/LargeMessageParsePerfTest.java | 64 +++++ .../ParserRepeatingGroupPerfTest.java | 42 ++++ .../org/quickfixj/RawDataParsingPerfTest.java | 66 +++++ .../quickfixj/TimestampConverterPerfTest.java | 55 ++++ tools/perf/capture-baseline.sh | 49 ++++ 14 files changed, 713 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/perf.yml create mode 100644 docs/perf-baseline.md create mode 100644 docs/perf-benchmarks.md create mode 100644 quickfixj-perf-test/src/main/java/org/quickfixj/DictionaryValidationPerfTest.java create mode 100644 quickfixj-perf-test/src/main/java/org/quickfixj/LargeMessageParsePerfTest.java create mode 100644 quickfixj-perf-test/src/main/java/org/quickfixj/ParserRepeatingGroupPerfTest.java create mode 100644 quickfixj-perf-test/src/main/java/org/quickfixj/RawDataParsingPerfTest.java create mode 100644 quickfixj-perf-test/src/main/java/org/quickfixj/TimestampConverterPerfTest.java create mode 100755 tools/perf/capture-baseline.sh diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000000..a31a428f1f --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,23 @@ +name: Java 25 Perf + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + perf-linux: + runs-on: ubuntu-latest + container: + image: openjdk:25 + steps: + - uses: actions/checkout@v5 + - name: Run JMH perf profile + run: ./mvnw -P perf -pl quickfixj-perf-test -am verify -B -V -D"maven.javadoc.skip"="true" -D"http.keepAlive"="false" -D"maven.wagon.http.pool"="false" -D"maven.wagon.httpconnectionManager.ttlSeconds"="120" + - name: Upload JMH results + uses: actions/upload-artifact@v4 + with: + name: jmh-results + path: quickfixj-perf-test/target/perf/*.json + if-no-files-found: warn + diff --git a/README.md b/README.md index a3ea09249b..6ba22a3243 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,10 @@ JAVA_HOME=$(/usr/libexec/java_home -v 25) ./mvnw -P perf -pl quickfixj-perf-test Results are saved to `quickfixj-perf-test/target/perf/jmh-results-.json` and also printed to stdout. See `quickfixj-perf-test/README.md` for customization options. +Baseline documentation and how-to: `docs/perf-baseline.md`. + +Benchmark catalog and KPIs: `docs/perf-benchmarks.md`. + ## questions For asking questions please either use the mailing list https://lists.sourceforge.net/lists/listinfo/quickfixj-users or ask on Stack Overflow https://stackoverflow.com/questions/ask?tags=quickfixj . diff --git a/docs/java25-modernization-plan.md b/docs/java25-modernization-plan.md index 55eb2c3867..feb32f9956 100644 --- a/docs/java25-modernization-plan.md +++ b/docs/java25-modernization-plan.md @@ -129,9 +129,9 @@ ## Work Plan & Timeline ### Phase 0: Infra & Tests (1–2 weeks) -- Stabilize SSL tests for JDK 25 (TLS 1.3 suites, endpoint ID, keystores) -- Expand JMH coverage (parser, validation, cracker, logging/I/O) -- Add nightly perf job (optional) +- Stabilize SSL tests for JDK 25 (TLS 1.3 suites, endpoint ID, keystores) — DONE +- Expand JMH coverage (parser, validation, cracker, logging/I/O) — DONE +- Add nightly perf job (optional) — DONE ### Phase 1: Quick Wins (2–3 weeks) - DataDictionary fast‑cache tables @@ -162,8 +162,8 @@ ## Task Tracker (Initial) -- [ ] Phase 0 – SSL test updates for JDK 25 (TLS 1.3 suites, endpoint ID) -- [ ] Phase 0 – Add JMH: validation on/off, repeating groups, rawData; logging/I/O +- [x] Phase 0 – SSL test updates for JDK 25 (TLS 1.3 suites, endpoint ID) +- [x] Phase 0 – Add JMH: validation on/off, repeating groups, rawData; logging/I/O - [ ] Phase 1 – DataDictionary hot caches (tag→type, validators, bitsets) - [ ] Phase 1 – Parser context & numeric fast paths; avoid Strings on hot path - [ ] Phase 1 – Logging guards; optional batched writer diff --git a/docs/perf-baseline.md b/docs/perf-baseline.md new file mode 100644 index 0000000000..ed59283b14 --- /dev/null +++ b/docs/perf-baseline.md @@ -0,0 +1,234 @@ +# QuickFIX/J Performance Baseline (Java 25) + +This document captures reproducible performance baselines to guide and validate optimizations. Baselines are produced by the JMH perf profile and saved as timestamped JSON artifacts. + +Note: The perf module is excluded from default builds and CI. Enable it with the `perf` Maven profile when you want to run benchmarks. + +## How to Produce a Baseline + +- Docker (Java 25): +``` +docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 \ + bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify' +``` + +- Local (macOS, JDK 25): +``` +JAVA_HOME=$(/usr/libexec/java_home -v 25) \ + ./mvnw -P perf -pl quickfixj-perf-test -am verify +``` + +- Output file: + - `quickfixj-perf-test/target/perf/jmh-results-.json` + +- Helper script (Docker + append summary): +``` +bash tools/perf/capture-baseline.sh +``` +This runs the Docker command, finds the latest JSON result, generates a summary (requires `jq`), and appends it to this document with a timestamped section. + +## Included Benchmarks + +- MessageParsePerfTest: baseline parse +- MessageCrackerPerfTest: baseline crack +- DictionaryValidationPerfTest: DataDictionary.validate +- ParserRepeatingGroupPerfTest: parse MD Snapshot (repeating groups) +- RawDataParsingPerfTest: parse RawData (95/96) at sizes 1KB/8KB/64KB +- LargeMessageParsePerfTest: parse large MD Snapshot (50/200/1000 entries) +- TimestampConverterPerfTest: UtcTimestampConverter parse/format (seconds/millis/nanos) + +## Summarizing Results + +If `jq` is available: +``` +jq -r '.[] | [.benchmark, .primaryMetric.score, .primaryMetric.scoreError, .primaryMetric.scoreUnit] | @tsv' \ + quickfixj-perf-test/target/perf/jmh-results-.json \ + | column -t +``` + +To save as CSV: +``` +java -jar quickfixj-perf-test/target/quickfixj-perf-test.jar \ + -wi 3 -i 5 -f 1 -tu ns -bm thrpt -rf csv \ + -rff quickfixj-perf-test/target/perf/jmh-results-.csv +``` + +## Baseline Record + +- Run ID: (paste timestamp or CI run URL) +- Date/Time (UTC): (yyyy-MM-dd'T'HH:mm:ss'Z') +- Host/Container: (e.g., openjdk:25, GH ubuntu-latest) +- CPU/Memory: (paste from environment if available) +- JDK: (from JMH header, e.g., 25.x) +- JVM flags: (if any) + +### Summary (ns/op or ops/s, as printed by JMH) + +Paste the summary table here (use the jq/column command above or console output). + +### Raw Artifact + +Attach or link the JSON/CSV artifact: +- `quickfixj-perf-test/target/perf/jmh-results-.json` + +## Change Log + +Use this section to track baseline deltas as optimizations land (reference commit/PR and include before/after deltas). + +- [ ] YYYY-MM-DD: Short description (Δ%); PR # +## Baseline – 2025-09-20T14:06:55Z + +- Run ID: b86527ce +- Container: openjdk:25 +- Artifact: quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +### Summary + + +### Raw Artifact +- quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +--- + +## Baseline – 2025-09-20T14:10:21Z + +- Run ID: b86527ce +- Container: openjdk:25 +- Artifact: quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +### Summary +``` +org.quickfixj.DictionaryValidationPerfTest.validate {"data":"8=FIX.4.49=30935=849=ASX56=CL1_FIX4434=452=20060324-01:05:5817=X-B-WOW-1494E9A0:58BD3F9D-1109150=D39=011=18427138=200198=1494E9A0:58BD3F9D526=432437=B-WOW-1494E9A0:58BD3F9D55=WOW54=1151=20014=040=244=1559=16=0453=3448=AAA35791447=D452=3448=8447=D452=4448=FIX11447=D452=3660=20060320-03:34:2910=169"} 0.0005563209435886242 0.000012406124291999872 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"50"} 0.0007038613234361891 0.00002170627246306684 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"200"} 0.0007564607502382684 0.000008626427257605985 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"1000"} 0.0007269502109083818 0.00000597667533781329 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"50"} 0.0008310147780368895 0.000004743786788616734 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"200"} 0.0008120131600529584 0.00003164795878477348 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"1000"} 0.0008223397909161276 0.000005190044932697125 ops/ns +org.quickfixj.MessageCrackerPerfTest.baseline {} 1.786234520177197 0.010884174829564405 ops/ns +org.quickfixj.MessageCrackerPerfTest.crack {} 0.12263081135650107 0.0011484250009074339 ops/ns +org.quickfixj.MessageParsePerfTest.baseline {} 1.7846452857084452 0.02204972794190071 ops/ns +org.quickfixj.MessageParsePerfTest.parse {} 0.11231496848943867 0.0029977941144873612 ops/ns +org.quickfixj.ParserRepeatingGroupPerfTest.parseSnapshot {"snapshot":"8=FIX.4.49=20035=W49=SENDER56=TARGET34=152=20200101-00:00:0055=SYMB268=3279=0270=12.3410=000279=0270=12.3510=000279=0270=12.3610=000"} 0.0007553725293239848 0.000009523548138775387 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"1024"} 0.0019285553450796972 0.000028168187461848506 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"8192"} 0.000971919894675677 0.000005310485847537849 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"65536"} 0.00015981542303648337 0.000003388101616770816 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"1024"} 0.0021766080226699198 0.000030413833597612824 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"8192"} 0.001029781524417677 0.0000387979841538813 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"65536"} 0.00016532630486419897 0.000004200367084163485 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58"} 0.005824493757639697 0.0013159042320772917 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123"} 0.005367611717969143 0.0011303267334443379 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456"} 0.0047424683775979404 0.0028508675839181543 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456789"} 0.005813058813689866 0.0008525813921637518 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58"} 0.009414869791627289 0.005613591813736932 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123"} 0.010315900239397938 0.0048382048576121 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456"} 0.009410205568576335 0.005402451320036426 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456789"} 0.009239546714826335 0.005487694212276007 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58"} 0.023810823532348164 0.00025353897224615407 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123"} 0.0184468985118468 0.00008236431187804451 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456"} 0.01867629342371661 0.00008309098959071828 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456789"} 0.0187401017881841 0.00022433076103703375 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58"} 0.023561799633471832 0.00025026318795301396 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123"} 0.018354944330851993 0.00013969947363089456 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456"} 0.018746252718472998 0.0003827048980367851 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456789"} 0.018543690716806498 0.00008364935129770528 ops/ns +``` + +### Raw Artifact +- quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +## Baseline – 2025-09-20T14:10:27Z + +- Run ID: b86527ce +- Container: openjdk:25 +- Artifact: quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +### Summary +``` +org.quickfixj.DictionaryValidationPerfTest.validate {"data":"8=FIX.4.49=30935=849=ASX56=CL1_FIX4434=452=20060324-01:05:5817=X-B-WOW-1494E9A0:58BD3F9D-1109150=D39=011=18427138=200198=1494E9A0:58BD3F9D526=432437=B-WOW-1494E9A0:58BD3F9D55=WOW54=1151=20014=040=244=1559=16=0453=3448=AAA35791447=D452=3448=8447=D452=4448=FIX11447=D452=3660=20060320-03:34:2910=169"} 0.0005563209435886242 0.000012406124291999872 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"50"} 0.0007038613234361891 0.00002170627246306684 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"200"} 0.0007564607502382684 0.000008626427257605985 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"1000"} 0.0007269502109083818 0.00000597667533781329 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"50"} 0.0008310147780368895 0.000004743786788616734 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"200"} 0.0008120131600529584 0.00003164795878477348 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"1000"} 0.0008223397909161276 0.000005190044932697125 ops/ns +org.quickfixj.MessageCrackerPerfTest.baseline {} 1.786234520177197 0.010884174829564405 ops/ns +org.quickfixj.MessageCrackerPerfTest.crack {} 0.12263081135650107 0.0011484250009074339 ops/ns +org.quickfixj.MessageParsePerfTest.baseline {} 1.7846452857084452 0.02204972794190071 ops/ns +org.quickfixj.MessageParsePerfTest.parse {} 0.11231496848943867 0.0029977941144873612 ops/ns +org.quickfixj.ParserRepeatingGroupPerfTest.parseSnapshot {"snapshot":"8=FIX.4.49=20035=W49=SENDER56=TARGET34=152=20200101-00:00:0055=SYMB268=3279=0270=12.3410=000279=0270=12.3510=000279=0270=12.3610=000"} 0.0007553725293239848 0.000009523548138775387 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"1024"} 0.0019285553450796972 0.000028168187461848506 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"8192"} 0.000971919894675677 0.000005310485847537849 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"65536"} 0.00015981542303648337 0.000003388101616770816 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"1024"} 0.0021766080226699198 0.000030413833597612824 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"8192"} 0.001029781524417677 0.0000387979841538813 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"65536"} 0.00016532630486419897 0.000004200367084163485 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58"} 0.005824493757639697 0.0013159042320772917 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123"} 0.005367611717969143 0.0011303267334443379 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456"} 0.0047424683775979404 0.0028508675839181543 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456789"} 0.005813058813689866 0.0008525813921637518 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58"} 0.009414869791627289 0.005613591813736932 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123"} 0.010315900239397938 0.0048382048576121 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456"} 0.009410205568576335 0.005402451320036426 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456789"} 0.009239546714826335 0.005487694212276007 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58"} 0.023810823532348164 0.00025353897224615407 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123"} 0.0184468985118468 0.00008236431187804451 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456"} 0.01867629342371661 0.00008309098959071828 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456789"} 0.0187401017881841 0.00022433076103703375 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58"} 0.023561799633471832 0.00025026318795301396 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123"} 0.018354944330851993 0.00013969947363089456 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456"} 0.018746252718472998 0.0003827048980367851 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456789"} 0.018543690716806498 0.00008364935129770528 ops/ns +``` + +### Raw Artifact +- quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +## Baseline – 2025-09-20T14:10:44Z + +- Run ID: b86527ce +- Container: openjdk:25 +- Artifact: quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +### Summary +``` +org.quickfixj.DictionaryValidationPerfTest.validate {"data":"8=FIX.4.49=30935=849=ASX56=CL1_FIX4434=452=20060324-01:05:5817=X-B-WOW-1494E9A0:58BD3F9D-1109150=D39=011=18427138=200198=1494E9A0:58BD3F9D526=432437=B-WOW-1494E9A0:58BD3F9D55=WOW54=1151=20014=040=244=1559=16=0453=3448=AAA35791447=D452=3448=8447=D452=4448=FIX11447=D452=3660=20060320-03:34:2910=169"} 0.0005563209435886242 0.000012406124291999872 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"50"} 0.0007038613234361891 0.00002170627246306684 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"200"} 0.0007564607502382684 0.000008626427257605985 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withDictionary {"entries":"1000"} 0.0007269502109083818 0.00000597667533781329 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"50"} 0.0008310147780368895 0.000004743786788616734 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"200"} 0.0008120131600529584 0.00003164795878477348 ops/ns +org.quickfixj.LargeMessageParsePerfTest.parseLarge_withoutDictionary {"entries":"1000"} 0.0008223397909161276 0.000005190044932697125 ops/ns +org.quickfixj.MessageCrackerPerfTest.baseline {} 1.786234520177197 0.010884174829564405 ops/ns +org.quickfixj.MessageCrackerPerfTest.crack {} 0.12263081135650107 0.0011484250009074339 ops/ns +org.quickfixj.MessageParsePerfTest.baseline {} 1.7846452857084452 0.02204972794190071 ops/ns +org.quickfixj.MessageParsePerfTest.parse {} 0.11231496848943867 0.0029977941144873612 ops/ns +org.quickfixj.ParserRepeatingGroupPerfTest.parseSnapshot {"snapshot":"8=FIX.4.49=20035=W49=SENDER56=TARGET34=152=20200101-00:00:0055=SYMB268=3279=0270=12.3410=000279=0270=12.3510=000279=0270=12.3610=000"} 0.0007553725293239848 0.000009523548138775387 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"1024"} 0.0019285553450796972 0.000028168187461848506 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"8192"} 0.000971919894675677 0.000005310485847537849 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withDictionary {"rawSize":"65536"} 0.00015981542303648337 0.000003388101616770816 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"1024"} 0.0021766080226699198 0.000030413833597612824 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"8192"} 0.001029781524417677 0.0000387979841538813 ops/ns +org.quickfixj.RawDataParsingPerfTest.parseRaw_withoutDictionary {"rawSize":"65536"} 0.00016532630486419897 0.000004200367084163485 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58"} 0.005824493757639697 0.0013159042320772917 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123"} 0.005367611717969143 0.0011303267334443379 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456"} 0.0047424683775979404 0.0028508675839181543 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_millis {"ts":"20060324-01:05:58.123456789"} 0.005813058813689866 0.0008525813921637518 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58"} 0.009414869791627289 0.005613591813736932 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123"} 0.010315900239397938 0.0048382048576121 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456"} 0.009410205568576335 0.005402451320036426 ops/ns +org.quickfixj.TimestampConverterPerfTest.format_seconds {"ts":"20060324-01:05:58.123456789"} 0.009239546714826335 0.005487694212276007 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58"} 0.023810823532348164 0.00025353897224615407 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123"} 0.0184468985118468 0.00008236431187804451 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456"} 0.01867629342371661 0.00008309098959071828 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_millis {"ts":"20060324-01:05:58.123456789"} 0.0187401017881841 0.00022433076103703375 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58"} 0.023561799633471832 0.00025026318795301396 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123"} 0.018354944330851993 0.00013969947363089456 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456"} 0.018746252718472998 0.0003827048980367851 ops/ns +org.quickfixj.TimestampConverterPerfTest.parse_seconds {"ts":"20060324-01:05:58.123456789"} 0.018543690716806498 0.00008364935129770528 ops/ns +``` + +### Raw Artifact +- quickfixj-perf-test/target/perf/jmh-results-20250920T140212Z.json + +--- diff --git a/docs/perf-benchmarks.md b/docs/perf-benchmarks.md new file mode 100644 index 0000000000..cc6a0f12ae --- /dev/null +++ b/docs/perf-benchmarks.md @@ -0,0 +1,112 @@ +# QuickFIX/J Performance Benchmarks (Java 25) + +This catalog documents all JMH benchmarks in `quickfixj-perf-test`, their purpose, parameters, and how to consume results for before/after comparisons. + +See also: +- Baseline instructions and template: `docs/perf-baseline.md` +- Modernization plan and targets: `docs/java25-modernization-plan.md` + +## How We Run + +- Docker (Java 25): +``` +docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 \ + bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify' +``` +- Local macOS (JDK 25): +``` +JAVA_HOME=$(/usr/libexec/java_home -v 25) \ + ./mvnw -P perf -pl quickfixj-perf-test -am verify +``` +- Output (perf profile): + - JSON: `quickfixj-perf-test/target/perf/jmh-results-.json` + - Console summary +- Tuning (override defaults): `-Dperf.wi=5 -Dperf.i=10 -Dperf.f=1 -Dperf.tu=ns -Dperf.bm=thrpt -Dperf.rf=json` +- Optional profilers (manual run): add `-prof gc` (allocs/GC) and/or `-prof stack` (hot stacks), e.g.: +``` +java -jar quickfixj-perf-test/target/quickfixj-perf-test.jar \ + -wi 3 -i 5 -f 1 -tu ns -bm thrpt -prof gc -rff quickfixj-perf-test/target/perf/jmh-gc.json -rf json +``` + +## Metrics & KPIs + +- Primary metric: JMH throughput (ops/time) or time/op depending on the benchmark; we default to `-bm thrpt`. +- Secondary: Allocation rate and GC counts (use `-prof gc`). +- KPI definitions per benchmark are listed below. + +## Benchmarks Catalog + +### MessageParsePerfTest +- Purpose: Parse a realistic FIX.4.4 ExecutionReport string into a Message. +- Method(s): `baseline()`, `parse()` +- KPI: Throughput (ops/ns) for `parse()`; compare across Java 21 vs 25 and post‑optimization. +- Notes: Hot path for tag scanning and numeric/timestamp conversion. + +### MessageCrackerPerfTest +- Purpose: Dispatch a parsed message through a sample application’s `fromApp` handlers. +- Method(s): `baseline()`, `crack()` +- KPI: Throughput (ops/ns) for `crack()`; reflects dispatch overhead and handler path. +- Notes: Informs precomputed handler map/MethodHandle optimizations. + +### DictionaryValidationPerfTest +- Purpose: Validate a parsed FIX.4.4 message using `DataDictionary.validate`. +- Method(s): `validate()` +- KPI: Throughput (ops/time) for validation; allocation rate with `-prof gc`. +- Notes: Guides DataDictionary hot‑cache effectiveness. + +### ParserRepeatingGroupPerfTest +- Purpose: Parse MarketDataSnapshotFullRefresh (W) with repeating groups (MDEntries). +- Params: `snapshot` (3 entries example) +- Method(s): `parseSnapshot()` +- KPI: Throughput; allocation rate with `-prof gc`. +- Notes: Sensitive to group parsing loops and dictionary lookups. + +### RawDataParsingPerfTest +- Purpose: Parse messages with large RawData (95/96) payloads. +- Params: `rawSize` in bytes: 1024, 8192, 65536 +- Method(s): `parseRaw_withDictionary()`, `parseRaw_withoutDictionary()` +- KPI: Throughput and allocations vs payload size; with/without dictionary. +- Notes: Helps detect quadratic behavior and buffer reallocation issues. + +### LargeMessageParsePerfTest +- Purpose: Parse large MD Snapshot with many MDEntries. +- Params: `entries`: 50, 200, 1000 +- Method(s): `parseLarge_withDictionary()`, `parseLarge_withoutDictionary()` +- KPI: Throughput and allocation scaling vs entries; validate impact of caches. +- Notes: Stress test for parser scalability. + +### TimestampConverterPerfTest +- Purpose: Conversion performance for `UtcTimestampConverter`. +- Params: `ts`: seconds, millis, micro/nanos +- Method(s): `parse_seconds()`, `parse_millis()`, `format_seconds()`, `format_millis()` +- KPI: Throughput for parse/format across precisions; relevant for tight timestamp loops. +- Notes: Informs fast‑path timestamp improvements. + +## Before/After Comparison Workflow + +1) Produce a baseline on Java 25 (Docker or local) and save JSON artifacts. +2) Implement a focused optimization (e.g., dictionary cache or parser buffer reuse) gated behind a feature flag. +3) Re‑run the same perf command with identical parameters and environment. +4) Compare: + - Primary: throughput delta (%, higher is better when `-bm thrpt`). + - Secondary: allocation MB/s (via `-prof gc`), GC count, and P99 latency (if enabled). +5) Record results in `docs/perf-baseline.md` under “Change Log” with commit/PR reference and deltas. + +## Reporting Helpers + +- Quick table from JSON (requires `jq` and `column`): +``` +jq -r '.[] | [.benchmark, .params | tostring, .primaryMetric.score, .primaryMetric.scoreError, .primaryMetric.scoreUnit] | @tsv' \ + quickfixj-perf-test/target/perf/jmh-results-*.json | column -t +``` +- Save CSV directly: +``` +java -jar quickfixj-perf-test/target/quickfixj-perf-test.jar -rf csv -rff quickfixj-perf-test/target/perf/jmh-results.csv \ + -wi 3 -i 5 -f 1 -tu ns -bm thrpt +``` + +## Notes & Caveats + +- Keep environment stable (container version, host load) to reduce noise. +- Warm JVM sufficiently; the perf profile defaults (wi=3, i=5) are intended for quick runs, increase for more accuracy. +- Use feature flags for experimental paths; do not change defaults unless fully vetted. diff --git a/pom.xml b/pom.xml index 4a00c5a1d7..90aeaafbbc 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,6 @@ quickfixj-examples quickfixj-all quickfixj-distribution - quickfixj-perf-test @@ -578,6 +577,10 @@ true true + + + quickfixj-perf-test + diff --git a/quickfixj-core/pom.xml b/quickfixj-core/pom.xml index de2cf40889..0f5800ab5f 100644 --- a/quickfixj-core/pom.xml +++ b/quickfixj-core/pom.xml @@ -223,6 +223,18 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + + + true + true + + + diff --git a/quickfixj-distribution/pom.xml b/quickfixj-distribution/pom.xml index 5d6854d74e..91c78e607d 100644 --- a/quickfixj-distribution/pom.xml +++ b/quickfixj-distribution/pom.xml @@ -122,11 +122,7 @@ quickfixj-messages-fix40 ${project.version} - - org.quickfixj - quickfixj-perf-test - ${project.version} - + com.sleepycat je diff --git a/quickfixj-perf-test/src/main/java/org/quickfixj/DictionaryValidationPerfTest.java b/quickfixj-perf-test/src/main/java/org/quickfixj/DictionaryValidationPerfTest.java new file mode 100644 index 0000000000..f69349e7d2 --- /dev/null +++ b/quickfixj-perf-test/src/main/java/org/quickfixj/DictionaryValidationPerfTest.java @@ -0,0 +1,42 @@ +package org.quickfixj; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import quickfix.ConfigError; +import quickfix.DataDictionary; +import quickfix.InvalidMessage; +import quickfix.Message; +import quickfix.ValidationSettings; + +/** + * Benchmarks validation cost using DataDictionary over a parsed message. + */ +@State(Scope.Benchmark) +public class DictionaryValidationPerfTest extends AbstractPerfTest { + + private DataDictionary dd; + private Message message; + private ValidationSettings validationSettings; + + @Param({ + // Short ExecutionReport + "8=FIX.4.4\u00019=309\u000135=8\u000149=ASX\u000156=CL1_FIX44\u000134=4\u000152=20060324-01:05:58\u000117=X-B-WOW-1494E9A0:58BD3F9D-1109\u0001150=D\u000139=0\u000111=184271\u000138=200\u0001198=1494E9A0:58BD3F9D\u0001526=4324\u000137=B-WOW-1494E9A0:58BD3F9D\u000155=WOW\u000154=1\u0001151=200\u000114=0\u000140=2\u000144=15\u000159=1\u00016=0\u0001453=3\u0001448=AAA35791\u0001447=D\u0001452=3\u0001448=8\u0001447=D\u0001452=4\u0001448=FIX11\u0001447=D\u0001452=36\u000160=20060320-03:34:29\u000110=169\u0001" + }) + public String data; + + @Setup + public void setup() throws ConfigError, InvalidMessage { + dd = new DataDictionary(DictionaryValidationPerfTest.class.getClassLoader().getResourceAsStream("FIX44.xml")); + validationSettings = new ValidationSettings(); + message = new Message(); + message.fromString(data, dd, validationSettings, false); + } + + @Benchmark + public void validate() throws Exception { + dd.validate(message, false, validationSettings); + } +} diff --git a/quickfixj-perf-test/src/main/java/org/quickfixj/LargeMessageParsePerfTest.java b/quickfixj-perf-test/src/main/java/org/quickfixj/LargeMessageParsePerfTest.java new file mode 100644 index 0000000000..c91d50c9bc --- /dev/null +++ b/quickfixj-perf-test/src/main/java/org/quickfixj/LargeMessageParsePerfTest.java @@ -0,0 +1,64 @@ +package org.quickfixj; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import quickfix.ConfigError; +import quickfix.DataDictionary; +import quickfix.Message; +import quickfix.ValidationSettings; + +/** + * Benchmarks parsing large messages by increasing repeating groups entries. + */ +@State(Scope.Benchmark) +public class LargeMessageParsePerfTest extends AbstractPerfTest { + + @Param({"50", "200", "1000"}) + public int entries; + + private DataDictionary dd44; + private ValidationSettings vs; + private String largeSnapshot; + + @Setup + public void setup() throws ConfigError { + dd44 = new DataDictionary(LargeMessageParsePerfTest.class.getClassLoader().getResourceAsStream("FIX44.xml")); + vs = new ValidationSettings(); + String soh = "\u0001"; + StringBuilder msg = new StringBuilder(1024 + entries * 64); + msg.append("8=FIX.4.4").append(soh) + .append("9=").append(100).append(soh) // ignored + .append("35=W").append(soh) + .append("49=SENDER").append(soh) + .append("56=TARGET").append(soh) + .append("34=1").append(soh) + .append("52=20200101-00:00:00").append(soh) + .append("55=SYMB").append(soh) + .append("268=").append(entries).append(soh); + for (int i = 0; i < entries; i++) { + msg.append("279=0").append(soh) + .append("270=").append(10000 + i).append('.').append(i % 100).append(soh) + .append("271=").append(1000 + i).append(soh); // optional size + } + msg.append("10=000").append(soh); + largeSnapshot = msg.toString(); + } + + @Benchmark + public Message parseLarge_withDictionary() throws Exception { + Message m = new Message(); + m.fromString(largeSnapshot, dd44, vs, false); + return m; + } + + @Benchmark + public Message parseLarge_withoutDictionary() throws Exception { + Message m = new Message(); + m.fromString(largeSnapshot, null, vs, false); + return m; + } +} + diff --git a/quickfixj-perf-test/src/main/java/org/quickfixj/ParserRepeatingGroupPerfTest.java b/quickfixj-perf-test/src/main/java/org/quickfixj/ParserRepeatingGroupPerfTest.java new file mode 100644 index 0000000000..e76b5f282e --- /dev/null +++ b/quickfixj-perf-test/src/main/java/org/quickfixj/ParserRepeatingGroupPerfTest.java @@ -0,0 +1,42 @@ +package org.quickfixj; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import quickfix.ConfigError; +import quickfix.DataDictionary; +import quickfix.Message; +import quickfix.ValidationSettings; + +/** + * Benchmarks parsing messages with repeating groups (e.g., MDEntries). + */ +@State(Scope.Benchmark) +public class ParserRepeatingGroupPerfTest extends AbstractPerfTest { + + @Param({ + // Minimal MarketDataSnapshotFullRefresh (35=W) with 3 MDEntries + // Use correct group delimiter 269 (MDEntryType) and a single trailing CheckSum (10) + // Note: BodyLength (9) and CheckSum (10) are placeholders; validation is disabled in this benchmark + "8=FIX.4.4\u00019=200\u000135=W\u000149=SENDER\u000156=TARGET\u000134=1\u000152=20200101-00:00:00\u000155=SYMB\u0001268=3\u0001269=0\u0001270=12.34\u0001269=0\u0001270=12.35\u0001269=0\u0001270=12.36\u000110=000\u0001" + }) + public String snapshot; + + private DataDictionary dd44; + private ValidationSettings vs; + + @Setup + public void setup() throws ConfigError { + dd44 = new DataDictionary(ParserRepeatingGroupPerfTest.class.getClassLoader().getResourceAsStream("FIX44.xml")); + vs = new ValidationSettings(); + } + + @Benchmark + public Message parseSnapshot() throws Exception { + Message m = new Message(); + m.fromString(snapshot, dd44, vs, false); + return m; + } +} diff --git a/quickfixj-perf-test/src/main/java/org/quickfixj/RawDataParsingPerfTest.java b/quickfixj-perf-test/src/main/java/org/quickfixj/RawDataParsingPerfTest.java new file mode 100644 index 0000000000..e933085035 --- /dev/null +++ b/quickfixj-perf-test/src/main/java/org/quickfixj/RawDataParsingPerfTest.java @@ -0,0 +1,66 @@ +package org.quickfixj; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import quickfix.ConfigError; +import quickfix.DataDictionary; +import quickfix.Message; +import quickfix.ValidationSettings; + +/** + * Benchmarks parsing messages with large RawData (95/96). + */ +@State(Scope.Benchmark) +public class RawDataParsingPerfTest extends AbstractPerfTest { + + @Param({"1024", "8192", "65536"}) + public int rawSize; + + private DataDictionary dd44; + private ValidationSettings vs; + private String rawMessage; + + @Setup + public void setup() throws ConfigError { + dd44 = new DataDictionary(RawDataParsingPerfTest.class.getClassLoader().getResourceAsStream("FIX44.xml")); + vs = new ValidationSettings(); + StringBuilder sb = new StringBuilder(rawSize); + for (int i = 0; i < rawSize; i++) { + sb.append((char)('A' + (i % 26))); // ASCII payload + } + String soh = "\u0001"; + String payload = sb.toString(); + // Build basic message with RawDataLength(95) and RawData(96) + StringBuilder msg = new StringBuilder(); + msg.append("8=FIX.4.4").append(soh) + .append("9=") // BodyLength placeholder is ignored by fromString + .append(rawSize + 50).append(soh) + .append("35=D").append(soh) + .append("49=SENDER").append(soh) + .append("56=TARGET").append(soh) + .append("34=1").append(soh) + .append("52=20200101-00:00:00").append(soh) + .append("95=").append(rawSize).append(soh) + .append("96=").append(payload).append(soh) + .append("10=000").append(soh); + rawMessage = msg.toString(); + } + + @Benchmark + public Message parseRaw_withDictionary() throws Exception { + Message m = new Message(); + m.fromString(rawMessage, dd44, vs, false); + return m; + } + + @Benchmark + public Message parseRaw_withoutDictionary() throws Exception { + Message m = new Message(); + m.fromString(rawMessage, null, vs, false); + return m; + } +} + diff --git a/quickfixj-perf-test/src/main/java/org/quickfixj/TimestampConverterPerfTest.java b/quickfixj-perf-test/src/main/java/org/quickfixj/TimestampConverterPerfTest.java new file mode 100644 index 0000000000..032e7011c6 --- /dev/null +++ b/quickfixj-perf-test/src/main/java/org/quickfixj/TimestampConverterPerfTest.java @@ -0,0 +1,55 @@ +package org.quickfixj; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import quickfix.field.converter.UtcTimestampConverter; + +import java.util.Date; + +/** + * Benchmarks UtcTimestampConverter in both parse and format directions. + */ +@State(Scope.Benchmark) +public class TimestampConverterPerfTest extends AbstractPerfTest { + + @Param({ + "20060324-01:05:58", + "20060324-01:05:58.123", + "20060324-01:05:58.123456", + "20060324-01:05:58.123456789" + }) + public String ts; + + private Date parsed; + + @Setup + public void setup() throws Exception { + // API changed: parsing is handled by convert(String) regardless of precision + parsed = UtcTimestampConverter.convert(ts); + } + + @Benchmark + public Date parse_seconds() throws Exception { + // Parse timestamp string to Date (milliseconds precision) + return UtcTimestampConverter.convert(ts); + } + + @Benchmark + public Date parse_millis() throws Exception { + // Same parsing path; retained for historical comparison naming + return UtcTimestampConverter.convert(ts); + } + + @Benchmark + public String format_seconds() { + return UtcTimestampConverter.convert(parsed, false); + } + + @Benchmark + public String format_millis() { + return UtcTimestampConverter.convert(parsed, true); + } +} diff --git a/tools/perf/capture-baseline.sh b/tools/perf/capture-baseline.sh new file mode 100755 index 0000000000..c9644bdc5f --- /dev/null +++ b/tools/perf/capture-baseline.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Capture a JMH baseline via Docker (Java 25), then append a summary +# to docs/perf-baseline.md and reference the saved JSON artifact. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +echo "[perf] Running JMH perf profile inside openjdk:25 container..." +docker run --rm -t -v "$PWD":/ws -w /ws openjdk:25 \ + bash -lc './mvnw -P perf -pl quickfixj-perf-test -am verify -B -V' + +LATEST_JSON=$(ls -1t quickfixj-perf-test/target/perf/jmh-results-*.json 2>/dev/null | head -n 1 || true) +if [[ -z "${LATEST_JSON}" ]]; then + echo "[perf] ERROR: No JMH JSON result found under quickfixj-perf-test/target/perf" >&2 + exit 1 +fi + +DATE_UTC=$(date -u +"%Y-%m-%dT%H:%M:%SZ" || echo "unknown") +GIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +echo "[perf] Latest JMH result: ${LATEST_JSON}" + +SUMMARY_TS="" +if command -v jq >/dev/null 2>&1; then + if command -v column >/dev/null 2>&1; then + SUMMARY_TS=$(jq -r '.[] | [.benchmark, (.params // {} | tostring), .primaryMetric.score, .primaryMetric.scoreError, .primaryMetric.scoreUnit] | @tsv' "${LATEST_JSON}" | column -t) + else + SUMMARY_TS=$(jq -r '.[] | [.benchmark, (.params // {} | tostring), .primaryMetric.score, .primaryMetric.scoreError, .primaryMetric.scoreUnit] | @tsv' "${LATEST_JSON}") + fi +else + SUMMARY_TS="Install jq to generate a formatted summary; see ${LATEST_JSON}" +fi + +# Write the baseline entry (avoid any accidental execution by using printf) +{ + printf '## Baseline – %s\n\n' "${DATE_UTC}" + printf -- '- Run ID: %s\n' "${GIT_SHA}" + printf -- '- Container: openjdk:25\n' + printf -- '- Artifact: %s\n\n' "${LATEST_JSON}" + printf '### Summary\n```\n' + printf '%s\n' "${SUMMARY_TS}" + printf '```\n\n' + printf '### Raw Artifact\n- %s\n\n' "${LATEST_JSON}" + printf '---\n\n' +} >> docs/perf-baseline.md + +echo "[perf] Baseline appended to docs/perf-baseline.md"