From 8af809484a56adc91943a52d74b63b889e5fe67b Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 11:28:26 -0700 Subject: [PATCH 1/3] fix: Workflow github --- .github/memory-benchmark.yml | 40 +++++++++ Makefile | 4 +- tests/perf/README.md | 10 +++ tests/perf/baseline.json | 158 +++++++++++++++++------------------ tests/perf/run_profile.py | 46 ++++++++++ 5 files changed, 178 insertions(+), 80 deletions(-) create mode 100644 .github/memory-benchmark.yml diff --git a/.github/memory-benchmark.yml b/.github/memory-benchmark.yml new file mode 100644 index 00000000..44d968f9 --- /dev/null +++ b/.github/memory-benchmark.yml @@ -0,0 +1,40 @@ +name: Memory benchmark + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + +permissions: + contents: read + +jobs: + memory-benchmark: + name: Memory benchmark + # Needs to match the arch the baseline was generated on. + runs-on: ubuntu-24.04-arm + if: | + contains(github.event.pull_request.labels.*.name, 'check-memory-benchmark') && + ( + github.event.pull_request.author_association == 'COLLABORATOR' || + github.event.pull_request.author_association == 'MEMBER' || + github.event.pull_request.author_association == 'OWNER' + ) + steps: + - uses: actions/checkout@v4 + + # Uses the Dockerfile environment for repeatable runs. + - name: Run memory benchmark + run: make memory-use-bench + + # Upload all three flamegraph views per scenario (peak/leaks/temporary). + - name: Upload flamegraph reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: memray-flamegraphs + path: tests/perf/reports/*.html + if-no-files-found: warn diff --git a/Makefile b/Makefile index 4070ff0e..4da1a66c 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,12 @@ MEMRAY_ITERATIONS ?= 100 MEMRAY_THRESHOLD ?= 1.1 SCENARIO ?= SCENARIO_ARG := $(if $(SCENARIO),--scenario $(SCENARIO),) +# In CI, use en vars to write the report to the job run +GH_SUMMARY_MOUNT := $(if $(GITHUB_STEP_SUMMARY),-v $(GITHUB_STEP_SUMMARY):$(GITHUB_STEP_SUMMARY),) .PHONY: memory-use-bench memory-use-bench: docker build -f tests/perf/Dockerfiles/$(PERF_ENV)-perf-Dockerfile -t c2pa-memray-$(PERF_ENV) . - docker run --rm -v $(PWD):/workspace -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS) + docker run --rm -v $(PWD):/workspace $(GH_SUMMARY_MOUNT) -e PYTHONPATH=/workspace/src -e PERF_ENV=$(PERF_ENV) -e MEMRAY_ITERATIONS=$(MEMRAY_ITERATIONS) -e MEMRAY_THRESHOLD=$(MEMRAY_THRESHOLD) -e GITHUB_TOKEN -e GITHUB_STEP_SUMMARY c2pa-memray-$(PERF_ENV) python -m tests.perf.run_profile $(SCENARIO_ARG) $(PERF_ARGS) @echo "" @echo "Reports written to tests/perf/reports/" @echo "Open tests/perf/reports/-{peak,leaks,temporary}.html in a browser" diff --git a/tests/perf/README.md b/tests/perf/README.md index 1f2ec022..52b12bad 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -67,6 +67,16 @@ The trailing `VAR=value` arguments (e.g. `PERF_ENV=ubuntu-24.04`, `PERF_ARGS=--u Reports are written to `tests/perf/reports/` on the local machine. Three HTML files per scenario, one per suffix (described below). Open any in a browser. After a run, the run also reports if the scenarios were or were not all within baseline threshold (baseline +10% memory use tolerance). +## Running in CI + +The `.github/workflows/memory-benchmark.yml` workflow runs the Docker-based benchmarks on a PR, but only when the PR has the `check-memory-benchmark` label. This runs `make memory-use-bench`, so: + +- A regression (peak or leaked > baseline +10%) makes the benchmark job exit non-zero. +- A values report table is written to the job's Step Summary. +- All three flamegraph HTML views per scenario are uploaded as the `memray-flamegraphs` artifact. + +The gate only acts as regression test once a `tests/perf/baseline.json` is committed on the branch. Without one, `run_profile.py` treats the run as baseline creation (exits 0, no gating). + ## Report views Each scenario produces three [memray flamegraphs](https://bloomberg.github.io/memray/flamegraph.html). All three are flamegraphs of the same run. They differ only in which allocations they count. diff --git a/tests/perf/baseline.json b/tests/perf/baseline.json index 302d648a..bd4265f9 100644 --- a/tests/perf/baseline.json +++ b/tests/perf/baseline.json @@ -2,139 +2,139 @@ "_meta": { "memray_version": "1.19.3", "python_version": "3.12.13", - "c2pa_native_version": "c2pa-v0.85.1", + "c2pa_native_version": "c2pa-v0.86.1", "iterations": 100, "perf_env": "python-3.12-slim", "arch": "aarch64" }, "reader_jpeg_legacy": { - "peak_bytes": 3814421, - "leaked_bytes": 3266116, - "total_allocations": 698899 + "peak_bytes": 3790506, + "leaked_bytes": 3337275, + "total_allocations": 694247 }, "reader_jpeg_with_context": { - "peak_bytes": 3822953, - "leaked_bytes": 3257471, - "total_allocations": 692953 + "peak_bytes": 3784932, + "leaked_bytes": 3331065, + "total_allocations": 688204 }, "reader_mp4": { - "peak_bytes": 4876441, - "leaked_bytes": 3257485, - "total_allocations": 2112991 + "peak_bytes": 4192391, + "leaked_bytes": 3331202, + "total_allocations": 1856171 }, "reader_wav": { - "peak_bytes": 5520266, - "leaked_bytes": 3267427, - "total_allocations": 400371 + "peak_bytes": 4440733, + "leaked_bytes": 3281502, + "total_allocations": 386809 }, "builder_sign_jpeg_legacy": { - "peak_bytes": 7695310, - "leaked_bytes": 3383623, - "total_allocations": 522425 + "peak_bytes": 7739038, + "leaked_bytes": 3438581, + "total_allocations": 515197 }, "builder_sign_jpeg_with_context": { - "peak_bytes": 7688236, - "leaked_bytes": 3376293, - "total_allocations": 516851 + "peak_bytes": 7732082, + "leaked_bytes": 3431373, + "total_allocations": 509328 }, "builder_sign_png_legacy": { - "peak_bytes": 7932767, - "leaked_bytes": 3383648, - "total_allocations": 1694629 + "peak_bytes": 7939922, + "leaked_bytes": 3401346, + "total_allocations": 1399159 }, "builder_sign_png_with_context": { - "peak_bytes": 7925490, - "leaked_bytes": 3376452, - "total_allocations": 1688908 + "peak_bytes": 7932212, + "leaked_bytes": 3393558, + "total_allocations": 1393313 }, "builder_sign_jpeg_parallel_split_pool": { - "peak_bytes": 45764159, - "leaked_bytes": 3818113, - "total_allocations": 528785 + "peak_bytes": 45802848, + "leaked_bytes": 3855293, + "total_allocations": 520103 }, "builder_sign_jpeg_parallel_split_barrier": { - "peak_bytes": 46225287, - "leaked_bytes": 3809216, - "total_allocations": 527412 + "peak_bytes": 45771974, + "leaked_bytes": 3853543, + "total_allocations": 519696 }, "builder_sign_png_parallel_split_pool": { - "peak_bytes": 46002549, - "leaked_bytes": 3817801, - "total_allocations": 1700731 + "peak_bytes": 46511069, + "leaked_bytes": 3831120, + "total_allocations": 1405010 }, "builder_sign_png_parallel_split_barrier": { - "peak_bytes": 45970433, - "leaked_bytes": 3812044, - "total_allocations": 1699396 + "peak_bytes": 45980602, + "leaked_bytes": 3826012, + "total_allocations": 1403879 }, "builder_sign_gif": { - "peak_bytes": 14544515, - "leaked_bytes": 3375865, - "total_allocations": 7183237 + "peak_bytes": 14556418, + "leaked_bytes": 3398315, + "total_allocations": 5573733 }, "builder_sign_heic": { - "peak_bytes": 4608484, - "leaked_bytes": 3376030, - "total_allocations": 771079 + "peak_bytes": 4624733, + "leaked_bytes": 3402824, + "total_allocations": 733409 }, "builder_sign_m4a": { - "peak_bytes": 18849082, - "leaked_bytes": 3376431, - "total_allocations": 2273497 + "peak_bytes": 18864770, + "leaked_bytes": 3402727, + "total_allocations": 2126542 }, "builder_sign_webp": { - "peak_bytes": 8900701, - "leaked_bytes": 3376432, - "total_allocations": 487683 + "peak_bytes": 8911369, + "leaked_bytes": 3397659, + "total_allocations": 473589 }, "builder_sign_avi": { - "peak_bytes": 7040387, - "leaked_bytes": 3376267, - "total_allocations": 40315553 + "peak_bytes": 7357514, + "leaked_bytes": 3610304, + "total_allocations": 34863789 }, "builder_sign_mp4": { - "peak_bytes": 6162851, - "leaked_bytes": 3376431, - "total_allocations": 1809672 + "peak_bytes": 6202809, + "leaked_bytes": 3426948, + "total_allocations": 1595487 }, "builder_sign_tiff": { - "peak_bytes": 13124728, - "leaked_bytes": 3376268, - "total_allocations": 5139967 + "peak_bytes": 13430248, + "leaked_bytes": 3661058, + "total_allocations": 5484643 }, "builder_sign_jpeg_parent_of": { - "peak_bytes": 14173992, - "leaked_bytes": 3377656, - "total_allocations": 1209933 + "peak_bytes": 14244270, + "leaked_bytes": 3457332, + "total_allocations": 1194872 }, "builder_sign_jpeg_component_of": { - "peak_bytes": 14175518, - "leaked_bytes": 3377891, - "total_allocations": 1232336 + "peak_bytes": 14246032, + "leaked_bytes": 3457636, + "total_allocations": 1217308 }, "builder_sign_jpeg_parent_and_component": { - "peak_bytes": 14530406, - "leaked_bytes": 3474418, - "total_allocations": 2160934 + "peak_bytes": 14602066, + "leaked_bytes": 3560736, + "total_allocations": 2137378 }, "builder_sign_jpeg_parent_and_component_mixed_mime": { - "peak_bytes": 14476171, - "leaked_bytes": 3378735, - "total_allocations": 2451587 + "peak_bytes": 14553513, + "leaked_bytes": 3464495, + "total_allocations": 2148041 }, "builder_sign_jpeg_two_components_same_mime": { - "peak_bytes": 14519270, - "leaked_bytes": 3473673, - "total_allocations": 2150782 + "peak_bytes": 14582525, + "leaked_bytes": 3560177, + "total_allocations": 2127097 }, "builder_sign_jpeg_two_components_mixed_mime": { - "peak_bytes": 14473127, - "leaked_bytes": 3377445, - "total_allocations": 2441195 + "peak_bytes": 14549485, + "leaked_bytes": 3462954, + "total_allocations": 2137855 }, "builder_sign_jpeg_archive_roundtrip": { - "peak_bytes": 14226832, - "leaked_bytes": 3426491, - "total_allocations": 1680290 + "peak_bytes": 14327685, + "leaked_bytes": 3527329, + "total_allocations": 1657777 } } \ No newline at end of file diff --git a/tests/perf/run_profile.py b/tests/perf/run_profile.py index 31593967..df469b47 100644 --- a/tests/perf/run_profile.py +++ b/tests/perf/run_profile.py @@ -169,6 +169,49 @@ def _fmt(n: int) -> str: return f"{n} B" +def _delta_pct(current: int, base: int) -> str: + """Signed percentage change vs baseline, or '-' when no baseline.""" + if not base: + return "-" + return f"{(current - base) / base * 100:+.1f}%" + + +def _write_github_summary(results: dict, baseline: dict) -> None: + """Append a values table to $GITHUB_STEP_SUMMARY when running in CI. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path or not results: + return + + lines = [ + "## Memory benchmark (memray)", + "", + f"Iterations: {ITERATIONS} · threshold: +{(THRESHOLD - 1) * 100:.0f}%" + f"{f' · env: {PERF_ENV}' if PERF_ENV else ''}", + "", + "| scenario | peak | leaked | allocs | peak Δ% | leaked Δ% | status |", + "|----------|------|--------|--------|---------|-----------|--------|", + ] + for name, m in results.items(): + b = baseline.get(name, {}) if baseline else {} + peak_base = b.get("peak_bytes", 0) + leaked_base = b.get("leaked_bytes", 0) + regressed = ( + (peak_base and m["peak_bytes"] > peak_base * THRESHOLD) + or (leaked_base and m["leaked_bytes"] > leaked_base * THRESHOLD) + ) + status = "REGRESSED" if regressed else "ok" + lines.append( + f"| {name} | {_fmt(m['peak_bytes'])} | {_fmt(m['leaked_bytes'])} " + f"| {m['total_allocations']} | {_delta_pct(m['peak_bytes'], peak_base)} " + f"| {_delta_pct(m['leaked_bytes'], leaked_base)} | {status} |" + ) + lines.append("") + + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + + def main() -> None: parser = argparse.ArgumentParser(description="c2pa-python memory profiler") parser.add_argument( @@ -274,6 +317,9 @@ def main() -> None: verb = "Updated" if baseline else "Created" print(f"\n{verb} baseline: {BASELINE_FILE}") + # Emit the report table to the PR's Step Summary in CI. + _write_github_summary(results, baseline) + if render_failures: print("\nFLAMEGRAPH RENDERS FAILED (capture + metrics still recorded):", file=sys.stderr) for r in render_failures: From d2277f035088136f2eb65fbeb510d05ef32cadba Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 11:32:18 -0700 Subject: [PATCH 2/3] fix: Put workflow file in proper lcoation --- .github/{ => workflows}/memory-benchmark.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/memory-benchmark.yml (100%) diff --git a/.github/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml similarity index 100% rename from .github/memory-benchmark.yml rename to .github/workflows/memory-benchmark.yml From b0099b2a35eca9eb77fc0ba2a425f823ff107728 Mon Sep 17 00:00:00 2001 From: Tania Mathern Date: Tue, 9 Jun 2026 12:40:37 -0700 Subject: [PATCH 3/3] fix: Workflow name --- .github/workflows/memory-benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml index 44d968f9..5232e1ee 100644 --- a/.github/workflows/memory-benchmark.yml +++ b/.github/workflows/memory-benchmark.yml @@ -1,4 +1,4 @@ -name: Memory benchmark +name: Python SDK memray memory benchmark on: pull_request: @@ -13,7 +13,7 @@ permissions: jobs: memory-benchmark: - name: Memory benchmark + name: Python SDK memray memory benchmark # Needs to match the arch the baseline was generated on. runs-on: ubuntu-24.04-arm if: | @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v4 # Uses the Dockerfile environment for repeatable runs. - - name: Run memory benchmark + - name: Run memray memory benchmark run: make memory-use-bench # Upload all three flamegraph views per scenario (peak/leaks/temporary).