From 4ab1f55dba73da41b4d9d415c82291fb0737ea14 Mon Sep 17 00:00:00 2001 From: Valentin Maerten Date: Sat, 18 Jul 2026 16:05:30 +0200 Subject: [PATCH] perf: reuse buffer when hashing source files for checksums io.CopyBuffer ignored the pre-allocated buffer because *os.File implements WriterTo: it took the (*os.File).WriteTo path, which allocates a fresh 32KiB buffer per file via os.genericWriteTo. Hashing many small source files thus allocated ~32KiB per file (~640MB for 20,000 files). Wrapping the file in a plain io.Reader forces io.CopyBuffer to reuse the caller's buffer, keeping the loop allocation-free. The checksum value is unchanged. --- internal/fingerprint/sources_checksum.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/fingerprint/sources_checksum.go b/internal/fingerprint/sources_checksum.go index 3d34136989..23a29b51dc 100644 --- a/internal/fingerprint/sources_checksum.go +++ b/internal/fingerprint/sources_checksum.go @@ -88,6 +88,10 @@ func (*ChecksumChecker) Kind() string { return "checksum" } +// readerOnly hides any WriterTo/ReaderFrom implementation of the wrapped +// reader, forcing io.CopyBuffer to use the caller-provided buffer. +type readerOnly struct{ io.Reader } + func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) { sources, err := Globs(t.Dir, t.Sources, t.ShouldUseGitignore()) if err != nil { @@ -101,14 +105,18 @@ func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) { if _, err := io.CopyBuffer(h, strings.NewReader(filepath.Base(f)), buf); err != nil { return "", err } - f, err := os.Open(f) + file, err := os.Open(f) if err != nil { return "", err } - if _, err = io.CopyBuffer(h, f, buf); err != nil { + // Wrap the file in a plain io.Reader so io.CopyBuffer cannot take the + // (*os.File).WriteTo fast path, which ignores buf and allocates a fresh + // 32KiB buffer for every file. Reusing buf keeps this loop allocation-free. + if _, err = io.CopyBuffer(h, readerOnly{file}, buf); err != nil { + file.Close() return "", err } - f.Close() + file.Close() } hash := h.Sum128()