perf: reuse buffer when hashing source files for checksums (#2925)

This commit is contained in:
Valentin Maerten
2026-08-03 22:30:20 +02:00
committed by GitHub
parent 5ecb94796d
commit 651aa44f50
2 changed files with 14 additions and 3 deletions

View File

@@ -14,6 +14,9 @@
- Added support for `enum.ref` in `--interactive` prompts. Required vars using
`enum.ref` now show the selection list like static enums, instead of falling
back to free-form input (#2817 by @vmaerten).
- Further improved fingerprinting performance on large repositories: hashing
source files now reuses a single buffer, reducing memory allocations by ~98%
and wall-clock time by ~7% (#2925 by @vmaerten).
## v3.52.0 - 2026-07-02

View File

@@ -88,6 +88,10 @@ func (*ChecksumChecker) Kind() string {
return "checksum"
}
// readerOnly hides any WriterTo/ReaderFrom implementation of the wrapped
// reader, forcing io.CopyBuffer to use the caller-provided buffer.
type readerOnly struct{ io.Reader }
func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) {
sources, err := Globs(t.Dir, t.Sources, t.ShouldUseGitignore())
if err != nil {
@@ -101,14 +105,18 @@ func (c *ChecksumChecker) checksum(t *ast.Task) (string, error) {
if _, err := io.CopyBuffer(h, strings.NewReader(filepath.Base(f)), buf); err != nil {
return "", err
}
f, err := os.Open(f)
file, err := os.Open(f)
if err != nil {
return "", err
}
if _, err = io.CopyBuffer(h, f, buf); err != nil {
// Wrap the file in a plain io.Reader so io.CopyBuffer cannot take the
// (*os.File).WriteTo fast path, which ignores buf and allocates a fresh
// 32KiB buffer for every file. Reusing buf keeps this loop allocation-free.
if _, err = io.CopyBuffer(h, readerOnly{file}, buf); err != nil {
file.Close()
return "", err
}
f.Close()
file.Close()
}
hash := h.Sum128()