From ba518f36ef94b69cb5a19f7c7d7803d3c07f40fd Mon Sep 17 00:00:00 2001 From: nichinichisou <128159543+nichinichisou0609@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:59:30 +0800 Subject: [PATCH] fix(publish): bound resource reader allocations --- tests/test_resource_publication.py | 42 ++++++++++++++++++++++++ tools/deployment/resource_publication.py | 12 ++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/test_resource_publication.py b/tests/test_resource_publication.py index 27adcea..1498882 100644 --- a/tests/test_resource_publication.py +++ b/tests/test_resource_publication.py @@ -11,6 +11,48 @@ from pack.groups import build_groups +@pytest.mark.parametrize('size,maximum,accepted', [ + (0, 0, True), (17, 128 * 1024 * 1024, True), (17, 17, True), + (18, 17, False), (1, 0, False), + (1024 * 1024 + 7, 1024 * 1024 + 7, True), + (1024 * 1024 + 8, 1024 * 1024 + 7, False), +]) +def test_store_reader_preserves_eof_and_budget(tmp_path, monkeypatch, size, maximum, accepted): + monkeypatch.syspath_prepend(str(Path(__file__).parents[1] / 'tools/deployment')) + from resource_publication import read_store_file + body = b'x' * size + source = tmp_path / 'object.bin' + source.write_bytes(body) + if accepted: + resolved, actual = read_store_file(tmp_path, source.name, maximum) + assert resolved == source.resolve() + assert actual == body + else: + with pytest.raises(ValueError, match='exceeds runtime budget'): + read_store_file(tmp_path, source.name, maximum) + + +def test_store_reader_handles_short_reads_without_budget_sized_allocation(tmp_path, monkeypatch): + import io + monkeypatch.syspath_prepend(str(Path(__file__).parents[1] / 'tools/deployment')) + from resource_publication import read_store_file + source = tmp_path / 'object.bin' + source.write_bytes(b'present') + body = b'short reads still require the complete body' + requests = [] + + class ShortReader(io.BytesIO): + def read(self, size=-1): + requests.append(size) + assert 0 < size <= 1024 * 1024 + return super().read(min(size, 3)) + + monkeypatch.setattr(Path, 'open', lambda *_args, **_kwargs: ShortReader(body)) + _, actual = read_store_file(tmp_path, source.name, 128 * 1024 * 1024) + assert actual == body + assert len(requests) > 1 + + @pytest.fixture def publication(tmp_path, monkeypatch): directory = Path(__file__).parents[1] / 'tools/deployment' diff --git a/tools/deployment/resource_publication.py b/tools/deployment/resource_publication.py index fadbf53..4c6d4c8 100644 --- a/tools/deployment/resource_publication.py +++ b/tools/deployment/resource_publication.py @@ -32,7 +32,17 @@ def read_store_file(store, relative, maximum): if not resolved.is_relative_to(store): raise ValueError('store object escaped root') with resolved.open('rb') as stream: - data = stream.read(maximum + 1) + # Read to EOF or the first excess byte without allocating the entire + # runtime budget for every small object in the catalogue. + remaining = maximum + 1 + chunks = [] + while remaining: + chunk = stream.read(min(remaining, 1024 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b''.join(chunks) if len(data) > maximum: raise ValueError('store object exceeds runtime budget') return resolved, data