Skip to content

Commit 7dc2050

Browse files
Pigbibicodex
andcommitted
fix: bound M0 publisher envelopes to ingress size
Co-Authored-By: Codex <noreply@openai.com>
1 parent 9e0dbcd commit 7dc2050

4 files changed

Lines changed: 92 additions & 3 deletions

docs/m0_research_publisher_envelope_contract.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,14 @@ canonical JSON(键排序、紧凑分隔符、禁止 NaN)计算。台账的
3434
`generated_at``computed_at` 必须相同,均为精确到秒的 UTC `Z` 时间戳。
3535
因此同一 source artifact、metadata 和 `--now` 总会生成字节相同的封套。
3636

37+
尽管 source snapshot 离线输入上限为 2 MiB,生成完成的 canonical envelope
38+
本身必须不超过 **262,144 bytes(256 KiB)**。这个限制按将要写入和 POST 的
39+
紧凑 UTF-8 JSON body 的实际字节数计算,而不是字符数、文件系统占用或 source
40+
snapshot 大小;本地输出文件末尾的换行符不属于 JSON body。超过该上限会以
41+
`publisher_envelope_size_exceeded` fail closed,既不写本地文件,也不发起网络
42+
请求。这个上限与 M0 接收端 Worker ingress 一致,避免“本地可生成但接收端无法
43+
接收”的跨模块失败。
44+
3745
## 默认离线构建
3846

3947
```bash

python/scripts/build_m0_research_publisher_envelope.py

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
PUBLISH_URL_ENV = "QSL_M0_RESEARCH_LEDGER_PUBLISH_URL"
4141
PUBLISH_TOKEN_ENV = "QSL_M0_RESEARCH_LEDGER_PUBLISH_TOKEN"
4242
MAX_SOURCE_SNAPSHOT_BYTES = 2 * 1024 * 1024
43+
# The receiving Worker ingress accepts at most 256 KiB. This is enforced on
44+
# the actual compact UTF-8 JSON body, not on a Python object estimate, source
45+
# artifact size, or character count.
46+
MAX_PUBLISHER_ENVELOPE_BYTES = 256 * 1024
4347

4448
_REPOSITORY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$")
4549
_REVISION = re.compile(r"^[0-9a-f]{40}$")
@@ -69,6 +73,21 @@ def calculate_ledger_sha256(ledger: Mapping[str, Any]) -> str:
6973
return hashlib.sha256(canonical_json(dict(ledger)).encode("utf-8")).hexdigest()
7074

7175

76+
def canonical_envelope_body(envelope: Mapping[str, Any]) -> bytes:
77+
"""Serialize the exact compact UTF-8 body used for local output and POST."""
78+
79+
if not isinstance(envelope, Mapping):
80+
raise M0ResearchPublisherEnvelopeError("publisher_envelope_invalid")
81+
return canonical_json(dict(envelope)).encode("utf-8")
82+
83+
84+
def _enforce_publisher_envelope_size(envelope: Mapping[str, Any]) -> None:
85+
"""Fail closed before a too-large envelope can be written or published."""
86+
87+
if len(canonical_envelope_body(envelope)) > MAX_PUBLISHER_ENVELOPE_BYTES:
88+
raise M0ResearchPublisherEnvelopeError("publisher_envelope_size_exceeded")
89+
90+
7291
def _exact_mapping(value: object, fields: frozenset[str], label: str) -> dict[str, Any]:
7392
if not isinstance(value, Mapping) or set(value) != fields:
7493
raise M0ResearchPublisherEnvelopeError(f"{label}_keys_invalid")
@@ -270,13 +289,15 @@ def validate_m0_research_publisher_envelope(payload: object) -> dict[str, Any]:
270289
expected_digest = calculate_ledger_sha256(normalized_ledger)
271290
if _require_sha256(envelope["ledger_sha256"], "ledger_sha256") != expected_digest:
272291
raise M0ResearchPublisherEnvelopeError("ledger_sha256_mismatch")
273-
return {
292+
normalized = {
274293
"schema_version": PUBLISHER_ENVELOPE_SCHEMA,
275294
"producer": normalized_producer,
276295
"source_artifact": normalized_artifact,
277296
"ledger_sha256": expected_digest,
278297
"ledger": normalized_ledger,
279298
}
299+
_enforce_publisher_envelope_size(normalized)
300+
return normalized
280301

281302

282303
def _publish_url_from_environment(environ: Mapping[str, str]) -> tuple[str, str]:
@@ -308,7 +329,7 @@ def publish_m0_research_publisher_envelope(
308329
url, token = _publish_url_from_environment(os.environ if environ is None else environ)
309330
request = urllib.request.Request(
310331
url,
311-
data=canonical_json(validated).encode("utf-8"),
332+
data=canonical_envelope_body(validated),
312333
method="POST",
313334
headers={
314335
"Authorization": f"Bearer {token}",
@@ -376,7 +397,7 @@ def main(argv: Sequence[str] | None = None) -> int:
376397
# leave a local file that an operator mistakes for an attempted POST.
377398
_publish_url_from_environment(os.environ)
378399
args.output.parent.mkdir(parents=True, exist_ok=True)
379-
args.output.write_text(canonical_json(envelope) + "\n", encoding="utf-8")
400+
args.output.write_bytes(canonical_envelope_body(envelope) + b"\n")
380401
if args.publish:
381402
publish_m0_research_publisher_envelope(envelope)
382403
print(
@@ -402,13 +423,15 @@ def main(argv: Sequence[str] | None = None) -> int:
402423

403424
__all__ = [
404425
"MAX_SOURCE_SNAPSHOT_BYTES",
426+
"MAX_PUBLISHER_ENVELOPE_BYTES",
405427
"M0ResearchPublisherEnvelopeError",
406428
"PUBLISHER_ENVELOPE_SCHEMA",
407429
"PUBLISH_TOKEN_ENV",
408430
"PUBLISH_URL_ENV",
409431
"build_m0_research_publisher_envelope",
410432
"build_source_artifact_metadata",
411433
"calculate_ledger_sha256",
434+
"canonical_envelope_body",
412435
"canonical_json",
413436
"canonical_timestamp",
414437
"load_source_snapshot",

python/tests/test_build_m0_research_publisher_envelope.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@ def getcode(self):
4444

4545

4646
class M0ResearchPublisherEnvelopeTest(unittest.TestCase):
47+
def test_schema_declares_the_cross_module_canonical_utf8_body_limit(self):
48+
schema = json.loads(
49+
(ROOT.parent / "schemas" / "qsl-m0-research-publisher-envelope.v1.schema.json").read_text(
50+
encoding="utf-8"
51+
)
52+
)
53+
self.assertEqual(schema["x-qsl-canonical-utf8-max-bytes"], 256 * 1024)
54+
self.assertIn("canonical UTF-8 JSON request body", schema["$comment"])
55+
4756
def _snapshot(self) -> dict[str, object]:
4857
return {
4958
"schema_version": "qsl_m0_research_source_snapshot.v1",
@@ -147,6 +156,28 @@ def test_build_is_deterministic_hash_bound_and_research_only(self):
147156
self.assertTrue(first["ledger"]["policy"]["no_order"])
148157
self.assertEqual(first["ledger_sha256"], publisher.calculate_ledger_sha256(first["ledger"]))
149158
self.assertEqual(publisher.validate_m0_research_publisher_envelope(first), first)
159+
self.assertLessEqual(
160+
len(publisher.canonical_envelope_body(first)),
161+
publisher.MAX_PUBLISHER_ENVELOPE_BYTES,
162+
)
163+
164+
def test_builder_fails_closed_when_actual_utf8_envelope_body_exceeds_worker_ingress_limit(self):
165+
oversized = self._snapshot()
166+
hypotheses = []
167+
for index in range(500):
168+
hypothesis = json.loads(json.dumps(oversized["hypotheses"][0]))
169+
hypothesis["hypothesis_id"] = f"m0r-large-{index:03d}"
170+
hypothesis["subject"]["identifier"] = f"SOXX-{index:03d}"
171+
hypotheses.append(hypothesis)
172+
oversized["hypotheses"] = hypotheses
173+
with self.assertRaisesRegex(publisher.M0ResearchPublisherEnvelopeError, "publisher_envelope_size_exceeded"):
174+
publisher.build_m0_research_publisher_envelope(
175+
source_snapshot=oversized,
176+
source_artifact=self._artifact("f" * 64),
177+
producer_repository="QuantStrategyLab/QuantRuntimeSettings",
178+
producer_revision="e" * 40,
179+
now="2026-08-21T12:00:00Z",
180+
)
150181

151182
def test_envelope_validation_rejects_digest_or_execution_policy_tampering(self):
152183
envelope = publisher.build_m0_research_publisher_envelope(
@@ -180,12 +211,37 @@ def test_cli_default_is_local_only_and_binds_the_exact_source_bytes(self):
180211
envelope = json.loads(output.read_text(encoding="utf-8"))
181212
self.assertEqual(envelope["source_artifact"]["sha256"], sha256)
182213
self.assertEqual(envelope["ledger_sha256"], publisher.calculate_ledger_sha256(envelope["ledger"]))
214+
self.assertEqual(output.read_bytes(), publisher.canonical_envelope_body(envelope) + b"\n")
183215

184216
missing_output = root / "missing.json"
185217
with self.assertRaisesRegex(publisher.M0ResearchPublisherEnvelopeError, "source_artifact_sha256_mismatch"):
186218
publisher.main(self._arguments(source, missing_output, "0" * 64))
187219
self.assertFalse(missing_output.exists())
188220

221+
def test_cli_oversize_fails_before_any_write_or_opt_in_publish(self):
222+
oversized = self._snapshot()
223+
hypotheses = []
224+
for index in range(500):
225+
hypothesis = json.loads(json.dumps(oversized["hypotheses"][0]))
226+
hypothesis["hypothesis_id"] = f"m0r-large-{index:03d}"
227+
hypothesis["subject"]["identifier"] = f"SOXX-{index:03d}"
228+
hypotheses.append(hypothesis)
229+
oversized["hypotheses"] = hypotheses
230+
with tempfile.TemporaryDirectory() as temp_dir:
231+
root = Path(temp_dir)
232+
source = root / "oversized-source.json"
233+
output = root / "must-not-exist.json"
234+
raw = json.dumps(oversized, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
235+
source.write_bytes(raw)
236+
arguments = self._arguments(source, output, hashlib.sha256(raw).hexdigest()) + ["--publish"]
237+
with patch.object(publisher.urllib.request, "urlopen", side_effect=AssertionError("network called")):
238+
with self.assertRaisesRegex(
239+
publisher.M0ResearchPublisherEnvelopeError,
240+
"publisher_envelope_size_exceeded",
241+
):
242+
publisher.main(arguments)
243+
self.assertFalse(output.exists())
244+
189245
def test_publish_requires_dedicated_environment_and_never_serializes_token(self):
190246
envelope = publisher.build_m0_research_publisher_envelope(
191247
source_snapshot=self._snapshot(),

schemas/qsl-m0-research-publisher-envelope.v1.schema.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
"$id": "https://quantstrategylab.github.io/QuantRuntimeSettings/schemas/qsl-m0-research-publisher-envelope.v1.schema.json",
44
"title": "QSL M0 Research Publisher Envelope v1",
55
"description": "Hash-bound transport for a locally aggregated M0 research ledger. It cannot express allocation, runtime, platform, broker, or execution authority.",
6+
"$comment": "The complete canonical UTF-8 JSON request body must be no larger than 262144 bytes. JSON Schema cannot measure a whole serialized document's UTF-8 byte length; publisher and ingress implementations must enforce this bound before write or POST.",
7+
"x-qsl-canonical-utf8-max-bytes": 262144,
68
"type": "object",
79
"additionalProperties": false,
810
"required": ["schema_version", "producer", "source_artifact", "ledger_sha256", "ledger"],

0 commit comments

Comments
 (0)