diff --git a/pixi.toml b/pixi.toml index d87fae5..e49a142 100644 --- a/pixi.toml +++ b/pixi.toml @@ -11,4 +11,4 @@ test = "mojo run -I src test/test_unicodedata.mojo" conformance = "mojo run -I src test/conformance.mojo" [dependencies] -mojo = ">=1.0.0b3,<2" +mojo = ">=1.0.0b3.dev0,<2" diff --git a/test/conformance.mojo b/test/conformance.mojo index 961a63a..c82b1e5 100644 --- a/test/conformance.mojo +++ b/test/conformance.mojo @@ -11,12 +11,19 @@ forms holds (UAX #15, NormalizationTest.txt header): NFKC: c4 == toNFKC(c1) == toNFKC(c2) == toNFKC(c3) == toNFKC(c4) == toNFKC(c5) NFKD: c5 == toNFKD(c1) == toNFKD(c2) == toNFKD(c3) == toNFKD(c4) == toNFKD(c5) -This is a scoreboard, not a gate: it always exits successfully. Pass the -name of a part (e.g. "@Part1") as an argument to print the first few -failing rows for that part. +This is a CI gate: it exits non-zero if any row fails, or if the corpus +does not contain exactly EXPECTED_ROWS rows (a guard against a truncated or +swapped data file silently shrinking coverage). Pass the name of a part +(e.g. "@Part1") as an argument to print the first few failing rows for that +part. """ -from std.sys import argv +from std.sys import argv, exit + +# The official UAX #15 NormalizationTest.txt corpus vendored in test/data has +# exactly this many data rows. If the file is truncated, replaced, or fails to +# parse, the row count drifts and coverage silently shrinks -- so we assert it. +comptime EXPECTED_ROWS = 20034 from unicodedata import Normalizer @@ -191,6 +198,21 @@ def main() raises: print("-----------------------------------------") print("TOTAL:", total_pass, "/", total, "rows passing") + # Real CI gate. Exit non-zero on any failure so a regression cannot ship + # green, and verify the corpus is intact so coverage cannot silently shrink. + var failed = False + if total != EXPECTED_ROWS: + print( + "GATE FAIL: parsed", total, "rows but expected", EXPECTED_ROWS, + "-- corpus is truncated, swapped, or misparsed.", + ) + failed = True + if total_pass != total: + print("GATE FAIL:", total - total_pass, "row(s) failed conformance.") + failed = True + if failed: + exit(1) + def _dump(s: String) -> String: """Hex codepoint dump of a string, for failure diagnostics.""" diff --git a/test/test_unicodedata.mojo b/test/test_unicodedata.mojo index 61be4e4..1c186ce 100644 --- a/test/test_unicodedata.mojo +++ b/test/test_unicodedata.mojo @@ -149,5 +149,29 @@ def test_normalizer_reuse() raises: assert_equal(norm.casefold(String("ẞ")), String("ss")) +def test_conformance_corpus_intact() raises: + """Regression guard for the conformance gate. + + The conformance harness (test/conformance.mojo) fails CI unless the corpus + parses to exactly 20034 rows. This mirrors that row count here so a + truncated, swapped, or misparsed NormalizationTest.txt -- which would + silently shrink coverage while still showing 100% -- fails the unit suite + too, independently of the harness. + """ + var raw = open("test/data/normalization_test.txt", "r").read() + var lines = raw.split("\n") + var rows = 0 + for li in range(len(lines)): + var line = String(StringSlice(lines[li]).strip()) + if line.byte_length() == 0: + continue + if line.startswith("@"): + continue + if len(line.split(";")) < 5: + continue + rows += 1 + assert_equal(rows, 20034) + + def main() raises: TestSuite.discover_tests[__functions_in_module()]().run()