From 916c734ba0ecb2b8de1e01fc963a389352901cc2 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:03:10 +0300 Subject: [PATCH 01/13] test: add an NER development set apart from the published sample, and require a setting to measure it tests/fixtures/ner-dev.json holds 31 invented sentences for tuning the NER pass: lower case, sentence case, all capitals, records, acronyms, news and mixed case, some with no entity so that false positives show. It shares no sentence with ner-sample.json, whose figures stay the published ones, and the measurement must take NER_SAMPLE to run on it. --- tests/fixtures/ner-dev.json | 42 ++++++++++++++++++++++++++++++++ tests/python/test_ner_measure.py | 15 ++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/fixtures/ner-dev.json diff --git a/tests/fixtures/ner-dev.json b/tests/fixtures/ner-dev.json new file mode 100644 index 0000000..cebb16c --- /dev/null +++ b/tests/fixtures/ner-dev.json @@ -0,0 +1,42 @@ +{ + "about": "A development set for tuning the redact task's NER pass, kept apart from ner-sample.json, whose figures are the ones published. Invented like the sample; some sentences hold no entity at all, so that false positives show, and the acronyms are there for the same reason.", + "items": [ + {"register": "lower case", "text": "can you tell oskar that the meeting moved to tuesday", "entities": [["PER", "oskar"]]}, + {"register": "lower case", "text": "landed in dublin, heading to the hotel now", "entities": [["LOC", "dublin"]]}, + {"register": "lower case", "text": "does anyone know if brightwave media is still hiring", "entities": [["ORG", "brightwave media"]]}, + {"register": "lower case", "text": "say hi to farah and yusra for me", "entities": [["PER", "farah"], ["PER", "yusra"]]}, + {"register": "lower case", "text": "we should visit kraków before the winter", "entities": [["LOC", "kraków"]]}, + {"register": "lower case", "text": "my sister works at pinecrest bank in leeds", "entities": [["ORG", "pinecrest bank"], ["LOC", "leeds"]]}, + {"register": "lower case", "text": "omg diego got the job at lumen studios!!", "entities": [["PER", "diego"], ["ORG", "lumen studios"]]}, + {"register": "lower case", "text": "is the train to vienna on time", "entities": [["LOC", "vienna"]]}, + {"register": "lower case", "text": "can you send me the report before friday", "entities": []}, + {"register": "lower case", "text": "the printer on the second floor is jammed again", "entities": []}, + {"register": "lower case", "text": "lol that was the best pizza ever", "entities": []}, + {"register": "lower case", "text": "remind me to water the plants tomorrow", "entities": []}, + + {"register": "sentence case", "text": "The weather was lovely all weekend.", "entities": []}, + {"register": "sentence case", "text": "Please bring the signed form to the front desk.", "entities": []}, + {"register": "sentence case", "text": "Our team will review the proposal next week.", "entities": []}, + {"register": "sentence case", "text": "The board approved the budget for the new year.", "entities": []}, + {"register": "sentence case", "text": "Thanks again, see you at the station.", "entities": []}, + + {"register": "all capitals", "text": "PLEASE CALL MARCUS LINDQVIST BEFORE NOON", "entities": [["PER", "MARCUS LINDQVIST"]]}, + {"register": "all capitals", "text": "URGENT: SHIPMENT FOR ORION FREIGHT IS DELAYED IN ROTTERDAM", "entities": [["ORG", "ORION FREIGHT"], ["LOC", "ROTTERDAM"]]}, + {"register": "all capitals", "text": "DO NOT FORWARD THIS EMAIL", "entities": []}, + + {"register": "records", "text": "Approved by K. Watanabe on behalf of Fernhill Insurance.", "entities": [["PER", "K. Watanabe"], ["ORG", "Fernhill Insurance"]]}, + {"register": "records", "text": "Patient: OYELARAN, Bisi. Ward: 4B.", "entities": [["PER", "OYELARAN"], ["PER", "Bisi"]]}, + {"register": "records", "text": "Contact Jean-Luc Moreau in Marseille.", "entities": [["PER", "Jean-Luc Moreau"], ["LOC", "Marseille"]]}, + {"register": "records", "text": "Signed: M. A. Delacroix, Lyon.", "entities": [["PER", "M. A. Delacroix"], ["LOC", "Lyon"]]}, + {"register": "records", "text": "Owner: PETROV, Anya; city: Sofia.", "entities": [["PER", "PETROV"], ["PER", "Anya"], ["LOC", "Sofia"]]}, + + {"register": "acronyms", "text": "Send the IBAN and BIC to the HR team by Friday.", "entities": []}, + {"register": "acronyms", "text": "The NATO summit in Brussels ended early.", "entities": [["ORG", "NATO"], ["LOC", "Brussels"]]}, + {"register": "acronyms", "text": "Our CEO met the FDA inspectors yesterday.", "entities": [["ORG", "FDA"]]}, + + {"register": "news", "text": "Talia Brenner joined Greenfold Energy as chief engineer in Oslo.", "entities": [["PER", "Talia Brenner"], ["ORG", "Greenfold Energy"], ["LOC", "Oslo"]]}, + {"register": "news", "text": "The museum in Seville reopened after repairs.", "entities": [["LOC", "Seville"]]}, + + {"register": "mixed case", "text": "meeting w/ Anil tmrw at the Kestrel office in bristol", "entities": [["PER", "Anil"], ["ORG", "Kestrel"], ["LOC", "bristol"]]} + ] +} diff --git a/tests/python/test_ner_measure.py b/tests/python/test_ner_measure.py index 3568f6c..77e6eb1 100644 --- a/tests/python/test_ner_measure.py +++ b/tests/python/test_ner_measure.py @@ -14,6 +14,7 @@ REPO = Path(__file__).resolve().parents[2] SAMPLE = REPO / "tests" / "fixtures" / "ner-sample.json" +DEV = REPO / "tests" / "fixtures" / "ner-dev.json" spec = importlib.util.spec_from_file_location("ner_measure", REPO / "scripts" / "ner_measure.py") measure = importlib.util.module_from_spec(spec) spec.loader.exec_module(measure) @@ -45,6 +46,20 @@ def test_every_register_holds_every_label(self): def test_the_script_reads_this_sample(self): self.assertEqual(measure.SAMPLE, SAMPLE) + def test_another_sample_can_be_measured(self): + self.assertEqual(measure.sample_path({}), SAMPLE) + self.assertEqual(measure.sample_path({"NER_SAMPLE": "tests/fixtures/ner-dev.json"}), DEV) + + def test_the_development_set_is_well_formed_and_apart_from_the_sample(self): + dev = json.loads(DEV.read_text(encoding="utf-8"))["items"] + self.assertEqual({item["text"] for item in dev} & {item["text"] for item in self.items}, set()) + self.assertTrue(any(not item["entities"] for item in dev), "it must show false positives too") + for item in dev: + with self.subTest(text=item["text"][:40]): + for label, text in item["entities"]: + self.assertIn(label, {"PER", "ORG", "LOC"}) + self.assertEqual(item["text"].count(text), 1) + def test_importing_the_script_ignores_the_benchmark_s_settings(self): # It borrows the benchmark's protocol helpers; a malformed BENCH_TARGETS is the benchmark's business. sys.modules.pop("benchmark", None) From 2b08dd46a3e71a7c8835c79a7d242fb3fd497411 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:07:30 +0300 Subject: [PATCH 02/13] feat: let the NER measurement run on another sample, such as the development set, with NER_SAMPLE --- docker/compose.yaml | 1 + scripts/ner_measure.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/compose.yaml b/docker/compose.yaml index 62bec5d..1d0297a 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -204,6 +204,7 @@ services: environment: <<: *python-env NER_TARGET: http://model-worker:8090 + NER_SAMPLE: ${NER_SAMPLE:-} BENCH_COMMIT: ${BENCH_COMMIT:-unknown} BENCH_RUNNER: ${BENCH_RUNNER:-local, unspecified} command: [python, scripts/ner_measure.py] diff --git a/scripts/ner_measure.py b/scripts/ner_measure.py index 3639cd5..14c6329 100644 --- a/scripts/ner_measure.py +++ b/scripts/ner_measure.py @@ -20,11 +20,17 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from benchmark import POLL_S, TERMINAL, call, cpu_model, wait_ready # noqa: E402 (the same protocol helpers) -SAMPLE = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "ner-sample.json" +REPO = Path(__file__).resolve().parents[1] +SAMPLE = REPO / "tests" / "fixtures" / "ner-sample.json" LABELS = ("PER", "ORG", "LOC") THRESHOLDS = (0.85, 0.5) # the task's default, and a lower one to show what it trades +def sample_path(environ) -> Path: + """The published sample, or NER_SAMPLE, such as the development set, relative to the repository.""" + return REPO / environ["NER_SAMPLE"] if environ.get("NER_SAMPLE") else SAMPLE + + def gold_spans(item: dict) -> list: return [(item["text"].index(text), item["text"].index(text) + len(text), label) for label, text in item["entities"]] @@ -91,7 +97,8 @@ def main() -> None: health = wait_ready(base) if health["backend"] != "onnx": raise SystemExit(f"{base} reports backend {health['backend']!r}; NER needs the onnx worker") - items = json.loads(SAMPLE.read_text(encoding="utf-8"))["items"] + sample = sample_path(os.environ) + items = json.loads(sample.read_text(encoding="utf-8"))["items"] spans, models = [], set() for number, item in enumerate(items, start=1): result = redact(base, item["text"], min(THRESHOLDS)) @@ -101,7 +108,7 @@ def main() -> None: now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") entities = sum(len(item["entities"]) for item in items) print(f"""Measured {now} UTC, commit `{os.environ.get("BENCH_COMMIT", "unknown")}`, runner: {os.environ.get("BENCH_RUNNER", "unspecified")}, CPU: {cpu_model()}. -Model {", ".join(f"`{model}`" for model in sorted(models))}; {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence at threshold {min(THRESHOLDS)}, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. +Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence at threshold {min(THRESHOLDS)}, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. | Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | | --- | --- | --- | --- | --- | --- | --- | --- |""") From 9b0c4aa9bb1fd4489e515c5e7faffd96891d6123 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:07:30 +0300 Subject: [PATCH 03/13] test: require a title-cased second NER pass for uncased sentences, and one entity for touching name pieces A sentence with no capital letter, or written all in capitals, must be read again in title case from a copy that keeps every offset, with sentence-case and cased text left alone, since title-casing "the board approved" made "Board" an organisation. Pieces of one name that touch, or meet at a hyphen, full stop or apostrophe, must be joined before the threshold, so the initial of "A. Okonkwo" carries the surname the model was less sure of. The aggregation takes the text to see what lies between pieces, and a stub model drives the two passes without numpy. --- tests/python/test_jobs.py | 126 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 121 insertions(+), 5 deletions(-) diff --git a/tests/python/test_jobs.py b/tests/python/test_jobs.py index 156d525..d83ebff 100644 --- a/tests/python/test_jobs.py +++ b/tests/python/test_jobs.py @@ -298,19 +298,20 @@ def setUp(self): ("B-LOC", 0.9), ("I-LOC", 0.88), ("I-LOC", 0.7), ("O", 1.0)) def test_first_piece_labels_words_and_the_threshold_uses_the_mean(self): - spans = _entities(self.window, self.rows_, ["PER", "LOC"], 0.85) + spans = _entities(self.window, self.rows_, ["PER", "LOC"], 0.85, "John Smithson lives in New York City") self.assertEqual(spans, [{"start": 0, "end": 13, "label": "PER", "source": "model:PER", "score": 0.925, "priority": len(tasks.RULES)}]) - spans = _entities(self.window, self.rows_, ["PER", "LOC"], 0.8) + spans = _entities(self.window, self.rows_, ["PER", "LOC"], 0.8, "John Smithson lives in New York City") self.assertEqual([(span["start"], span["end"], span["label"], span["score"]) for span in spans], [(0, 13, "PER", 0.925), (23, 36, "LOC", 0.8267)]) def test_unwanted_and_misc_entities_are_dropped(self): - self.assertEqual([span["label"] for span in _entities(self.window, self.rows_, ["LOC"], 0.5)], ["LOC"]) + self.assertEqual([span["label"] for span in _entities(self.window, self.rows_, ["LOC"], 0.5, "John Smithson lives in New York City")], + ["LOC"]) rows = self.rows(("O", 1.0), ("B-MISC", 0.99), ("I-MISC", 0.99), ("I-MISC", 0.99), ("O", 1.0), ("O", 1.0), ("I-ORG", 0.9), ("I-ORG", 0.9), ("B-ORG", 0.9), ("O", 1.0)) - spans = _entities(self.window, rows, ["PER", "ORG", "LOC"], 0.5) - # An inside tag without a beginning starts an entity; a fresh beginning closes it. + spans = _entities(self.window, rows, ["PER", "ORG", "LOC"], 0.5, "John Smithson lives in New York City") + # An inside tag without a beginning starts an entity; a fresh beginning after a space closes it. self.assertEqual([(span["start"], span["end"], span["label"]) for span in spans], [(23, 31, "ORG"), (32, 36, "ORG")]) @@ -424,6 +425,121 @@ def test_batch_size_must_be_positive(self): self.score(self.documents(5), batch_size=0) +class NerJoinTests(unittest.TestCase): + """Pieces of one name that touch, or meet at a hyphen, full stop or apostrophe, are one entity, joined before + the threshold, so a confident initial carries the surname the model was less sure of.""" + + Window = NerAggregationTests.Window + rows = NerAggregationTests.rows + LABELS = NerAggregationTests.LABELS + + def spans(self, text, words, labels, min_score=0.85): + window = self.Window([None, *range(len(words)), None], [(0, 0), *words, (0, 0)]) + rows = self.rows(("O", 1.0), *labels, ("O", 1.0)) + return [(span["start"], span["end"], span["label"], span["score"]) + for span in _entities(window, rows, ["PER", "LOC"], min_score, text)] + + def test_an_initial_carries_the_surname_that_touches_it(self): + # "A. Okonkwo": the model starts a second entity at the full stop, less sure of it than of the initial. + text = "A. Okonkwo" + spans = self.spans(text, [(0, 1), (1, 2), (3, 10)], [("B-PER", 1.0), ("B-PER", 0.83), ("I-PER", 0.83)]) + self.assertEqual(spans, [(0, 10, "PER", 0.8867)]) + + def test_pieces_meeting_at_a_hyphen_are_one_name(self): + text = "Mei-Ling Chou" + spans = self.spans(text, [(0, 3), (3, 4), (4, 8), (9, 13)], + [("B-PER", 0.9), ("O", 0.9), ("B-PER", 0.9), ("I-PER", 0.9)]) + self.assertEqual(spans, [(0, 13, "PER", 0.9)]) + + def test_entities_apart_or_of_another_kind_stay_apart(self): + text = "Ines and Pedro in Lima" + spans = self.spans(text, [(0, 4), (5, 8), (9, 14), (15, 17), (18, 22)], + [("B-PER", 0.95), ("O", 1.0), ("B-PER", 0.95), ("O", 1.0), ("B-LOC", 0.95)]) + self.assertEqual([(start, end, label) for start, end, label, _ in spans], + [(0, 4, "PER"), (9, 14, "PER"), (18, 22, "LOC")]) + touching = self.spans("ParisLondon", [(0, 5), (5, 11)], [("B-LOC", 0.95), ("B-PER", 0.95)]) + self.assertEqual([(start, end, label) for start, end, label, _ in touching], [(0, 5, "LOC"), (5, 11, "PER")]) + + +class CasedVariantTests(unittest.TestCase): + """The NER model was trained on cased news, so a sentence written all in lower case or all in capitals gets a + second pass in title case. The copy keeps every offset, and other sentences are left alone.""" + + def test_a_lower_case_sentence_is_title_cased_letter_for_letter(self): + text = "can you tell oskar that the meeting moved to tuesday" + self.assertEqual(tasks._cased_variant(text), "Can You Tell Oskar That The Meeting Moved To Tuesday") + + def test_an_all_capitals_sentence_is_title_cased(self): + self.assertEqual(tasks._cased_variant("PLEASE CALL MARCUS LINDQVIST"), "Please Call Marcus Lindqvist") + + def test_cased_and_sentence_case_text_needs_no_second_pass(self): + # Title-casing "the board approved the budget" makes "Board" an organisation; sentence case stays as written. + for text in ["Lena Okafor said the depot in Rotterdam would open.", "The board approved the budget.", + "Customer: WIERZBICKI, Tomasz.", "12:30 - 14:00", ""]: + with self.subTest(text=text): + self.assertIsNone(tasks._cased_variant(text)) + + def test_only_the_sentences_that_need_it_change(self): + self.assertEqual(tasks._cased_variant("Lena Okafor called. ask priya about lisbon!\nDO NOT REPLY"), + "Lena Okafor called. Ask Priya About Lisbon!\nDo Not Reply") + + def test_every_offset_is_kept_even_where_case_would_change_length(self): + # "ß" upper-cases to two letters; such a character stays as it is. + text = "ßtraße in münchen und istanbul" + variant = tasks._cased_variant(text) + self.assertEqual(variant, "ßtraße In München Und Istanbul") + self.assertEqual(len(variant), len(text)) + + +class NerPassesTests(unittest.TestCase): + """_ner runs the model on the text and, when a sentence is uncased, on its title-cased copy; spans from both + come back in the text's own offsets, the text's first. A stub model tags every capitalised word as a person.""" + + class Encoding: + def __init__(self, text): + import re as _re + words = [(match.start(), match.end()) for match in _re.finditer(r"\S+", text)] + self.text, self.overflowing = text, [] + self.word_ids, self.offsets = [None, *range(len(words)), None], [(0, 0), *words, (0, 0)] + + class Tokenizer: + def __init__(self): + self.texts = [] + + def encode(self, text): + self.texts.append(text) + return NerPassesTests.Encoding(text) + + @staticmethod + def probabilities(session, names, window): + rows = [] + for position, word_id in enumerate(window.word_ids): + row = [0.0] * len(tasks.NER_LABELS) + start, end = window.offsets[position] + capitalised = word_id is not None and window.text[start:end][:1].isupper() + row[tasks.NER_LABELS.index("B-PER" if capitalised else "O")] = 0.99 + rows.append(row) + return rows + + def run_ner(self, text): + tokenizer, events = self.Tokenizer(), [] + with patch.object(tasks, "_onnx", return_value=(tokenizer, object(), set())), \ + patch.object(tasks, "_probabilities", self.probabilities): + spans = tasks._ner(text, ["PER"], 0.85, "/unused", lambda completed, total: events.append((completed, total))) + return [(span["start"], span["end"]) for span in spans], tokenizer.texts, events + + def test_an_uncased_sentence_is_read_again_in_title_case(self): + spans, texts, events = self.run_ner("ask priya") + self.assertEqual(texts, ["ask priya", "Ask Priya"]) + self.assertEqual(spans, [(0, 3), (4, 9)]) # offsets of the original text + self.assertEqual(events, [(0, 2), (1, 2), (2, 2)]) + + def test_cased_text_is_read_once(self): + spans, texts, events = self.run_ner("Ask Priya.") + self.assertEqual(texts, ["Ask Priya."]) + self.assertEqual(events, [(0, 1), (1, 1)]) + + class ModelNameTests(unittest.TestCase): """Every result names its model from one table, which the MCP server's table is checked against.""" From 8dc2a1d3c401f5447f2991c732498c1ecc45dc82 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:10:49 +0300 Subject: [PATCH 04/13] test: score joined name pieces by their most confident piece, so an unsure piece cannot hide a sure one Measured on the development set, averaging the joined pieces dropped "Jean-Luc Moreau" and "M. A. Delacroix" below the threshold, so nothing of either was masked where a confident first piece had been before. A joined name must pass when any of its pieces does, with that piece's score. --- tests/python/test_jobs.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/python/test_jobs.py b/tests/python/test_jobs.py index d83ebff..cf1ffc0 100644 --- a/tests/python/test_jobs.py +++ b/tests/python/test_jobs.py @@ -427,7 +427,8 @@ def test_batch_size_must_be_positive(self): class NerJoinTests(unittest.TestCase): """Pieces of one name that touch, or meet at a hyphen, full stop or apostrophe, are one entity, joined before - the threshold, so a confident initial carries the surname the model was less sure of.""" + the threshold and scored by their most confident piece: a confident initial carries the surname the model was + less sure of, and an unsure piece never hides a confident one.""" Window = NerAggregationTests.Window rows = NerAggregationTests.rows @@ -443,7 +444,7 @@ def test_an_initial_carries_the_surname_that_touches_it(self): # "A. Okonkwo": the model starts a second entity at the full stop, less sure of it than of the initial. text = "A. Okonkwo" spans = self.spans(text, [(0, 1), (1, 2), (3, 10)], [("B-PER", 1.0), ("B-PER", 0.83), ("I-PER", 0.83)]) - self.assertEqual(spans, [(0, 10, "PER", 0.8867)]) + self.assertEqual(spans, [(0, 10, "PER", 1.0)]) def test_pieces_meeting_at_a_hyphen_are_one_name(self): text = "Mei-Ling Chou" @@ -451,6 +452,16 @@ def test_pieces_meeting_at_a_hyphen_are_one_name(self): [("B-PER", 0.9), ("O", 0.9), ("B-PER", 0.9), ("I-PER", 0.9)]) self.assertEqual(spans, [(0, 13, "PER", 0.9)]) + def test_an_unsure_piece_does_not_hide_a_confident_one(self): + # "Jean-Luc Moreau": averaged with "Luc Moreau" the name fell below the threshold and nothing was masked. + text = "Jean-Luc Moreau" + spans = self.spans(text, [(0, 4), (4, 5), (5, 8), (9, 15)], + [("B-PER", 0.95), ("O", 0.9), ("B-PER", 0.6), ("I-PER", 0.6)]) + self.assertEqual(spans, [(0, 15, "PER", 0.95)]) + # Joined pieces that are all unsure stay below it. + self.assertEqual(self.spans(text, [(0, 4), (4, 5), (5, 8), (9, 15)], + [("B-PER", 0.6), ("O", 0.9), ("B-PER", 0.6), ("I-PER", 0.6)]), []) + def test_entities_apart_or_of_another_kind_stay_apart(self): text = "Ines and Pedro in Lima" spans = self.spans(text, [(0, 4), (5, 8), (9, 14), (15, 17), (18, 22)], From 2b2670ea2104769f4d98d4fbaaea0c20eb6c24b3 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:12:09 +0300 Subject: [PATCH 05/13] feat: read uncased sentences again in title case, and join the pieces of one name before the threshold The NER model was trained on cased news and found almost no lower-case name. A sentence written all in lower case or all in capitals is now read a second time from a title-cased copy that keeps every offset; sentence-case and mixed-case sentences are left alone, because title-casing "the board approved" made "Board" an organisation. The text's own reading comes first, so it wins where both find a span. Pieces of one name that touch or meet at a hyphen, full stop or apostrophe are joined before the threshold and scored by their most confident piece, so the initial of "A. Okonkwo" carries the surname the model was less sure of, and an unsure piece never hides a confident one. The numpy step is now _probabilities, which the tests replace. On the development set at 0.85, exact recall rose from 0.18 to 0.64 in lower case, 0.00 to 0.33 in all capitals and 0.27 to 0.45 in records, with no new false positive in its sentences without entities. --- worker/ai_bridge/tasks.py | 89 ++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/worker/ai_bridge/tasks.py b/worker/ai_bridge/tasks.py index fd6d4d8..98e1424 100644 --- a/worker/ai_bridge/tasks.py +++ b/worker/ai_bridge/tasks.py @@ -397,16 +397,12 @@ def find(text: str): ) -def _close(spans: list, current, entities: list, min_score: float) -> None: - if current is None: - return - score = sum(current["scores"]) / len(current["scores"]) - if current["label"] in entities and score >= min_score: - spans.append({"start": current["start"], "end": current["end"], "label": current["label"], - "source": "model:" + current["label"], "score": round(score, 4), "priority": len(RULES)}) +# What may sit between two pieces of one name: nothing, or a hyphen, full stop or apostrophe ("A. Okonkwo", +# "Mei-Ling", "O'Neill"). A space is not one: two names side by side stay two entities. +JOINERS = frozenset({"", "-", ".", "'", "\u2019"}) -def _entities(window, probabilities: list, entities: list, min_score: float) -> list: +def _entities(window, probabilities: list, entities: list, min_score: float, text: str) -> list: # Each word takes its first word piece's label, the usual BERT aggregation; later pieces only extend it. words = [] for position, word_id in enumerate(window.word_ids): @@ -419,38 +415,81 @@ def _entities(window, probabilities: list, entities: list, min_score: float) -> row = probabilities[position] label_id = max(range(len(row)), key=row.__getitem__) words.append([start, end, NER_LABELS[label_id], float(row[label_id]), word_id]) - spans, current = [], None + pieces, current = [], None for start, end, label, score, _ in words: prefix, _, kind = label.partition("-") if prefix == "B" or (prefix == "I" and (current is None or current["label"] != kind)): - _close(spans, current, entities, min_score) current = {"start": start, "end": end, "label": kind, "scores": [score]} + pieces.append(current) elif prefix == "I": current["end"] = end current["scores"].append(score) else: - _close(spans, current, entities, min_score) current = None - _close(spans, current, entities, min_score) - return spans + # The model often starts a new entity at the full stop of an initial and is less sure of what follows, so touching + # pieces are joined before the threshold and scored by their most confident piece: the initial carries the + # surname, and an unsure piece never hides a confident one, as an average over "Jean-Luc Moreau" did. + joined = [] + for piece in pieces: + score = sum(piece["scores"]) / len(piece["scores"]) + if joined and joined[-1]["label"] == piece["label"] and text[joined[-1]["end"]:piece["start"]] in JOINERS: + joined[-1]["end"] = piece["end"] + joined[-1]["score"] = max(joined[-1]["score"], score) + else: + joined.append({"start": piece["start"], "end": piece["end"], "label": piece["label"], "score": score}) + return [{"start": piece["start"], "end": piece["end"], "label": piece["label"], "source": "model:" + piece["label"], + "score": round(piece["score"], 4), "priority": len(RULES)} + for piece in joined if piece["label"] in entities and piece["score"] >= min_score] + + +# A sentence ends at . ! or ? before whitespace, or at a line break. +SENTENCE_END = re.compile(r"[.!?](?=\s)|\n") +LETTERS = re.compile(r"[^\W\d_]+") + + +def _cased_variant(text: str): + """A copy of text, offset for offset, with each sentence written all in lower case or all in capitals in title + case, or None when no sentence is. The model was trained on cased news and finds almost no lower-case name; + sentence case and mixed case are left alone, since title-casing "the board approved" makes "Board" an + organisation. A character whose case change would alter its length stays as it is.""" + chars, start = list(text), 0 + for end in [match.end() for match in SENTENCE_END.finditer(text)] + [len(text)]: + cased = [char for char in text[start:end] if char.isupper() or char.islower()] + if cased and not (any(char.isupper() for char in cased) and any(char.islower() for char in cased)): + for word in LETTERS.finditer(text, start, end): + for index in range(word.start(), word.end()): + changed = text[index].upper() if index == word.start() else text[index].lower() + if len(changed) == 1: + chars[index] = changed + start = end + variant = "".join(chars) + return variant if variant != text else None + + +def _probabilities(session, names, window) -> list: + import numpy as np + inputs = _tensors([window]) + outputs = session.run(None, {key: value for key, value in inputs.items() if key in names}) + logits = _output(session, outputs, "logits")[0] + shifted = np.exp(logits - logits.max(axis=-1, keepdims=True)) + return (shifted / shifted.sum(axis=-1, keepdims=True)).tolist() -def _ner(text: str, entities: list, min_score: float, model_dir: str, progress) -> list: - import numpy as np - # Overlapping 512 piece windows cover long text; offsets stay relative to the whole text. +def _ner(text: str, entities: list, min_score: float, model_dir: str, progress) -> list: + # Overlapping 512 piece windows cover long text; offsets stay relative to the whole text. An uncased sentence is + # read again from a title-cased copy with the same offsets; the text's own reading comes first, so it wins where + # both find the same span. tokenizer, session, names = _onnx(Path(model_dir) / "redact", 512, stride=64) - encoding = tokenizer.encode(text) - windows = [encoding, *encoding.overflowing] + variant = _cased_variant(text) + windows = [] + for reading in [text] if variant is None else [text, variant]: + encoding = tokenizer.encode(reading) + windows.extend((reading, window) for window in [encoding, *encoding.overflowing]) progress(0, len(windows)) candidates = [] - for index, window in enumerate(windows): - inputs = _tensors([window]) - outputs = session.run(None, {key: value for key, value in inputs.items() if key in names}) - logits = _output(session, outputs, "logits")[0] - shifted = np.exp(logits - logits.max(axis=-1, keepdims=True)) - probabilities = (shifted / shifted.sum(axis=-1, keepdims=True)).tolist() - candidates.extend(_entities(window, probabilities, entities, min_score)) + for index, (reading, window) in enumerate(windows): + candidates.extend(_entities(window, _probabilities(session, names, window), entities, min_score, reading)) progress(index + 1, len(windows)) return candidates From f477ed2c4ba0ec19eade2b3afbffa46e224e00aa Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:14:17 +0300 Subject: [PATCH 06/13] test: require the NER measurement to report names masked under any label For redaction the question is whether a name is hidden, and a name masked as an organisation is still hidden. The scorer must count, one to one, the gold spans overlapped by a model span of any label, and the per-label rows must keep counting only their own label. --- tests/python/test_ner_measure.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/python/test_ner_measure.py b/tests/python/test_ner_measure.py index 77e6eb1..3d2de4b 100644 --- a/tests/python/test_ner_measure.py +++ b/tests/python/test_ner_measure.py @@ -76,16 +76,18 @@ def test_gold_spans_are_the_offsets_of_their_texts(self): def test_exact_and_overlapping_matches_are_counted_separately(self): gold = [(0, 11, "PER"), (20, 27, "LOC")] predicted = [(0, 4, "PER"), (20, 27, "LOC"), (30, 35, "ORG")] - self.assertEqual(measure.score(gold, predicted), {"gold": 2, "predicted": 3, "exact": 1, "overlapping": 2}) + self.assertEqual(measure.score(gold, predicted), + {"gold": 2, "predicted": 3, "exact": 1, "overlapping": 2, "masked": 2}) def test_one_predicted_span_matches_one_gold_span_at_most(self): # One PER span over "ines and pedro" found one name, not both. self.assertEqual(measure.score([(0, 4, "PER"), (9, 14, "PER")], [(0, 14, "PER")]), - {"gold": 2, "predicted": 1, "exact": 0, "overlapping": 1}) + {"gold": 2, "predicted": 1, "exact": 0, "overlapping": 1, "masked": 1}) - def test_a_wrong_label_is_neither_exact_nor_overlapping(self): + def test_a_wrong_label_is_neither_exact_nor_overlapping_but_still_masks(self): + # For redaction what matters is that the name is hidden; the label it is hidden under is secondary. self.assertEqual(measure.score([(0, 5, "PER")], [(0, 5, "ORG")]), - {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0}) + {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 1}) def test_only_the_model_s_spans_at_the_threshold_are_predictions(self): spans = [{"start": 0, "end": 3, "label": "PER", "source": "model:PER", "score": 0.9}, @@ -105,12 +107,15 @@ def test_rows_add_up_by_register_and_by_label_for_each_threshold(self): self.assertEqual([name for threshold, name, _ in rows if threshold == 0.85], ["informal", "news", "PER", "ORG", "LOC"]) counts = {(threshold, name): value for threshold, name, value in rows} - self.assertEqual(counts[(0.85, "informal")], {"gold": 2, "predicted": 0, "exact": 0, "overlapping": 0}) - self.assertEqual(counts[(0.5, "informal")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1}) - # Acme predicted as a place is an ORG missed and a LOC that is wrong, in each label's own row. - self.assertEqual(counts[(0.85, "ORG")], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0}) - self.assertEqual(counts[(0.85, "LOC")], {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0}) - self.assertEqual(counts[(0.85, "PER")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1}) + self.assertEqual(counts[(0.85, "informal")], + {"gold": 2, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 0}) + self.assertEqual(counts[(0.5, "informal")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) + # Acme predicted as a place is an ORG missed and a LOC that is wrong, in each label's own row; in the + # register's row it is still a name masked. + self.assertEqual(counts[(0.85, "news")], {"gold": 2, "predicted": 2, "exact": 1, "overlapping": 1, "masked": 2}) + self.assertEqual(counts[(0.85, "ORG")], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 0}) + self.assertEqual(counts[(0.85, "LOC")], {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 0}) + self.assertEqual(counts[(0.85, "PER")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) def test_rates_have_two_decimals_and_an_empty_denominator_is_not_a_number(self): self.assertEqual(measure.rate(1, 4), "0.25") From 7707b8b5c70f9a58a90a635524cd7f586ec047ca Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:14:45 +0300 Subject: [PATCH 07/13] feat: report the share of entities masked under any label, which is what a redaction hides --- scripts/ner_measure.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/ner_measure.py b/scripts/ner_measure.py index 14c6329..f5138d7 100644 --- a/scripts/ner_measure.py +++ b/scripts/ner_measure.py @@ -40,17 +40,23 @@ def predictions(spans: list, threshold: float) -> list: if span["source"].startswith("model:") and span["score"] >= threshold] -def score(gold: list, predicted: list) -> dict: - """Exact: same label and boundaries. Overlapping: same label and a shared character, each predicted span matching - one gold span at most, so one span over two names finds one of them.""" - unused, overlapping = list(predicted), 0 +def matched(gold: list, predicted: list, same_label: bool) -> int: + """Gold spans overlapped by a predicted span, each predicted span matching one gold span at most.""" + unused, count = list(predicted), 0 for start, end, label in gold: - match = next((p for p in unused if p[2] == label and p[0] < end and start < p[1]), None) + match = next((p for p in unused if (p[2] == label or not same_label) and p[0] < end and start < p[1]), None) if match is not None: unused.remove(match) - overlapping += 1 + count += 1 + return count + + +def score(gold: list, predicted: list) -> dict: + """Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted + span, so one span over two names finds one of them. Masked: a shared character under any label, which is what a + redaction hides.""" return {"gold": len(gold), "predicted": len(predicted), "exact": len(set(gold) & set(predicted)), - "overlapping": overlapping} + "overlapping": matched(gold, predicted, True), "masked": matched(gold, predicted, False)} def rate(part: int, whole: int) -> str: @@ -108,14 +114,15 @@ def main() -> None: now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") entities = sum(len(item["entities"]) for item in items) print(f"""Measured {now} UTC, commit `{os.environ.get("BENCH_COMMIT", "unknown")}`, runner: {os.environ.get("BENCH_RUNNER", "unspecified")}, CPU: {cpu_model()}. -Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence at threshold {min(THRESHOLDS)}, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. +Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence at threshold {min(THRESHOLDS)}, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. -| Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | -| --- | --- | --- | --- | --- | --- | --- | --- |""") +| Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | Masked | +| --- | --- | --- | --- | --- | --- | --- | --- | --- |""") for threshold, name, counts in tally(items, spans, THRESHOLDS): print(f"| {threshold} | {name} | {counts['gold']} | {counts['predicted']} | " f"{rate(counts['exact'], counts['predicted'])} | {rate(counts['exact'], counts['gold'])} | " - f"{rate(counts['overlapping'], counts['predicted'])} | {rate(counts['overlapping'], counts['gold'])} |") + f"{rate(counts['overlapping'], counts['predicted'])} | {rate(counts['overlapping'], counts['gold'])} | " + f"{rate(counts['masked'], counts['gold'])} |") if __name__ == "__main__": From 76ebba068993e2d8384db226bf6cf99f7d77751b Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:17:58 +0300 Subject: [PATCH 08/13] test: require a surname written first in capitals before a given name to be read again in title case --- tests/python/test_jobs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/python/test_jobs.py b/tests/python/test_jobs.py index cf1ffc0..7fdf2b3 100644 --- a/tests/python/test_jobs.py +++ b/tests/python/test_jobs.py @@ -486,10 +486,17 @@ def test_an_all_capitals_sentence_is_title_cased(self): def test_cased_and_sentence_case_text_needs_no_second_pass(self): # Title-casing "the board approved the budget" makes "Board" an organisation; sentence case stays as written. for text in ["Lena Okafor said the depot in Rotterdam would open.", "The board approved the budget.", - "Customer: WIERZBICKI, Tomasz.", "12:30 - 14:00", ""]: + "Send the IBAN and BIC to HR, finance and IT.", "12:30 - 14:00", ""]: with self.subTest(text=text): self.assertIsNone(tasks._cased_variant(text)) + def test_a_surname_first_in_capitals_before_a_given_name_is_title_cased(self): + # Records write "SURNAME, Given"; in capitals the model takes the surname for a place, or misses it. + self.assertEqual(tasks._cased_variant("Customer: WIERZBICKI, Tomasz. Delivery city: Gdansk."), + "Customer: Wierzbicki, Tomasz. Delivery city: Gdansk.") + # Capitals before a comma and a lower-case word, or without a comma, are not that convention. + self.assertIsNone(tasks._cased_variant("Ask HR, finance or IT; the WHO report is due.")) + def test_only_the_sentences_that_need_it_change(self): self.assertEqual(tasks._cased_variant("Lena Okafor called. ask priya about lisbon!\nDO NOT REPLY"), "Lena Okafor called. Ask Priya About Lisbon!\nDo Not Reply") From ac647f02feffe852faf829b02baa09e00b8bca5b Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:21:28 +0300 Subject: [PATCH 09/13] test: keep the more confident of two model readings of a span, and measure each threshold with its own job The text read as written called "WIERZBICKI" a place at 0.71 and its title-cased copy called it a person at 0.998, and the merge kept whichever came first. Of two model candidates with the same extent the more confident must win, while a rule still outranks the model. A low threshold lets weaker spans into that merge, where they can displace stronger ones, so the measurement must ask for each threshold in its own job instead of filtering one low-threshold job. --- tests/python/test_jobs.py | 14 ++++++++++++++ tests/python/test_ner_measure.py | 15 ++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/python/test_jobs.py b/tests/python/test_jobs.py index 7fdf2b3..551ce25 100644 --- a/tests/python/test_jobs.py +++ b/tests/python/test_jobs.py @@ -272,6 +272,20 @@ def test_overlapping_candidates_are_merged_not_dropped(self): [(0, 21, "PER"), (30, 40, "IPV4")]) + def test_of_two_model_readings_of_one_span_the_more_confident_is_kept(self): + # The text read as written calls "WIERZBICKI" a place at 0.71; its title-cased copy, a person at 0.998. + model = len(tasks.RULES) + merged = _merge([ + {"start": 10, "end": 20, "label": "LOC", "source": "model:LOC", "score": 0.71, "priority": model}, + {"start": 10, "end": 20, "label": "PER", "source": "model:PER", "score": 0.998, "priority": model}]) + self.assertEqual([(span["label"], span["score"]) for span in merged], [("PER", 0.998)]) + # A rule still outranks the model, however confident the model is. + merged = _merge([ + {"start": 0, "end": 12, "label": "PER", "source": "model:PER", "score": 0.99, "priority": model}, + {"start": 0, "end": 12, "label": "PHONE", "source": "rule:phone", "score": 0.8, "priority": 5}]) + self.assertEqual([span["label"] for span in merged], ["PHONE"]) + + class NerAggregationTests(unittest.TestCase): """Drives the BIO aggregation with a stub window so it is covered without the model.""" diff --git a/tests/python/test_ner_measure.py b/tests/python/test_ner_measure.py index 3d2de4b..418f7fa 100644 --- a/tests/python/test_ner_measure.py +++ b/tests/python/test_ner_measure.py @@ -89,20 +89,21 @@ def test_a_wrong_label_is_neither_exact_nor_overlapping_but_still_masks(self): self.assertEqual(measure.score([(0, 5, "PER")], [(0, 5, "ORG")]), {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 1}) - def test_only_the_model_s_spans_at_the_threshold_are_predictions(self): + def test_only_the_model_s_spans_are_predictions(self): spans = [{"start": 0, "end": 3, "label": "PER", "source": "model:PER", "score": 0.9}, {"start": 4, "end": 7, "label": "LOC", "source": "model:LOC", "score": 0.6}, {"start": 9, "end": 13, "label": "EMAIL", "source": "rule:email", "score": 1.0}] - self.assertEqual(measure.predictions(spans, 0.85), [(0, 3, "PER")]) - self.assertEqual(measure.predictions(spans, 0.5), [(0, 3, "PER"), (4, 7, "LOC")]) + self.assertEqual(measure.predictions(spans), [(0, 3, "PER"), (4, 7, "LOC")]) def test_rows_add_up_by_register_and_by_label_for_each_threshold(self): items = [{"register": "informal", "text": "ada in rome", "entities": [["PER", "ada"], ["LOC", "rome"]]}, {"register": "news", "text": "Bo joined Acme.", "entities": [["PER", "Bo"], ["ORG", "Acme"]]}] - spans = [[{"start": 7, "end": 11, "label": "LOC", "source": "model:LOC", "score": 0.6}], - [{"start": 0, "end": 2, "label": "PER", "source": "model:PER", "score": 0.99}, - {"start": 10, "end": 14, "label": "LOC", "source": "model:LOC", "score": 0.9}]] - rows = measure.tally(items, spans, (0.85, 0.5)) + # Each threshold is its own job: a lower one lets weaker spans into the worker's merge, where they can + # displace stronger ones, so filtering one low-threshold job's spans is not the same as asking at 0.85. + news = [{"start": 0, "end": 2, "label": "PER", "source": "model:PER", "score": 0.99}, + {"start": 10, "end": 14, "label": "LOC", "source": "model:LOC", "score": 0.9}] + spans = {0.85: [[], news], 0.5: [[{"start": 7, "end": 11, "label": "LOC", "source": "model:LOC", "score": 0.6}], news]} + rows = measure.tally(items, spans) # Registers keep the sample's order, then the labels follow. self.assertEqual([name for threshold, name, _ in rows if threshold == 0.85], ["informal", "news", "PER", "ORG", "LOC"]) From 180ac8236716b68b1c677fc56e86ed4983f13724 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:23:30 +0300 Subject: [PATCH 10/13] fix: keep the more confident of two model readings of a span, and measure each threshold with its own job The merge now orders candidates of the same extent and source by score, so the title-cased reading of "WIERZBICKI, Tomasz" as a person at 0.998 replaces the as-written reading as a place at 0.71; a rule still outranks the model. The measurement asks for each threshold in its own job again, since a lower one lets weaker spans into the merge, where they can displace stronger ones. On the development set at 0.85, exact recall rose to 0.67 in all capitals and 0.64 in records, with no new false positive. --- scripts/ner_measure.py | 40 +++++++++++++++++++-------------------- worker/ai_bridge/tasks.py | 23 +++++++++++++++------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/scripts/ner_measure.py b/scripts/ner_measure.py index f5138d7..a1e0d0b 100644 --- a/scripts/ner_measure.py +++ b/scripts/ner_measure.py @@ -1,8 +1,8 @@ """Informational precision and recall of the redact task's NER pass on an invented, labelled sample. -Runs every sentence in tests/fixtures/ner-sample.json through the ONNX worker's redact task once, with PER, ORG and -LOC at the lower threshold, and compares the model's spans (rule spans are not NER) with the sample's labels at each -threshold; a threshold only filters spans by their score, so one job per sentence serves both. It asserts +Runs every sentence in tests/fixtures/ner-sample.json through the ONNX worker's redact task with PER, ORG and LOC, +once per threshold, and compares the model's spans (rule spans are not NER) with the sample's labels. Each threshold +is its own job: a lower one lets weaker spans into the worker's merge, where they can displace stronger ones. It asserts only that every job succeeds with spans inside its text, never a score: this is a small invented sample, so the figures describe this model on these registers, not a quality guarantee. The output is a Markdown block for docs/testing.md; progress goes to stderr. @@ -35,9 +35,8 @@ def gold_spans(item: dict) -> list: return [(item["text"].index(text), item["text"].index(text) + len(text), label) for label, text in item["entities"]] -def predictions(spans: list, threshold: float) -> list: - return [(span["start"], span["end"], span["label"]) for span in spans - if span["source"].startswith("model:") and span["score"] >= threshold] +def predictions(spans: list) -> list: + return [(span["start"], span["end"], span["label"]) for span in spans if span["source"].startswith("model:")] def matched(gold: list, predicted: list, same_label: bool) -> int: @@ -82,15 +81,15 @@ def add(total: dict, counts: dict) -> None: total[key] = total.get(key, 0) + value -def tally(items: list, spans: list, thresholds: tuple) -> list: - """Rows of (threshold, register or label, counts): each register in the sample's order, then each label, whose - rows count only that label's gold and predicted spans.""" +def tally(items: list, spans: dict) -> list: + """Rows of (threshold, register or label, counts) from each threshold's spans per item: each register in the + sample's order, then each label, whose rows count only that label's gold and predicted spans.""" registers = list(dict.fromkeys(item["register"] for item in items)) rows = [] - for threshold in thresholds: + for threshold, found_per_item in spans.items(): totals = {name: {} for name in registers + list(LABELS)} - for item, found in zip(items, spans): - gold, predicted = gold_spans(item), predictions(found, threshold) + for item, found in zip(items, found_per_item): + gold, predicted = gold_spans(item), predictions(found) add(totals[item["register"]], score(gold, predicted)) for label in LABELS: add(totals[label], score([g for g in gold if g[2] == label], [p for p in predicted if p[2] == label])) @@ -105,20 +104,21 @@ def main() -> None: raise SystemExit(f"{base} reports backend {health['backend']!r}; NER needs the onnx worker") sample = sample_path(os.environ) items = json.loads(sample.read_text(encoding="utf-8"))["items"] - spans, models = [], set() - for number, item in enumerate(items, start=1): - result = redact(base, item["text"], min(THRESHOLDS)) - models.add(result["model"]) - spans.append(result["spans"]) - print(f"sentence {number}/{len(items)}", file=sys.stderr) + spans, models = {threshold: [] for threshold in THRESHOLDS}, set() + for threshold in THRESHOLDS: + for number, item in enumerate(items, start=1): + result = redact(base, item["text"], threshold) + models.add(result["model"]) + spans[threshold].append(result["spans"]) + print(f"threshold {threshold} sentence {number}/{len(items)}", file=sys.stderr) now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") entities = sum(len(item["entities"]) for item in items) print(f"""Measured {now} UTC, commit `{os.environ.get("BENCH_COMMIT", "unknown")}`, runner: {os.environ.get("BENCH_RUNNER", "unspecified")}, CPU: {cpu_model()}. -Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence at threshold {min(THRESHOLDS)}, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. +Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. | Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | Masked | | --- | --- | --- | --- | --- | --- | --- | --- | --- |""") - for threshold, name, counts in tally(items, spans, THRESHOLDS): + for threshold, name, counts in tally(items, spans): print(f"| {threshold} | {name} | {counts['gold']} | {counts['predicted']} | " f"{rate(counts['exact'], counts['predicted'])} | {rate(counts['exact'], counts['gold'])} | " f"{rate(counts['overlapping'], counts['predicted'])} | {rate(counts['overlapping'], counts['gold'])} | " diff --git a/worker/ai_bridge/tasks.py b/worker/ai_bridge/tasks.py index 98e1424..d5ea5ba 100644 --- a/worker/ai_bridge/tasks.py +++ b/worker/ai_bridge/tasks.py @@ -445,18 +445,25 @@ def _entities(window, probabilities: list, entities: list, min_score: float, tex # A sentence ends at . ! or ? before whitespace, or at a line break. SENTENCE_END = re.compile(r"[.!?](?=\s)|\n") LETTERS = re.compile(r"[^\W\d_]+") +# What follows a surname written first in capitals in records: a comma and a given name ("WIERZBICKI, Tomasz"). +GIVEN_NAME_AFTER = re.compile(r",\s+([^\W\d_]+)") def _cased_variant(text: str): """A copy of text, offset for offset, with each sentence written all in lower case or all in capitals in title case, or None when no sentence is. The model was trained on cased news and finds almost no lower-case name; sentence case and mixed case are left alone, since title-casing "the board approved" makes "Board" an - organisation. A character whose case change would alter its length stays as it is.""" + organisation, except for a surname in capitals before a comma and a given name, the records convention, which + the model otherwise takes for a place or misses. A character whose case change would alter its length stays.""" chars, start = list(text), 0 for end in [match.end() for match in SENTENCE_END.finditer(text)] + [len(text)]: cased = [char for char in text[start:end] if char.isupper() or char.islower()] - if cased and not (any(char.isupper() for char in cased) and any(char.islower() for char in cased)): - for word in LETTERS.finditer(text, start, end): + uncased = bool(cased) and not (any(char.isupper() for char in cased) and any(char.islower() for char in cased)) + for word in LETTERS.finditer(text, start, end): + given = GIVEN_NAME_AFTER.match(text, word.end()) + surname_first = (len(word.group()) > 1 and word.group().isupper() and given is not None + and given.group(1)[0].isupper() and not given.group(1).isupper()) + if uncased or surname_first: for index in range(word.start(), word.end()): changed = text[index].upper() if index == word.start() else text[index].lower() if len(changed) == 1: @@ -478,8 +485,8 @@ def _probabilities(session, names, window) -> list: def _ner(text: str, entities: list, min_score: float, model_dir: str, progress) -> list: # Overlapping 512 piece windows cover long text; offsets stay relative to the whole text. An uncased sentence is - # read again from a title-cased copy with the same offsets; the text's own reading comes first, so it wins where - # both find the same span. + # read again from a title-cased copy with the same offsets; where both readings find the same span, _merge keeps + # the more confident. tokenizer, session, names = _onnx(Path(model_dir) / "redact", 512, stride=64) variant = _cased_variant(text) windows = [] @@ -495,10 +502,12 @@ def _ner(text: str, entities: list, min_score: float, model_dir: str, progress) def _merge(candidates: list) -> list: - # Leftmost first, then longest, then the stronger source. A candidate overlapping the kept + # Leftmost first, then longest, then the stronger source, then the more confident: of the two readings of an + # uncased sentence, a sure "person" beats an unsure "place" for the same words. A candidate overlapping the kept # span extends it instead of being dropped, so no part of either stays visible. spans = [] - for span in sorted(candidates, key=lambda span: (span["start"], span["start"] - span["end"], span["priority"])): + for span in sorted(candidates, + key=lambda span: (span["start"], span["start"] - span["end"], span["priority"], -span["score"])): if spans and span["start"] < spans[-1]["end"]: spans[-1]["end"] = max(spans[-1]["end"], span["end"]) continue From bdeb4215b9a6adccb85d9e80b9f69ce41fa3bd7e Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:29:37 +0300 Subject: [PATCH 11/13] test: require the second NER reading to cover only the rewritten sentences, and the measurement to wait out 429 A cased sentence must be read once and the second reading must cover only the sentences it rewrote, so its spans cannot relabel a sentence that needed no help and a long text with one lower-case line costs one more window, not a second pass over everything. The measurement must wait out a full worker, which two thresholds and a development-set run inside five minutes reach, and stop cleanly if it stays full. --- tests/python/test_jobs.py | 9 +++++++++ tests/python/test_ner_measure.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/python/test_jobs.py b/tests/python/test_jobs.py index 551ce25..50b42dc 100644 --- a/tests/python/test_jobs.py +++ b/tests/python/test_jobs.py @@ -566,6 +566,15 @@ def test_an_uncased_sentence_is_read_again_in_title_case(self): self.assertEqual(spans, [(0, 3), (4, 9)]) # offsets of the original text self.assertEqual(events, [(0, 2), (1, 2), (2, 2)]) + def test_only_the_rewritten_sentences_are_read_again(self): + # The cased sentence is read once; the second reading covers the lower-case one alone, so it costs one more + # window rather than the whole text again, and its spans land at the original offsets. + spans, texts, events = self.run_ner("Ask Priya now. ask nadia") + self.assertEqual(texts, ["Ask Priya now. ask nadia", " Ask Nadia"]) + self.assertIn((15, 18), spans) # "ask" + self.assertIn((19, 24), spans) # "nadia" + self.assertEqual(events[-1], (2, 2)) + def test_cased_text_is_read_once(self): spans, texts, events = self.run_ner("Ask Priya.") self.assertEqual(texts, ["Ask Priya."]) diff --git a/tests/python/test_ner_measure.py b/tests/python/test_ner_measure.py index 418f7fa..c89f1ba 100644 --- a/tests/python/test_ner_measure.py +++ b/tests/python/test_ner_measure.py @@ -118,6 +118,24 @@ def test_rows_add_up_by_register_and_by_label_for_each_threshold(self): self.assertEqual(counts[(0.85, "LOC")], {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 0}) self.assertEqual(counts[(0.85, "PER")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) + def test_a_full_worker_is_waited_out(self): + # Two thresholds over the sample and a development-set run within five minutes exceed the worker's 128 slots. + from urllib.error import HTTPError + + full = HTTPError("http://worker/v1/jobs", 429, "Too Many Requests", {}, None) + done = {"id": "a" * 32, "status": "succeeded", "result": {"model": "m", "text": "x", "spans": []}} + with mock.patch.object(measure, "call", side_effect=[full, full, done]) as call, mock.patch.object(measure.time, "sleep") as sleep: + self.assertEqual(measure.redact("http://worker", "x", 0.85)["spans"], []) + self.assertEqual(call.call_count, 3) + self.assertEqual(sleep.call_count, 2) + + def test_a_worker_that_stays_full_ends_the_run(self): + from urllib.error import HTTPError + + full = HTTPError("http://worker/v1/jobs", 429, "Too Many Requests", {}, None) + with mock.patch.object(measure, "call", side_effect=full), mock.patch.object(measure.time, "sleep"), self.assertRaises(SystemExit): + measure.redact("http://worker", "x", 0.85) + def test_rates_have_two_decimals_and_an_empty_denominator_is_not_a_number(self): self.assertEqual(measure.rate(1, 4), "0.25") self.assertEqual(measure.rate(2, 3), "0.67") From 2d14a59842c013e41bb634e61305a15bb49d49eb Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:35:43 +0300 Subject: [PATCH 12/13] fix: read again only the rewritten sentences, and let the NER measurement wait out a full worker The second reading now covers each run of rewritten sentences alone, its spans shifted back to the text's offsets, so it cannot relabel a sentence that needed no help and a long text with one lower-case line costs a window or two rather than a second pass over everything. Both sets measure exactly as before. This commit's parent, 180ac82, also added the surname-first rule, which its message omits. The measurement waits out HTTP 429 for up to 400 s, since the worker holds finished jobs 300 s by default. --- scripts/ner_measure.py | 17 +++++++++++++++-- worker/ai_bridge/tasks.py | 38 +++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/scripts/ner_measure.py b/scripts/ner_measure.py index a1e0d0b..489edc4 100644 --- a/scripts/ner_measure.py +++ b/scripts/ner_measure.py @@ -16,6 +16,7 @@ import sys import time from pathlib import Path +from urllib.error import HTTPError sys.path.insert(0, str(Path(__file__).resolve().parent)) from benchmark import POLL_S, TERMINAL, call, cpu_model, wait_ready # noqa: E402 (the same protocol helpers) @@ -23,6 +24,8 @@ REPO = Path(__file__).resolve().parents[1] SAMPLE = REPO / "tests" / "fixtures" / "ner-sample.json" LABELS = ("PER", "ORG", "LOC") +# The worker counts finished jobs against its capacity for 300 s by default; wait that out rather than fail. +CAPACITY_WAIT_S, CAPACITY_RETRIES = 10, 40 THRESHOLDS = (0.85, 0.5) # the task's default, and a lower one to show what it trades @@ -63,8 +66,18 @@ def rate(part: int, whole: int) -> str: def redact(base: str, text: str, threshold: float) -> dict: - job = call(base, "POST", "/v1/jobs", {"task": "redact", "timeout_ms": 120000, - "input": {"text": text, "entities": list(LABELS), "min_score": threshold}}) + body = {"task": "redact", "timeout_ms": 120000, + "input": {"text": text, "entities": list(LABELS), "min_score": threshold}} + for attempt in range(CAPACITY_RETRIES + 1): + try: + job = call(base, "POST", "/v1/jobs", body) + break + except HTTPError as error: + if error.code != 429: + raise + if attempt == CAPACITY_RETRIES: + raise SystemExit(f"the worker stayed at capacity for {CAPACITY_RETRIES * CAPACITY_WAIT_S} s") + time.sleep(CAPACITY_WAIT_S) while job["status"] not in TERMINAL: time.sleep(POLL_S) job = call(base, "GET", "/v1/jobs/" + job["id"]) diff --git a/worker/ai_bridge/tasks.py b/worker/ai_bridge/tasks.py index d5ea5ba..3ff1568 100644 --- a/worker/ai_bridge/tasks.py +++ b/worker/ai_bridge/tasks.py @@ -450,12 +450,17 @@ def _entities(window, probabilities: list, entities: list, min_score: float, tex def _cased_variant(text: str): - """A copy of text, offset for offset, with each sentence written all in lower case or all in capitals in title - case, or None when no sentence is. The model was trained on cased news and finds almost no lower-case name; + return _recase(text)[0] + + +def _recase(text: str): + """(copy, ranges): a copy of text, offset for offset, with each sentence written all in lower case or all in + capitals in title case, or (None, []) when no sentence is. The model was trained on cased news and finds almost no lower-case name; sentence case and mixed case are left alone, since title-casing "the board approved" makes "Board" an organisation, except for a surname in capitals before a comma and a given name, the records convention, which - the model otherwise takes for a place or misses. A character whose case change would alter its length stays.""" - chars, start = list(text), 0 + the model otherwise takes for a place or misses. A character whose case change would alter its length stays. + The ranges are the runs of rewritten sentences, the only part a second reading needs.""" + chars, start, ranges = list(text), 0, [] for end in [match.end() for match in SENTENCE_END.finditer(text)] + [len(text)]: cased = [char for char in text[start:end] if char.isupper() or char.islower()] uncased = bool(cased) and not (any(char.isupper() for char in cased) and any(char.islower() for char in cased)) @@ -468,9 +473,14 @@ def _cased_variant(text: str): changed = text[index].upper() if index == word.start() else text[index].lower() if len(changed) == 1: chars[index] = changed + if chars[start:end] != list(text[start:end]): + if ranges and ranges[-1][1] == start: + ranges[-1] = (ranges[-1][0], end) + else: + ranges.append((start, end)) start = end variant = "".join(chars) - return variant if variant != text else None + return (variant, ranges) if variant != text else (None, []) def _probabilities(session, names, window) -> list: @@ -484,19 +494,21 @@ def _probabilities(session, names, window) -> list: def _ner(text: str, entities: list, min_score: float, model_dir: str, progress) -> list: - # Overlapping 512 piece windows cover long text; offsets stay relative to the whole text. An uncased sentence is - # read again from a title-cased copy with the same offsets; where both readings find the same span, _merge keeps - # the more confident. + # Overlapping 512 piece windows cover long text; offsets stay relative to the whole text. Each run of uncased + # sentences is read again from its title-cased copy, and only that run, so a second reading costs a window or two + # and cannot relabel a sentence that needed no help; where both readings find the same span, _merge keeps the + # more confident. tokenizer, session, names = _onnx(Path(model_dir) / "redact", 512, stride=64) - variant = _cased_variant(text) + variant, ranges = _recase(text) windows = [] - for reading in [text] if variant is None else [text, variant]: + for offset, reading in [(0, text)] + [(start, variant[start:end]) for start, end in ranges]: encoding = tokenizer.encode(reading) - windows.extend((reading, window) for window in [encoding, *encoding.overflowing]) + windows.extend((offset, reading, window) for window in [encoding, *encoding.overflowing]) progress(0, len(windows)) candidates = [] - for index, (reading, window) in enumerate(windows): - candidates.extend(_entities(window, _probabilities(session, names, window), entities, min_score, reading)) + for index, (offset, reading, window) in enumerate(windows): + for span in _entities(window, _probabilities(session, names, window), entities, min_score, reading): + candidates.append({**span, "start": span["start"] + offset, "end": span["end"] + offset}) progress(index + 1, len(windows)) return candidates From 69002495d3642a59304155394cbea9a3bbb54667 Mon Sep 17 00:00:00 2001 From: Cristian Girlea Date: Thu, 1 Oct 2026 15:44:00 +0300 Subject: [PATCH 13/13] docs: publish the NER recall after the second reading, and describe it where clients and agents read The testing notes publish the model workflow's table for this pull request, run 36862817554 at 2d14a59: exact recall at 0.85 rose from 0.13 to 0.47 in informal messages and from 0.78 to 0.92 in records, news stayed at 1.00, and 0.63 and 0.97 of entities are masked. They say how the rules were chosen on a separate development set, what still leaks, and that a case-insensitive model is the deeper fix. The protocol describes the second reading, the joined score and the merge order, and the ONNX redact description now says lower-case and all-capitals sentences are read again. --- docs/protocol.md | 2 +- docs/testing.md | 45 ++++++++++++++++++---------------- mcp_server/bridge_mcp/tools.py | 5 ++-- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index 34d4f35..c8590b0 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -25,7 +25,7 @@ Successful reranking returns `{"model":"...","rankings":[{"index":0,"score":1.0} Submit `{"task":"embed","input":{"texts":["..."]}}` with 1–32 texts of 1–8192 characters each, totalling at most 200000 characters. A successful result is `{"model":"...","dimensions":384,"vectors":[[...]]}` with one unit-length vector per text in input order; components are rounded to six decimals, so the dot product of two vectors is their cosine similarity. The service returns vectors and stores nothing; the caller owns any index built from them. The deterministic backend identifies itself as `hashing-bow-not-a-model`: a signed feature hash of word tokens that is stable across processes and ignores word order. The ONNX backend is `sentence-transformers/all-MiniLM-L6-v2` with mean pooling; it truncates input beyond 256 word pieces and is trained on English text. -Submit `{"task":"redact","input":{"text":"...","entities":["PER"],"min_score":0.85}}` with 1–8192 characters of valid Unicode text. `entities` is an optional list of distinct labels among `PER`, `ORG` and `LOC` (default `["PER"]`; `MISC` is never offered) and `min_score` an optional number from 0 to 1 (default 0.85); both shape only the model pass. A successful result is `{"model":"...","text":"...","spans":[{"start":0,"end":5,"label":"EMAIL","source":"rule:email","score":1.0}]}`: the text with each span replaced by `[LABEL]`, and the spans sorted and disjoint, with offsets counted in Unicode code points of the original text. Deterministic rules run on every backend and are tagged `rule:`: `CARD` (any 13–19 digit window of a digit run that passes Luhn, longest first), `IBAN` (mod 97, trimming trailing groups the pattern swallowed; a lower-case one also needs a country code from the IBAN registry, and mixed case is never taken), `EMAIL` (letters of any script in the local part and the domain, and a letter or punycode top-level label), `IPV4` and `IPV6` (any form Python's `ipaddress` parser accepts that holds a digit, not starting inside a word, an IPv4-mapped tail and a `%zone` identifier included, the colon of `address: message` excluded) at score 1.0, and `PHONE` at 0.8 because it is a pattern, not a proof: 7–15 digits with a leading plus or separators, or at least ten bare digits, never a date shape. Known pattern false positives remain: a dotted quadruple of small numbers such as a version string matches `IPV4`, hex words joined by colons that the parser accepts, such as `dead::beef`, and eight colon-separated groups of up to four digits, such as `10:20:30:40:50:60:70:80`, match `IPV6`, and separated digit groups such as amounts can match `PHONE`; filter by `source` and `score` where that matters. On the default backend, or when `entities` is empty, the model name is `rules-only-not-a-model` and no model runs. The ONNX backend adds spans tagged `model:PER`, `model:ORG` or `model:LOC` carrying the model's mean word confidence, from `Xenova/bert-base-NER:int8`, the int8 ONNX conversion of `dslim/bert-base-NER` (CoNLL-2003, English news). Overlapping candidates are merged into one span carrying the leftmost, then longest, then strongest label, so no part of any candidate stays visible. Redaction is assistive: rules cannot see unstructured personal data, and the model misses names unlike its news training data: on an invented sample it found every entity in news-style sentences but about one in eight in informal lower-case messages ([the measurement](testing.md#measured-ner-precision-and-recall-informational)), so it is not a compliance control. +Submit `{"task":"redact","input":{"text":"...","entities":["PER"],"min_score":0.85}}` with 1–8192 characters of valid Unicode text. `entities` is an optional list of distinct labels among `PER`, `ORG` and `LOC` (default `["PER"]`; `MISC` is never offered) and `min_score` an optional number from 0 to 1 (default 0.85); both shape only the model pass. A successful result is `{"model":"...","text":"...","spans":[{"start":0,"end":5,"label":"EMAIL","source":"rule:email","score":1.0}]}`: the text with each span replaced by `[LABEL]`, and the spans sorted and disjoint, with offsets counted in Unicode code points of the original text. Deterministic rules run on every backend and are tagged `rule:`: `CARD` (any 13–19 digit window of a digit run that passes Luhn, longest first), `IBAN` (mod 97, trimming trailing groups the pattern swallowed; a lower-case one also needs a country code from the IBAN registry, and mixed case is never taken), `EMAIL` (letters of any script in the local part and the domain, and a letter or punycode top-level label), `IPV4` and `IPV6` (any form Python's `ipaddress` parser accepts that holds a digit, not starting inside a word, an IPv4-mapped tail and a `%zone` identifier included, the colon of `address: message` excluded) at score 1.0, and `PHONE` at 0.8 because it is a pattern, not a proof: 7–15 digits with a leading plus or separators, or at least ten bare digits, never a date shape. Known pattern false positives remain: a dotted quadruple of small numbers such as a version string matches `IPV4`, hex words joined by colons that the parser accepts, such as `dead::beef`, and eight colon-separated groups of up to four digits, such as `10:20:30:40:50:60:70:80`, match `IPV6`, and separated digit groups such as amounts can match `PHONE`; filter by `source` and `score` where that matters. On the default backend, or when `entities` is empty, the model name is `rules-only-not-a-model` and no model runs. The ONNX backend adds spans tagged `model:PER`, `model:ORG` or `model:LOC` carrying the model's mean word confidence, from `Xenova/bert-base-NER:int8`, the int8 ONNX conversion of `dslim/bert-base-NER` (CoNLL-2003, English news). The model was trained on cased text, so a sentence written all in lower case or all in capitals, and a surname in capitals before a comma and a given name, are read a second time from a title-cased copy with the same offsets. Pieces of one name that touch or meet at a hyphen, full stop or apostrophe are joined before the threshold and carry the score of their most confident piece; any other model span carries its mean word confidence. Overlapping candidates are merged into one span carrying the leftmost, then longest, then strongest label, then the more confident of two model readings, so no part of any candidate stays visible. Redaction is assistive: rules cannot see unstructured personal data, and the model misses names unlike its news training data: on an invented sample it masked every entity in news-style sentences and nearly every one in terse records, but about three in five in informal messages ([the measurement](testing.md#measured-ner-precision-and-recall-informational)), so it is not a compliance control. Failures use generic error objects with `code` and `message`. Task errors include `task_failed`, `worker_crashed`, `worker_start_failed` and `deadline_exceeded`. Internal exception text is not sent to clients. HTTP contract failures return 400, missing auth 401, missing routes/jobs 404, a connection idle for five seconds while its body is read 408 (a stall while reading headers closes the connection without a response), oversized bodies 413 and capacity exhaustion 429. diff --git a/docs/testing.md b/docs/testing.md index a03b810..10c1574 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -42,7 +42,7 @@ PHP branch support was verified locally on 2026-09-17 on PHP 8.2.33, 8.3.33, 8.4 | Layer | Scope and counting | | --- | --- | -| Python worker | 88 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation and batched cross-encoder scoring driven by stubs, the NER measurement's sample and scoring, progress bounds, and the model-name table every result reports from | +| Python worker | 109 tests: lifecycle, validation and HTTP, process crashes, cancellation, timeouts, retention, allocation/exit races, redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading and the merge driven by stubs, batched cross-encoder scoring, the NER measurement's sample and scoring, progress bounds, and the model-name table every result reports from | | MCP server | 119 tests, offline with hash-pinned wheels: the protocol client against an in-process worker, every tool through the SDK's in-memory client with test tasks enabled, the entry point as a subprocess, result validators on synthetic data, the index builder (chunking, the format and each corruption it refuses, building against the worker, caching, its command line) `bridge_search` (the model gate, path confinement, reloads, rerank on and off), the reachability of every answer in the LLM evaluation, that a default search sees only part of its corpus and that its two-step question takes two steps, that the demo worker reaches none of the semantic evaluation's answers, that a tool waits out a full worker, before each of a search's jobs and within one budget, and says so when it stays full, the evaluation runner's command, isolation checks, scoring and per-backend launcher and index check without starting `claude`, a check that the server's model names match the worker's, that every path parameter says where it resolves and every position in a result says what it counts, and a check that every timing claim in a tool description is a measured median | | PHP contracts + HTTP | 134 checks per PHP version; this **includes** the 101 contract checks, not 134 + 101 | | FrankenPHP | 12 jobs submitted by six concurrent test threads through a reused PHP worker, a 512-document `top_k` request, embedding and redaction round trips, and invalid-input, missing-job and multibyte boundary checks | @@ -63,7 +63,7 @@ The original prototype was not entirely developed with TDD. Review corrections u - No full Symfony kernel or Laravel application/Octane integration test, including configuration caching and multi-user authorization. - No fresh consumer Composer-install smoke test or Packagist release yet; repository examples use a small local autoloader. - No sustained load/soak benchmark, throughput claim, p95/p99 latency, warm-model latency, measured memory-per-job limit or CPU cost per inference; the only timing figures are the cold-job medians below. -- No representative ranking-quality evaluation (for example NDCG/MRR), multilingual accuracy evaluation, NER quality on real data or GPU benchmark; the NER figures below come from 48 invented sentences. +- No representative ranking-quality evaluation (for example NDCG/MRR), multilingual accuracy evaluation, NER quality on real data or GPU benchmark; the NER figures below come from 48 invented sentences by the same author as the development set used to tune the redact task. - No repeated or semantic LLM evaluation. The one recorded run below is a single session per question per model on the demo worker, so it shows that the tools can be used, not a success rate or retrieval quality. - No verified native Windows/macOS service execution or ARM runtime; Docker tests target Linux amd64/Python 3.12. - No durable delivery, restart recovery, automatic retry, idempotency or multi-instance result routing. Jobs and results live in memory. @@ -101,33 +101,36 @@ docker compose -f docker/compose.yaml --profile model down ## Measured NER precision and recall (informational) -`scripts/ner_measure.py` runs every sentence of `tests/fixtures/ner-sample.json` through the ONNX worker's redact task with PER, ORG and LOC, at the task's default threshold and a lower one, and compares the model's spans with the sample's labels; rule spans are not counted. The model workflow prints a fresh table to its job summary on every run, and fails only if a job fails, never on a score. The table below is from that workflow, run 36856834447 on pull request #17 at branch head `f37290b`; the commit in its header is GitHub's merge ref. The sample is invented and small, so these figures describe this pinned model on these three registers, not a quality guarantee. +`scripts/ner_measure.py` runs every sentence of `tests/fixtures/ner-sample.json` through the ONNX worker's redact task with PER, ORG and LOC, at the task's default threshold and a lower one, each its own job, and compares the model's spans with the sample's labels; rule spans are not counted. The model workflow prints a fresh table to its job summary on every run, and fails only if a job fails, never on a score. The table below is from that workflow, run 36862817554 on pull request #18 at branch head `2d14a59`; the commit in its header is GitHub's merge ref. The sample is invented and small, so these figures describe this pinned model on these three registers, not a quality guarantee. -Measured 2026-10-01 UTC, commit `2d80a9f0a8c31ac78891dcde140f487a2259efd3`, runner: GitHub-hosted ubuntu-latest, CPU: AMD EPYC 7763 64-Core Processor. -Model `Xenova/bert-base-NER:int8`; 48 invented sentences with 104 labelled entities; one redact job per sentence at threshold 0.5, entities PER, ORG and LOC, and a higher threshold keeps the spans whose score reaches it. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. +Measured 2026-10-01 UTC, commit `0fe5726cbd3cf1a94825a4f3d196494d8a966e32`, runner: GitHub-hosted ubuntu-latest, CPU: AMD EPYC 7763 64-Core Processor. +Model `Xenova/bert-base-NER:int8`; `tests/fixtures/ner-sample.json`, 48 invented sentences with 104 labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. -| Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | -| --- | --- | --- | --- | --- | --- | --- | --- | -| 0.85 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | -| 0.85 | informal | 30 | 5 | 0.80 | 0.13 | 1.00 | 0.17 | -| 0.85 | records | 36 | 35 | 0.80 | 0.78 | 0.89 | 0.86 | -| 0.85 | PER | 42 | 30 | 0.80 | 0.57 | 0.93 | 0.67 | -| 0.85 | ORG | 24 | 18 | 0.94 | 0.71 | 0.94 | 0.71 | -| 0.85 | LOC | 38 | 30 | 0.97 | 0.76 | 0.97 | 0.76 | -| 0.5 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | -| 0.5 | informal | 30 | 7 | 0.57 | 0.13 | 0.71 | 0.17 | -| 0.5 | records | 36 | 38 | 0.79 | 0.83 | 0.87 | 0.92 | -| 0.5 | PER | 42 | 32 | 0.78 | 0.60 | 0.91 | 0.69 | -| 0.5 | ORG | 24 | 20 | 0.90 | 0.75 | 0.90 | 0.75 | -| 0.5 | LOC | 38 | 31 | 0.94 | 0.76 | 0.94 | 0.76 | +| Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | Masked | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 0.85 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 | +| 0.85 | informal | 30 | 19 | 0.74 | 0.47 | 0.84 | 0.53 | 0.63 | +| 0.85 | records | 36 | 35 | 0.94 | 0.92 | 0.94 | 0.92 | 0.97 | +| 0.85 | PER | 42 | 35 | 0.94 | 0.79 | 1.00 | 0.83 | 0.83 | +| 0.85 | ORG | 24 | 21 | 0.90 | 0.79 | 0.90 | 0.79 | 0.79 | +| 0.85 | LOC | 38 | 36 | 0.92 | 0.87 | 0.92 | 0.87 | 0.87 | +| 0.5 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 | +| 0.5 | informal | 30 | 26 | 0.73 | 0.63 | 0.81 | 0.70 | 0.87 | +| 0.5 | records | 36 | 36 | 0.94 | 0.94 | 0.94 | 0.94 | 1.00 | +| 0.5 | PER | 42 | 38 | 0.95 | 0.86 | 1.00 | 0.90 | 0.90 | +| 0.5 | ORG | 24 | 24 | 0.88 | 0.88 | 0.88 | 0.88 | 0.88 | +| 0.5 | LOC | 38 | 38 | 0.89 | 0.89 | 0.89 | 0.89 | 0.89 | -The model found every entity in news-style sentences, the register of its CoNLL-2003 training text. In informal messages it found only the capitalised names: every lower-case name, place and company was missed, and lowering the threshold to 0.5 added only wrong spans, because the model labels those words as no entity at all. In terse records it found about four in five, and 0.5 found a few more at a small cost in precision. In a local run the records it missed were a name written surname first in capitals (`WIERZBICKI, Tomasz`), names cut into pieces at an initial or a hyphen, `A. Okonkwo` with only its initial tagged, so the surname would stay visible, and a clinic and a city with each other's labels. +Since pull request #18, the redact task reads a sentence written all in lower case or all in capitals a second time, from a title-cased copy with the same offsets, and likewise a surname in capitals before a comma and a given name (`WIERZBICKI, Tomasz`), the records convention. It joins pieces of one name that touch or meet at a hyphen, full stop or apostrophe before the threshold, scored by the most confident piece, so the initial of `A. Okonkwo` carries the surname; and of two readings of the same span it keeps the more confident. Those rules were chosen on a separate development set, `tests/fixtures/ner-dev.json`, so this sample stayed unseen until the final measurement; both are small and were written by the same author, so the figures describe these sentences, not informal text in general. -int8 inference is not bit-identical across CPUs. A local run on an AMD Ryzen 9 9950X matched the news and informal rows at 0.85 but differed by a span or two in records and ORG, and at 0.5 it predicted two fewer wrong spans in informal messages, so read a difference of a span or two between runners as noise. The redact tool's description quotes these figures; re-measure and update both together. To reproduce locally after acquiring the models: +Against the run before those changes on the same runner type, exact recall at 0.85 rose from 0.13 to 0.47 in informal messages and from 0.78 to 0.92 in records, and news stayed at 1.00; the share of entities masked under any label, which is what a redaction hides, is 0.63 in informal messages and 0.97 in records, and 0.87 and 1.00 at 0.5. What still leaks: a lower-case name in a sentence that also holds a capital (`Regards, olumide`), a handle (`@nadia`) and some lower-case first names side by side (`ines and pedro`). A title-cased reading also takes a filler word into a name (`lol tomasz`) or calls a person an organisation, which still hides the name. The deeper fix would be a case-insensitive NER model, which would need its own pinning and measurement. + +int8 inference is not bit-identical across CPUs: a local run on an AMD Ryzen 9 9950X has differed from GitHub's EPYC runners by a span or two in records and ORG, so read such differences as noise. The redact tool's description quotes these figures; re-measure and update both together. To reproduce locally after acquiring the models: ```sh docker compose -f docker/compose.yaml up -d model-worker docker compose -f docker/compose.yaml run --rm --no-deps -T model-ner +NER_SAMPLE=tests/fixtures/ner-dev.json docker compose -f docker/compose.yaml run --rm --no-deps -T model-ner ``` ## LLM evaluation (informational) diff --git a/mcp_server/bridge_mcp/tools.py b/mcp_server/bridge_mcp/tools.py index fc0970b..e5dfe58 100644 --- a/mcp_server/bridge_mcp/tools.py +++ b/mcp_server/bridge_mcp/tools.py @@ -59,8 +59,9 @@ # The recall figures come from scripts/ner_measure.py's table in docs/testing.md; re-measure and update both. "redact": ("Xenova/bert-base-NER:int8", "checksum and pattern rules plus dslim/bert-base-NER (the " "int8 ONNX conversion published as Xenova/bert-base-NER) for PER, ORG and LOC. Trained on " - "English news: on an invented sample it found every entity in news-style sentences, about four in " - "five in terse records and about one in eight in informal lower-case messages (docs/testing.md)."), + "cased English news, so lower-case and all-capitals sentences are read again in title case: on an " + "invented sample it masked every entity in news-style sentences, nearly every one in terse records " + "and about three in five in informal messages (docs/testing.md)."), }, }