Entity Disposition
diff --git a/src/anonymizer/interface/results.py b/src/anonymizer/interface/results.py
index 900653a6..60a88b19 100644
--- a/src/anonymizer/interface/results.py
+++ b/src/anonymizer/interface/results.py
@@ -64,6 +64,13 @@ class AnonymizerResult(_DisplayMixin):
mode was used. Set by ``run()`` / ``preview()``; consumed by
``evaluate()`` to dispatch the rewrite judges. Mutually exclusive
with ``replace_method``.
+ strict_entity_protection: Whether the rewrite ran with strict entity
+ protection. Set by ``run()`` / ``preview()``; consumed by
+ ``evaluate()`` so the entity-coverage judge scores in strict mode
+ (no benefit-of-the-doubt for missed quasi-identifiers).
+ data_summary: Optional dataset context supplied with the original input.
+ Preserved for ``evaluate()`` so entity-coverage judging uses the
+ same context as detection.
"""
dataframe: pd.DataFrame
@@ -72,6 +79,9 @@ class AnonymizerResult(_DisplayMixin):
failed_records: list[FailedRecord]
replace_method: ReplaceMethod | None = None
rewrite_config: PrivacyGoal | None = None
+ entity_labels: list[str] | None = None
+ strict_entity_protection: bool = False
+ data_summary: str | None = None
_display_cycle_index: int = field(default=0, init=False, repr=False)
def __repr__(self) -> str:
@@ -105,6 +115,13 @@ class PreviewResult(_DisplayMixin):
rewrite_config: The privacy goal that produced this preview when rewrite
mode was used. Set by ``preview()``; consumed by ``evaluate()`` to
dispatch the rewrite judges. Mutually exclusive with ``replace_method``.
+ strict_entity_protection: Whether the rewrite ran with strict entity
+ protection. Set by ``preview()``; consumed by ``evaluate()`` so the
+ entity-coverage judge scores in strict mode (no benefit-of-the-doubt
+ for missed quasi-identifiers).
+ data_summary: Optional dataset context supplied with the original input.
+ Preserved for ``evaluate()`` so entity-coverage judging uses the
+ same context as detection.
"""
dataframe: pd.DataFrame
@@ -114,6 +131,9 @@ class PreviewResult(_DisplayMixin):
preview_num_records: int
replace_method: ReplaceMethod | None = None
rewrite_config: PrivacyGoal | None = None
+ entity_labels: list[str] | None = None
+ strict_entity_protection: bool = False
+ data_summary: str | None = None
_display_cycle_index: int = field(default=0, init=False, repr=False)
def __repr__(self) -> str:
diff --git a/tests/conftest.py b/tests/conftest.py
index 69f2c6c2..f1179e23 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -133,6 +133,7 @@ def stub_slim_model_selection() -> ModelSelection:
repairer="known",
),
evaluate=EvaluateModelSelection(
+ entity_coverage_judge="known",
detection_validity_judge="known",
replace_type_fidelity_judge="known",
replace_relational_consistency_judge="known",
diff --git a/tests/engine/test_entity_coverage_judge.py b/tests/engine/test_entity_coverage_judge.py
new file mode 100644
index 00000000..6be3a872
--- /dev/null
+++ b/tests/engine/test_entity_coverage_judge.py
@@ -0,0 +1,207 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from unittest.mock import Mock
+
+import pytest
+
+from anonymizer.config.models import EvaluateModelSelection
+from anonymizer.engine.evaluation.entity_coverage_judge import (
+ EntityCoverageWorkflow,
+ _coverage_prompt,
+ _filter_covered_leaked_entities,
+ _is_leaked_value_covered,
+ _parse_leaked_entities,
+)
+
+
+def test_coverage_prompt_omits_data_summary_context_when_summary_absent() -> None:
+ without_summary = _coverage_prompt(entity_labels=None, strict_entity_protection=False)
+ with_blank_summary = _coverage_prompt(
+ entity_labels=None,
+ strict_entity_protection=False,
+ data_summary=" ",
+ )
+
+ assert without_summary == with_blank_summary
+ assert "
" not in without_summary
+
+
+def test_coverage_prompt_includes_data_summary_as_interpretive_context() -> None:
+ prompt = _coverage_prompt(
+ entity_labels=["first_name"],
+ strict_entity_protection=False,
+ data_summary="Customer support transcripts.",
+ )
+
+ assert "\nCustomer support transcripts.\n" in prompt
+ assert "Use this context only to interpret literal values and their semantic types." in prompt
+ assert "Do not infer or invent entities that are absent from the original text." in prompt
+
+
+def test_filter_covered_leaked_entities_removes_subspans_and_composites() -> None:
+ detected = [
+ {"value": "Mstr Marzella", "label": "givenname"},
+ {"value": "Nawabganj", "label": "city"},
+ {"value": "382210", "label": "zipcode"},
+ {"value": "44 Dunsfold Drive", "label": "street"},
+ {"value": "Chihuahuan Desert", "label": "location"},
+ {"value": "Annex Building", "label": "place_name"},
+ ]
+ leaked = [
+ {"value": "Mstr", "label": "title"}, # subspan of a single final
+ {"value": "Nawabganj - 382210", "label": "city"}, # composite of two whole finals
+ {"value": "44", "label": "buildingnum"}, # short subspan
+ # "Chihuahuan Desert Festival" adds the content token "festival" on top of the
+ # detected "Chihuahuan Desert" — a named event, so it is a real leak (NOT covered).
+ {"value": "Chihuahuan Desert Festival", "label": "event"},
+ {"value": "m", "label": "sex"}, # short token, not covered
+ {"value": "Ann", "label": "first_name"}, # partial token of "Annex", not covered
+ {"value": "uncovered value", "label": "unique_id"},
+ ]
+
+ assert _filter_covered_leaked_entities(leaked, detected) == [
+ {"value": "Chihuahuan Desert Festival", "label": "event"},
+ {"value": "m", "label": "sex"},
+ {"value": "Ann", "label": "first_name"},
+ {"value": "uncovered value", "label": "unique_id"},
+ ]
+
+
+@pytest.mark.parametrize(
+ ("leaked_value", "final_values"),
+ [
+ ("Mstr", ["Mstr Marzella"]), # subspan of a single final entity
+ ("the Nawabganj", ["Nawabganj"]), # grammatical stopword ignored
+ ("44", ["44 Dunsfold Drive"]), # short numeric subspan
+ ("White House", ["White House Road"]), # contiguous, in-order multi-token subspan
+ ("Nawabganj - 382210", ["Nawabganj", "382210"]), # composite of whole finals
+ ("Nawabganj", ["Nawabganj", "382210"]), # exact match against one final
+ ("José", ["José García"]), # accented subspan (Unicode tokenizer)
+ ("Zürich", ["Zürich"]), # accented exact match
+ ],
+)
+def test_is_leaked_value_covered_true(leaked_value: str, final_values: list[str]) -> None:
+ assert _is_leaked_value_covered(leaked_value, final_values) is True
+
+
+@pytest.mark.parametrize(
+ ("leaked_value", "final_values"),
+ [
+ # Cross-entity: pieces come from unrelated final entities -> a real, distinct leak.
+ ("John Smith", ["John Doe", "Jane Smith"]),
+ # Reverse / non-contiguous order within a SINGLE final entity: shared tokens are
+ # NOT enough — order and adjacency are required, so a reversed span is not covered.
+ ("John Doe", ["Doe John"]),
+ ("Ann Lee", ["Lee Ann Boulevard"]),
+ ("John Doe", ["Doe John Memorial Highway"]),
+ # Content descriptor is NOT ignored: a named event is a distinct leak.
+ ("Davos Summit", ["Davos"]),
+ ("Chihuahuan Desert Festival", ["Chihuahuan Desert"]),
+ # Partial-token substrings must NOT count as covered (no raw substring matching).
+ ("Ann", ["Annex Building"]),
+ ("Sara", ["Sarah Connor"]),
+ ("ana", ["Banana Republic"]),
+ # Short-token safeguard: a single letter is not covered by a longer token it prefixes.
+ ("m", ["Mstr Marzella"]),
+ # Nothing in common.
+ ("uncovered value", ["Mstr Marzella", "Nawabganj"]),
+ # No final entities -> nothing can be covered.
+ ("Alice", []),
+ ],
+)
+def test_is_leaked_value_covered_false(leaked_value: str, final_values: list[str]) -> None:
+ assert _is_leaked_value_covered(leaked_value, final_values) is False
+
+
+def test_filter_covered_leaked_entities_keeps_cross_entity_reconstruction() -> None:
+ """A leak whose tokens are spread across unrelated final entities is a real leak."""
+ detected = [
+ {"value": "John Doe", "label": "first_name"},
+ {"value": "Jane Smith", "label": "first_name"},
+ ]
+ leaked = [{"value": "John Smith", "label": "first_name"}]
+
+ assert _filter_covered_leaked_entities(leaked, detected) == leaked
+
+
+def test_filter_covered_leaked_entities_passthrough_on_no_final_entities() -> None:
+ leaked = [{"value": "Alice", "label": "first_name"}]
+
+ assert _filter_covered_leaked_entities(leaked, []) == leaked
+ assert _filter_covered_leaked_entities(leaked, None) == leaked
+
+
+def test_parse_leaked_entities_accepts_gemma_thought_prefixed_json_fence() -> None:
+ raw = """thought
+```json
+{
+ "leaked_entities": [
+ {
+ "value": "Alice",
+ "label": "givenname",
+ "reasoning": "The given name was not detected."
+ }
+ ]
+}
+```"""
+
+ assert _parse_leaked_entities(raw) == [
+ {
+ "value": "Alice",
+ "label": "givenname",
+ "reasoning": "The given name was not detected.",
+ }
+ ]
+
+
+def test_coverage_prompt_extracts_independently_before_deterministic_filtering() -> None:
+ prompt = _coverage_prompt(entity_labels=["sex", "title"], strict_entity_protection=False)
+
+ assert "Work independently from the anonymizer" in prompt
+ assert "deterministic postprocessing step" in prompt
+ assert "" not in prompt
+ assert "_final_entities_for_coverage_judge" not in prompt
+
+
+def test_coverage_prompt_requires_systematic_structured_text_scan() -> None:
+ prompt = _coverage_prompt(entity_labels=["sex", "title"], strict_entity_protection=False)
+
+ assert "salutations, signatures" in prompt
+ assert "Tables, bullets, forms" in prompt
+ assert "Short or single-token values" in prompt
+ assert "Honorifics attached to person names" in prompt
+
+
+def _stub_evaluate_selection() -> EvaluateModelSelection:
+ return EvaluateModelSelection(
+ entity_coverage_judge="nemotron-super",
+ detection_validity_judge="gpt-oss-120b",
+ replace_type_fidelity_judge="gpt-oss-120b",
+ replace_relational_consistency_judge="gpt-oss-120b",
+ replace_attribute_fidelity_judge="gpt-oss-120b",
+ rewrite_judge="nemotron-30b-thinking",
+ )
+
+
+def test_column_config_builds_prompt_and_resolves_model() -> None:
+ """Smoke-guard the real prompt-build path: a broken ``_coverage_prompt`` signature
+ (e.g. a stray required parameter) must fail loudly here, since ``run_non_critical``
+ swallows exceptions downstream and would otherwise mask it as ``entity_coverage=None``.
+ """
+ workflow = EntityCoverageWorkflow(
+ adapter=Mock(),
+ entity_labels=["sex", "title"],
+ strict_entity_protection=True,
+ data_summary="Customer support transcripts.",
+ )
+
+ config = workflow.column_config(_stub_evaluate_selection())
+
+ assert config.model_alias == "nemotron-super"
+ # Prompt built without error and carries the instance-specific context.
+ assert isinstance(config.prompt, str) and config.prompt
+ assert "Customer support transcripts." in config.prompt # data_summary threaded in
+ assert "sex, title" in config.prompt # entity_labels scope threaded in
diff --git a/tests/engine/test_replace_runner.py b/tests/engine/test_replace_runner.py
index adc4d970..27ca62b8 100644
--- a/tests/engine/test_replace_runner.py
+++ b/tests/engine/test_replace_runner.py
@@ -15,9 +15,10 @@
from anonymizer.engine.constants import (
COL_ATTRIBUTE_FIDELITY_JUDGE,
COL_ATTRIBUTE_FIDELITY_VALID,
- COL_DETECTION_JUDGE,
COL_DETECTION_VALID,
COL_ENTITIES_BY_VALUE,
+ COL_ENTITY_COVERAGE,
+ COL_ENTITY_COVERAGE_JUDGE,
COL_FINAL_ENTITIES,
COL_RELATIONAL_CONSISTENCY_JUDGE,
COL_RELATIONAL_CONSISTENCY_VALID,
@@ -167,7 +168,7 @@ def test_evaluate_uses_merged_dd_workflow_for_judges(
)
judge_defaults = {
- COL_DETECTION_JUDGE: {"all_valid": True, "invalid_entities": []},
+ COL_ENTITY_COVERAGE_JUDGE: {"leaked_entities": []},
COL_TYPE_FIDELITY_JUDGE: {"all_valid": True, "invalid_replacements": []},
COL_RELATIONAL_CONSISTENCY_JUDGE: {"all_consistent": True, "relations": []},
COL_ATTRIBUTE_FIDELITY_JUDGE: {"all_valid": True, "entities": []},
@@ -210,9 +211,11 @@ def fake_attach_ids(df: pd.DataFrame) -> pd.DataFrame:
call_columns = adapter.run_workflow.call_args.kwargs["columns"]
assert {c.name for c in call_columns} == set(judge_defaults)
- # And each judge's VALID column ended up on the result, with True (default payload above).
+ # And each judge's output column ended up on the result.
+ # Entity coverage is a float (1.0 = full coverage); replace judges use bool True.
+ assert COL_ENTITY_COVERAGE in result.dataframe.columns
+ assert result.dataframe[COL_ENTITY_COVERAGE].iloc[0] == 1.0
for col in (
- COL_DETECTION_VALID,
COL_TYPE_FIDELITY_VALID,
COL_RELATIONAL_CONSISTENCY_VALID,
COL_ATTRIBUTE_FIDELITY_VALID,
@@ -221,6 +224,55 @@ def fake_attach_ids(df: pd.DataFrame) -> pd.DataFrame:
assert bool(result.dataframe[col].iloc[0]) is True
+def test_evaluate_threads_entity_labels_and_data_summary_into_coverage_prompt(
+ stub_model_configs: list[ModelConfig],
+ stub_evaluate_model_selection: EvaluateModelSelection,
+) -> None:
+ """Replace-mode ``evaluate()`` must forward ``entity_labels`` and ``data_summary``
+ all the way into the coverage judge's prompt (the same context the rewrite path
+ supplies), so the judge scopes and interprets leaks against the run's taxonomy.
+ """
+ saved_trace = pd.DataFrame(
+ {
+ COL_TEXT: ["Alice works at Acme"],
+ COL_FINAL_ENTITIES: [{"entities": []}],
+ COL_REPLACED_TEXT: ["Maya works at NovaCorp"],
+ COL_REPLACEMENT_MAP: [{"replacements": []}],
+ COL_ENTITIES_BY_VALUE: [{"entities_by_value": []}],
+ }
+ )
+
+ def fake_run_workflow(df: pd.DataFrame, *, columns, **_: object) -> WorkflowRunResult:
+ out = df.copy()
+ for column in columns:
+ out[column.name] = [{"leaked_entities": []}] * len(out)
+ return WorkflowRunResult(dataframe=out, failed_records=[])
+
+ def fake_attach_ids(df: pd.DataFrame) -> pd.DataFrame:
+ out = df.copy()
+ out[RECORD_ID_COLUMN] = [f"id-{i}" for i in range(len(out))]
+ return out
+
+ adapter = Mock()
+ adapter.run_workflow.side_effect = fake_run_workflow
+ adapter._attach_record_ids.side_effect = fake_attach_ids
+
+ runner = ReplacementWorkflow(adapter=adapter)
+ runner.evaluate(
+ saved_trace,
+ replace_method=Redact(),
+ model_configs=stub_model_configs,
+ selected_models=stub_evaluate_model_selection,
+ entity_labels=["first_name", "organization"],
+ data_summary="Employee HR records.",
+ )
+
+ call_columns = adapter.run_workflow.call_args.kwargs["columns"]
+ coverage_col = next(c for c in call_columns if c.name == COL_ENTITY_COVERAGE_JUDGE)
+ assert "first_name, organization" in coverage_col.prompt
+ assert "Employee HR records." in coverage_col.prompt
+
+
def test_evaluate_preserves_all_rows_when_llm_drops_some(
stub_model_configs: list[ModelConfig],
stub_evaluate_model_selection: EvaluateModelSelection,
@@ -256,7 +308,7 @@ def test_evaluate_preserves_all_rows_when_llm_drops_some(
)
judge_payload = {
- COL_DETECTION_JUDGE: {"all_valid": True, "invalid_entities": []},
+ COL_ENTITY_COVERAGE_JUDGE: {"leaked_entities": []},
COL_TYPE_FIDELITY_JUDGE: {"all_valid": True, "invalid_replacements": []},
COL_RELATIONAL_CONSISTENCY_JUDGE: {"all_consistent": True, "relations": []},
COL_ATTRIBUTE_FIDELITY_JUDGE: {"all_valid": True, "entities": []},
@@ -298,10 +350,12 @@ def fake_run_workflow(df: pd.DataFrame, *, columns, **_: object) -> WorkflowRunR
# Row count is preserved end-to-end.
assert len(result.dataframe) == 2
- # First row got a real verdict.
- assert bool(result.dataframe[COL_DETECTION_VALID].iloc[0]) is True
+ # First row got a real verdict (entity_coverage is a float, replace judges are bool).
+ assert result.dataframe[COL_ENTITY_COVERAGE].iloc[0] == 1.0
+ assert bool(result.dataframe[COL_TYPE_FIDELITY_VALID].iloc[0]) is True
# Second row (LLM-dropped) is surfaced as Unavailable, not dropped.
- assert result.dataframe[COL_DETECTION_VALID].iloc[1] is None
+ # Float column: None becomes NaN in pandas, so use pd.isna instead of `is None`.
+ assert pd.isna(result.dataframe[COL_ENTITY_COVERAGE].iloc[1])
assert result.dataframe[COL_TYPE_FIDELITY_VALID].iloc[1] is None
assert result.dataframe[COL_RELATIONAL_CONSISTENCY_VALID].iloc[1] is None
assert result.dataframe[COL_ATTRIBUTE_FIDELITY_VALID].iloc[1] is None
diff --git a/tests/engine/test_rewrite_workflow.py b/tests/engine/test_rewrite_workflow.py
index 9d85f676..6e7a824c 100644
--- a/tests/engine/test_rewrite_workflow.py
+++ b/tests/engine/test_rewrite_workflow.py
@@ -438,6 +438,7 @@ def test_detection_judge_partial_row_loss_preserves_all_rows(
model_configs=stub_model_configs,
selected_models=stub_evaluate_model_selection,
privacy_goal=_PRIVACY_GOAL,
+ compute_detection_validity=True,
)
assert len(result.dataframe) == 2
@@ -939,6 +940,7 @@ def test_evaluate_produces_detection_valid_column(
model_configs=stub_model_configs,
selected_models=stub_evaluate_model_selection,
privacy_goal=_PRIVACY_GOAL,
+ compute_detection_validity=True,
)
assert COL_DETECTION_VALID in result.dataframe.columns
@@ -976,6 +978,7 @@ def test_evaluate_skips_passthrough_rows(
model_configs=stub_model_configs,
selected_models=stub_evaluate_model_selection,
privacy_goal=_PRIVACY_GOAL,
+ compute_detection_validity=True,
)
detection_call_df = wf._detection_judge_wf.evaluate.call_args.args[0]
@@ -1010,6 +1013,7 @@ def test_evaluate_passthrough_rows_get_none_judge_defaults(
model_configs=stub_model_configs,
selected_models=stub_evaluate_model_selection,
privacy_goal=_PRIVACY_GOAL,
+ compute_detection_validity=True,
)
passthrough_result = result.dataframe[
diff --git a/tests/interface/test_anonymizer_interface.py b/tests/interface/test_anonymizer_interface.py
index f307a609..d5cbd809 100644
--- a/tests/interface/test_anonymizer_interface.py
+++ b/tests/interface/test_anonymizer_interface.py
@@ -12,12 +12,13 @@
from data_designer.config.models import ModelConfig
from anonymizer import RunConfig
-from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, Rewrite
+from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, EvaluateConfig, Rewrite
from anonymizer.config.models import ModelSelection, ReplaceModelSelection
from anonymizer.config.replace_strategies import Redact, Substitute
from anonymizer.engine.constants import (
COL_DETECTED_ENTITIES,
COL_DETECTION_VALID,
+ COL_ENTITIES_BY_VALUE,
COL_FINAL_ENTITIES,
COL_JUDGE_EVALUATION,
COL_REPLACED_TEXT,
@@ -896,15 +897,50 @@ def test_evaluate_rewrite_result_adds_detection_valid(stub_input: AnonymizerInpu
"needs_human_review": [False],
COL_JUDGE_EVALUATION: [None],
COL_DETECTION_VALID: [0.9],
+ COL_ENTITIES_BY_VALUE: [{}],
}
)
rewrite_runner.evaluate.return_value = RewriteResult(dataframe=eval_df, failed_records=[])
- evaluated = anonymizer.evaluate(run_result)
+ evaluated = anonymizer.evaluate(run_result, config=EvaluateConfig(compute_detection_validity=True))
assert COL_DETECTION_VALID in evaluated.dataframe.columns
+def test_evaluate_config_detection_validity_defaults_off() -> None:
+ """Goal 2: detection-validity scoring is opt-in — the config default must be False
+ so a plain ``EvaluateConfig()`` never triggers the extra detection judge."""
+ assert EvaluateConfig().compute_detection_validity is False
+
+
+def test_evaluate_defaults_skip_detection_validity_in_runner_call(stub_input: AnonymizerInput) -> None:
+ """With the default ``EvaluateConfig()``, evaluate() must forward
+ ``compute_detection_validity=False`` to the rewrite runner (the judge is not run)."""
+ config = AnonymizerConfig(rewrite=Rewrite())
+ anonymizer, _, _, rewrite_runner = _make_anonymizer()
+
+ run_result = anonymizer.run(config=config, data=stub_input)
+
+ eval_df = pd.DataFrame(
+ {
+ COL_TEXT: ["Alice works at Acme"],
+ COL_REWRITTEN_TEXT: ["Beth works at Globex"],
+ "utility_score": [0.85],
+ "leakage_mass": [0.3],
+ "weighted_leakage_rate": [0.23],
+ "any_high_leaked": [False],
+ "needs_human_review": [False],
+ COL_JUDGE_EVALUATION: [None],
+ COL_ENTITIES_BY_VALUE: [{}],
+ }
+ )
+ rewrite_runner.evaluate.return_value = RewriteResult(dataframe=eval_df, failed_records=[])
+
+ anonymizer.evaluate(run_result, config=EvaluateConfig())
+
+ assert rewrite_runner.evaluate.call_args.kwargs["compute_detection_validity"] is False
+
+
def test_evaluate_rewrite_raises_without_rewrite_config() -> None:
"""evaluate() must raise ValueError when result has no rewrite_config and no replace_method."""
anonymizer, _, _, _ = _make_anonymizer()
@@ -990,3 +1026,107 @@ def test_evaluate_rewrite_calls_validate_with_check_rewrite_false(stub_input: An
"evaluate() on a rewrite result must pass check_rewrite=False to avoid "
"requiring rewrite pipeline model aliases that are unused during evaluation"
)
+
+
+# ---------------------------------------------------------------------------
+# Tests: strict_entity_protection flows config -> result -> coverage judge
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("strict", [True, False])
+def test_run_persists_strict_entity_protection(stub_input: AnonymizerInput, strict: bool) -> None:
+ """run() must copy config.rewrite.strict_entity_protection onto the result."""
+ config = AnonymizerConfig(rewrite=Rewrite(strict_entity_protection=strict))
+ anonymizer, _, _, _ = _make_anonymizer()
+
+ result = anonymizer.run(config=config, data=stub_input)
+
+ assert result.strict_entity_protection is strict
+
+
+@pytest.mark.parametrize("strict", [True, False])
+def test_preview_persists_strict_entity_protection(stub_input: AnonymizerInput, strict: bool) -> None:
+ """preview() must copy config.rewrite.strict_entity_protection onto the PreviewResult."""
+ config = AnonymizerConfig(rewrite=Rewrite(strict_entity_protection=strict))
+ anonymizer, _, _, _ = _make_anonymizer()
+
+ preview = anonymizer.preview(config=config, data=stub_input, num_records=1)
+
+ assert preview.strict_entity_protection is strict
+
+
+def test_evaluate_passes_strict_entity_protection_to_coverage_judge(stub_input: AnonymizerInput) -> None:
+ """evaluate() must forward the result's strict flag into EntityCoverageWorkflow.
+
+ Regression: strict_entity_protection was dropped after run()/preview(), so the
+ entity-coverage judge always scored in non-strict mode regardless of config.
+ """
+ config = AnonymizerConfig(rewrite=Rewrite(strict_entity_protection=True))
+ anonymizer, _, _, rewrite_runner = _make_anonymizer()
+
+ run_result = anonymizer.run(config=config, data=stub_input)
+
+ eval_df = pd.DataFrame(
+ {
+ COL_TEXT: ["Alice works at Acme"],
+ COL_REWRITTEN_TEXT: ["Beth works at Globex"],
+ "utility_score": [0.85],
+ "leakage_mass": [0.3],
+ "weighted_leakage_rate": [0.23],
+ "any_high_leaked": [False],
+ "needs_human_review": [False],
+ COL_JUDGE_EVALUATION: [None],
+ COL_DETECTION_VALID: [1.0],
+ }
+ )
+ rewrite_runner.evaluate.return_value = RewriteResult(dataframe=eval_df, failed_records=[])
+
+ with patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as mock_coverage_wf:
+ mock_coverage_wf.return_value.run_non_critical.return_value = (eval_df, [])
+ anonymizer.evaluate(run_result)
+
+ assert mock_coverage_wf.call_args is not None, "EntityCoverageWorkflow was not constructed"
+ assert mock_coverage_wf.call_args.kwargs["strict_entity_protection"] is True
+
+
+def test_run_and_preview_persist_data_summary(stub_input: AnonymizerInput) -> None:
+ """run()/preview() must preserve input context for later evaluation."""
+ data = stub_input.model_copy(update={"data_summary": "Customer support transcripts."})
+ config = AnonymizerConfig(rewrite=Rewrite())
+ anonymizer, _, _, _ = _make_anonymizer()
+
+ result = anonymizer.run(config=config, data=data)
+ preview = anonymizer.preview(config=config, data=data, num_records=1)
+
+ assert result.data_summary == "Customer support transcripts."
+ assert preview.data_summary == "Customer support transcripts."
+
+
+def test_evaluate_passes_data_summary_to_coverage_judge(stub_input: AnonymizerInput) -> None:
+ """evaluate() must forward the input summary to EntityCoverageWorkflow."""
+ data = stub_input.model_copy(update={"data_summary": "Customer support transcripts."})
+ config = AnonymizerConfig(rewrite=Rewrite())
+ anonymizer, _, _, rewrite_runner = _make_anonymizer()
+ run_result = anonymizer.run(config=config, data=data)
+
+ eval_df = pd.DataFrame(
+ {
+ COL_TEXT: ["Alice works at Acme"],
+ COL_REWRITTEN_TEXT: ["Beth works at Globex"],
+ "utility_score": [0.85],
+ "leakage_mass": [0.3],
+ "weighted_leakage_rate": [0.23],
+ "any_high_leaked": [False],
+ "needs_human_review": [False],
+ COL_JUDGE_EVALUATION: [None],
+ COL_DETECTION_VALID: [1.0],
+ }
+ )
+ rewrite_runner.evaluate.return_value = RewriteResult(dataframe=eval_df, failed_records=[])
+
+ with patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as mock_coverage_wf:
+ mock_coverage_wf.return_value.run_non_critical.return_value = (eval_df, [])
+ evaluated = anonymizer.evaluate(run_result)
+
+ assert mock_coverage_wf.call_args.kwargs["data_summary"] == "Customer support transcripts."
+ assert evaluated.data_summary == "Customer support transcripts."
diff --git a/tests/interface/test_anonymizer_logging.py b/tests/interface/test_anonymizer_logging.py
index 26a52b21..3aaa3edb 100644
--- a/tests/interface/test_anonymizer_logging.py
+++ b/tests/interface/test_anonymizer_logging.py
@@ -6,22 +6,30 @@
import logging
import re
from pathlib import Path
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
import pandas as pd
import pytest
-from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, Rewrite
-from anonymizer.config.replace_strategies import Redact
+from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, EvaluateConfig, Rewrite
+from anonymizer.config.replace_strategies import Redact, Substitute
from anonymizer.engine.constants import (
+ COL_ATTRIBUTE_FIDELITY_VALID,
COL_DETECTED_ENTITIES,
+ COL_DETECTION_VALID,
+ COL_ENTITIES_BY_VALUE,
+ COL_ENTITY_COVERAGE,
COL_FINAL_ENTITIES,
+ COL_JUDGE_EVALUATION,
+ COL_RELATIONAL_CONSISTENCY_VALID,
COL_REPLACED_TEXT,
COL_REWRITTEN_TEXT,
COL_TEXT,
+ COL_TYPE_FIDELITY_VALID,
)
from anonymizer.engine.detection.detection_workflow import EntityDetectionResult, EntityDetectionWorkflow
-from anonymizer.engine.ndd.adapter import FailedRecord
+from anonymizer.engine.evaluation.entity_coverage_judge import EntityCoverageWorkflow
+from anonymizer.engine.ndd.adapter import RECORD_ID_COLUMN, FailedRecord, NddAdapter
from anonymizer.engine.replace.replace_runner import ReplacementResult, ReplacementWorkflow
from anonymizer.engine.rewrite.rewrite_workflow import RewriteResult, RewriteWorkflow
from anonymizer.interface.anonymizer import Anonymizer
@@ -64,6 +72,9 @@ def _make_logging_anonymizer(
)
replace_runner = Mock(spec=ReplacementWorkflow)
replace_runner.run.return_value = ReplacementResult(dataframe=_replace_df, failed_records=replace_failures or [])
+ _replace_eval_df = _replace_df.copy()
+ _replace_eval_df[COL_ENTITY_COVERAGE] = [1.0, 1.0]
+ replace_runner.evaluate.return_value = ReplacementResult(dataframe=_replace_eval_df, failed_records=[])
_rewrite_df = pd.DataFrame(
{
COL_TEXT: ["Alice works at Acme", "Bob likes cats"],
@@ -76,6 +87,14 @@ def _make_logging_anonymizer(
)
rewrite_runner = Mock(spec=RewriteWorkflow)
rewrite_runner.run.return_value = RewriteResult(dataframe=_rewrite_df, failed_records=[])
+ _rewrite_eval_df = _rewrite_df.copy()
+ _rewrite_eval_df[COL_ENTITIES_BY_VALUE] = [
+ {"entities_by_value": [{"value": "Alice", "labels": ["first_name"]}]},
+ {"entities_by_value": [{"value": "Bob", "labels": ["first_name"]}]},
+ ]
+ _rewrite_eval_df[COL_JUDGE_EVALUATION] = [{"privacy": {"score": "high"}}] * 2
+ _rewrite_eval_df[COL_ENTITY_COVERAGE] = [1.0, 1.0]
+ rewrite_runner.evaluate.return_value = RewriteResult(dataframe=_rewrite_eval_df, failed_records=[])
return Anonymizer(
detection_workflow=detection_workflow,
replace_runner=replace_runner,
@@ -154,6 +173,231 @@ def test_preview_logs_preview_mode(stub_input: AnonymizerInput, caplog: pytest.L
assert "👀 Preview mode: 📂 Loaded 2 records" in messages
+def test_evaluate_replace_logs_stages(stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(replace=Redact()), data=stub_input)
+ caplog.clear()
+
+ with caplog.at_level(logging.INFO, logger="anonymizer"):
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "🧪 Running Redact evaluation on 2 records" in messages
+ assert "⚖️ Running replace judges" in messages
+ assert "📋 Replace judges complete" in messages
+ assert re.search(r"Replace judges complete.*\[\d+\.\ds\]", messages), "Replace evaluation timing not found"
+ assert "🎉 Evaluation complete — 2 records processed" in messages
+ assert re.search(r"Evaluation complete.*\[\d+\.\ds\]", messages), "Total evaluation timing not found"
+
+
+def test_evaluate_rewrite_logs_stages(stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(rewrite=Rewrite()), data=stub_input)
+ caplog.clear()
+
+ with (
+ patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as coverage_workflow,
+ caplog.at_level(logging.INFO, logger="anonymizer"),
+ ):
+ coverage_workflow.return_value.run_non_critical.return_value = (
+ anonymizer._rewrite_runner.evaluate.return_value.dataframe,
+ [],
+ )
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "🧪 Running rewrite evaluation on 2 records" in messages
+ assert "⚖️ Running rewrite judges" in messages
+ assert "📋 Rewrite judges complete" in messages
+ assert re.search(r"Rewrite judges complete.*\[\d+\.\ds\]", messages), "Rewrite evaluation timing not found"
+ assert "🔎 Running entity coverage" in messages
+ assert "📋 Entity coverage complete" in messages
+ assert re.search(r"Entity coverage complete.*\[\d+\.\ds\]", messages), "Coverage timing not found"
+ assert "🎉 Evaluation complete — 2 records processed" in messages
+
+
+def test_evaluate_rewrite_logs_unavailable_entity_coverage(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(rewrite=Rewrite()), data=stub_input)
+ coverage_df = anonymizer._rewrite_runner.evaluate.return_value.dataframe.copy()
+ coverage_df[COL_ENTITY_COVERAGE] = None
+ caplog.clear()
+
+ with (
+ patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as coverage_workflow,
+ caplog.at_level(logging.INFO, logger="anonymizer"),
+ ):
+ coverage_workflow.return_value.run_non_critical.return_value = (coverage_df, [])
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "Entity coverage score unavailable for 2/2 records" in messages
+ assert "📋 Entity coverage complete" in messages
+ assert "🎉 Evaluation complete — 2 records processed" in messages
+
+
+def test_evaluate_debug_logs_config_and_failures_without_sensitive_context(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(replace=Redact()), data=stub_input)
+ run_result.data_summary = "confidential dataset description"
+ anonymizer._replace_runner.evaluate.return_value = ReplacementResult(
+ dataframe=anonymizer._replace_runner.evaluate.return_value.dataframe,
+ failed_records=[FailedRecord(record_id="r1", step="entity-coverage-judge", reason="timeout")],
+ )
+ caplog.clear()
+
+ with caplog.at_level(logging.DEBUG, logger="anonymizer"):
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "evaluation config: mode=Redact" in messages
+ assert "data_summary_provided=True" in messages
+ assert "active evaluation judges: entity_coverage_judge" in messages
+ assert "evaluation models:" in messages
+ assert "1 evaluation failed record(s)" in messages
+ assert "r1 (entity-coverage-judge: timeout)" in messages
+ assert "confidential dataset description" not in messages
+
+
+def test_evaluate_logs_unavailable_scores_for_all_active_replace_judges(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(replace=Substitute()), data=stub_input)
+ unavailable_df = anonymizer._replace_runner.evaluate.return_value.dataframe.copy()
+ for column in (
+ COL_ENTITY_COVERAGE,
+ COL_DETECTION_VALID,
+ COL_TYPE_FIDELITY_VALID,
+ COL_RELATIONAL_CONSISTENCY_VALID,
+ COL_ATTRIBUTE_FIDELITY_VALID,
+ ):
+ unavailable_df[column] = None
+ anonymizer._replace_runner.evaluate.return_value = ReplacementResult(
+ dataframe=unavailable_df,
+ failed_records=[],
+ )
+ caplog.clear()
+
+ with caplog.at_level(logging.INFO, logger="anonymizer"):
+ anonymizer.evaluate(run_result, config=EvaluateConfig(compute_detection_validity=True))
+
+ messages = caplog.text
+ assert (
+ "Replace evaluation scores unavailable — entity coverage: 2/2; detection validity: 2/2; "
+ "type fidelity: 2/2; relational consistency: 2/2; attribute fidelity: 2/2."
+ ) in messages
+ assert "📋 Replace judges complete" in messages
+ assert "🎉 Evaluation complete — 2 records processed" in messages
+
+
+def test_evaluate_rewrite_warns_when_rewrite_judge_is_unavailable_but_coverage_succeeds(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(rewrite=Rewrite()), data=stub_input)
+ evaluation_df = anonymizer._rewrite_runner.evaluate.return_value.dataframe.copy()
+ evaluation_df[COL_JUDGE_EVALUATION] = [{"privacy": {"score": "high"}}, None]
+ anonymizer._rewrite_runner.evaluate.return_value = RewriteResult(
+ dataframe=evaluation_df,
+ failed_records=[FailedRecord(record_id="r2", step="rewrite-final-judge", reason="timeout")],
+ )
+ caplog.clear()
+
+ with (
+ patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as coverage_workflow,
+ caplog.at_level(logging.INFO, logger="anonymizer"),
+ ):
+ coverage_workflow.return_value.run_non_critical.return_value = (evaluation_df, [])
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "Rewrite judge score unavailable for 1/2 records" in messages
+ assert "Entity coverage score unavailable" not in messages
+
+
+def test_evaluate_rewrite_does_not_warn_for_expected_no_entity_passthrough(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(rewrite=Rewrite()), data=stub_input)
+ evaluation_df = anonymizer._rewrite_runner.evaluate.return_value.dataframe.copy()
+ evaluation_df[COL_ENTITIES_BY_VALUE] = [
+ {"entities_by_value": []},
+ {"entities_by_value": [{"value": "Bob", "labels": ["first_name"]}]},
+ ]
+ evaluation_df[COL_JUDGE_EVALUATION] = [None, {"privacy": {"score": "high"}}]
+ anonymizer._rewrite_runner.evaluate.return_value = RewriteResult(dataframe=evaluation_df, failed_records=[])
+ caplog.clear()
+
+ with (
+ patch("anonymizer.interface.anonymizer.EntityCoverageWorkflow") as coverage_workflow,
+ caplog.at_level(logging.INFO, logger="anonymizer"),
+ ):
+ coverage_workflow.return_value.run_non_critical.return_value = (evaluation_df, [])
+ anonymizer.evaluate(run_result)
+
+ assert "Rewrite judge score unavailable" not in caplog.text
+
+
+def test_evaluate_substitute_warns_only_for_the_unavailable_judge(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ anonymizer = _make_logging_anonymizer()
+ run_result = anonymizer.run(config=AnonymizerConfig(replace=Substitute()), data=stub_input)
+ evaluation_df = anonymizer._replace_runner.evaluate.return_value.dataframe.copy()
+ evaluation_df[COL_TYPE_FIDELITY_VALID] = [True, None]
+ evaluation_df[COL_RELATIONAL_CONSISTENCY_VALID] = [True, True]
+ evaluation_df[COL_ATTRIBUTE_FIDELITY_VALID] = [True, True]
+ anonymizer._replace_runner.evaluate.return_value = ReplacementResult(dataframe=evaluation_df, failed_records=[])
+ caplog.clear()
+
+ with caplog.at_level(logging.INFO, logger="anonymizer"):
+ anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "Type fidelity score unavailable for 1/2 records" in messages
+ assert "Replace evaluation scores unavailable" not in messages
+ assert "Entity coverage score unavailable" not in messages
+
+
+def test_evaluate_caught_replace_workflow_exception_logs_debug_and_public_warning(
+ stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture
+) -> None:
+ setup_anonymizer = _make_logging_anonymizer()
+ run_result = setup_anonymizer.run(config=AnonymizerConfig(replace=Redact()), data=stub_input)
+ run_result.trace_dataframe[COL_ENTITIES_BY_VALUE] = [
+ {"entities_by_value": [{"value": "Alice", "labels": ["first_name"]}]},
+ {"entities_by_value": [{"value": "Bob", "labels": ["first_name"]}]},
+ ]
+
+ adapter = Mock(spec=NddAdapter)
+ adapter._attach_record_ids.side_effect = lambda dataframe: dataframe.assign(
+ **{RECORD_ID_COLUMN: [f"r{i}" for i in range(len(dataframe))]}
+ )
+ adapter.run_workflow.side_effect = RuntimeError("simulated workflow outage")
+ anonymizer = Anonymizer(
+ data_designer=Mock(),
+ replace_runner=ReplacementWorkflow(adapter=adapter),
+ )
+ caplog.clear()
+
+ with (
+ patch.object(EntityCoverageWorkflow, "column_config", return_value=Mock()),
+ caplog.at_level(logging.DEBUG, logger="anonymizer"),
+ ):
+ evaluated = anonymizer.evaluate(run_result)
+
+ messages = caplog.text
+ assert "Replace judges workflow failed; evaluation scores may be unavailable." in messages
+ assert "Entity coverage score unavailable for 2/2 records" in messages
+ assert evaluated.trace_dataframe[COL_ENTITY_COVERAGE].isna().all()
+
+
def test_preview_set_preview_num_records_capped(stub_input: AnonymizerInput) -> None:
"""preview() must propagate preview_num_records so downstream workflows use DataDesigner.preview()."""
anonymizer = _make_logging_anonymizer()