feat: rewrite replace chg and LLM entity augment chg and couple bugs

src/anonymizer/engine/constants.py

@@ -94,7 +94,8 @@
 COL_SENSITIVITY_DISPOSITION = "_sensitivity_disposition"
 COL_SENSITIVITY_DISPOSITION_BLOCK = "_sensitivity_disposition_block"
 COL_REWRITE_DISPOSITION_BLOCK = "_rewrite_disposition_block"
-COL_REPLACEMENT_MAP_FOR_PROMPT = "_replacement_map_for_prompt"
+COL_PREREPLACE_TEXT = "_prereplace_text"
+COL_PREREPLACE_TAGGED_TEXT = "_prereplace_tagged_text"
 COL_FULL_REWRITE = "_full_rewrite"
 COL_MEANING_UNITS = "_meaning_units"
 COL_MEANING_UNITS_SERIALIZED = "_meaning_units_serialized"

src/anonymizer/engine/detection/detection_workflow.py

@@ -682,6 +682,24 @@ def _get_augment_prompt(*, data_summary: str | None, labels: list[str], strict_l
 - Filename Exclusions: The "filename" label should be reserved for user-created documents or data exports (e.g., .pdf, .xlsx, .csv, .txt).
 - Executable Distinction: Do not tag executable binaries (ending in .exe, .dll, or .sys) as filename. Treat these extensions as non-sensitive system identifiers.
 
+Additional extraction requirements:
+- The "value" field must be the EXACT verbatim span from the input text.
+  Copy the text character-for-character exactly as it appears.
+  Do NOT normalize, correct spelling, expand abbreviations, decode encodings,
+  infer hidden values, translate text, reformat numbers, or otherwise modify
+  the extracted span.
+- Extract only text that is explicitly present in the input.
+  Never reconstruct, guess, or generate a value that does not appear verbatim.
+- Identifiers may be disguised, fragmented, hyphenated, misspelled, obfuscated,
+  spaced out, mixed with punctuation, or written in words instead of digits.
+  Detect the identifier and extract the exact text as written.
+  Examples of disguised identifiers to detect:
+  - Phone numbers, SSNs, and credit card numbers spoken as digit words,
+    including "o" or "oh" used in place of zero:
+    "nine o two, five five five, one two three four"
+  - Names spelled out letter by letter with hyphens or commas:
+    "J-O-H-N", "M, A, R, Y"
+
 <<EXAMPLE_BLOCK>>
 
 ---

src/anonymizer/engine/rewrite/repair.py

@@ -18,11 +18,11 @@
     COL_ANY_HIGH_LEAKED,
     COL_LEAKAGE_MASS,
     COL_LEAKED_PRIVACY_ITEMS,
+    COL_PREREPLACE_TEXT,
     COL_PRIVACY_QA,
     COL_PRIVACY_QA_REANSWER,
     COL_REWRITTEN_TEXT,
     COL_REWRITTEN_TEXT_NEXT,
-    COL_TEXT,
     COL_UTILITY_SCORE,
 )
 from anonymizer.engine.ndd.adapter import NddAdapter
@@ -168,7 +168,7 @@ def _render_repair_prompt(row: dict[str, Any], params: RepairParams) -> str:
     replacements = {
         "<<PRIVACY_GOAL>>": params.privacy_goal_str,
         "<<MAX_PRIVACY_LEAK>>": str(params.max_privacy_leak),
-        "<<ORIGINAL_TEXT>>": str(row.get(COL_TEXT, "")),
+        "<<ORIGINAL_TEXT>>": str(row.get(COL_PREREPLACE_TEXT, "")),
         "<<REWRITTEN_TEXT>>": str(row.get(COL_REWRITTEN_TEXT, "")),
         "<<LEAKAGE_MASS>>": str(row.get(COL_LEAKAGE_MASS, 0.0)),
         "<<HIGH_WARN>>": "\nWARNING: HIGH-SENSITIVITY LEAK DETECTED - must be fixed!"
@@ -201,7 +201,7 @@ def _make_repair_column(repairer_alias: str) -> Any:
         required_columns=[
             COL_LEAKED_PRIVACY_ITEMS,
             COL_REWRITTEN_TEXT,
-            COL_TEXT,
+            COL_PREREPLACE_TEXT,
             COL_LEAKAGE_MASS,
             COL_ANY_HIGH_LEAKED,
             COL_UTILITY_SCORE,

src/anonymizer/engine/rewrite/rewrite_generation.py

@@ -14,13 +14,15 @@
 from anonymizer.config.rewrite import PrivacyGoal
 from anonymizer.engine.constants import (
     COL_FULL_REWRITE,
+    COL_PREREPLACE_TAGGED_TEXT,
+    COL_PREREPLACE_TEXT,
     COL_REPLACEMENT_MAP,
-    COL_REPLACEMENT_MAP_FOR_PROMPT,
     COL_REWRITE_DISPOSITION_BLOCK,
     COL_REWRITTEN_TEXT,
     COL_SENSITIVITY_DISPOSITION,
     COL_TAG_NOTATION,
     COL_TAGGED_TEXT,
+    COL_TEXT,
     _jinja,
 )
 from anonymizer.engine.ndd.model_loader import resolve_model_alias
@@ -49,7 +51,7 @@ def _get_rewrite_prompt(privacy_goal: PrivacyGoal, data_summary: str | None = No
 
 <instructions>
 Your task is to rewrite the text below so that it protects the privacy of the entities described,
-following the entity protection rules and replacement map provided. The rewrite must read naturally as
+following the entity protection rules provided. The rewrite must read naturally as
 plain, fluent text — no tags, brackets, or annotation artifacts.
 
 Apply each protection decision consistently across ALL occurrences of the same entity value.
@@ -85,16 +87,8 @@ def _get_rewrite_prompt(privacy_goal: PrivacyGoal, data_summary: str | None = No
 Entities NOT listed above may be kept as-is.
 </sensitivity_disposition>
 
-{% if <<REPLACEMENT_MAP_COL>>.replacements %}
-<replacement_map>
-Synthetic replacement values for entities with protection_method "replace":
-<<REPLACEMENT_MAP>>
-</replacement_map>
-{% endif %}
 <output_requirements>
 Apply each protection method as follows:
-- "replace": Substitute the entity value with the corresponding synthetic value from the replacement map.
-  Use the synthetic value consistently for every occurrence.
 - "generalize": Replace with a broader category or range
   (e.g., a specific city → "a city in the Pacific Northwest", exact age → "in their late 30s").
 - "remove": Omit the detail entirely. Rewrite the surrounding sentence so it reads naturally without it.
@@ -113,10 +107,8 @@ def _get_rewrite_prompt(privacy_goal: PrivacyGoal, data_summary: str | None = No
             "<<PRIVACY_GOAL>>": privacy_goal.to_prompt_string(),
             "<<DATA_CONTEXT>>": data_context_section,
             "<<TAG_NOTATION>>": COL_TAG_NOTATION,
-            "<<TAGGED_TEXT>>": _jinja(COL_TAGGED_TEXT),
+            "<<TAGGED_TEXT>>": _jinja(COL_PREREPLACE_TAGGED_TEXT),
             "<<REWRITE_DISPOSITION_BLOCK>>": COL_REWRITE_DISPOSITION_BLOCK,
-            "<<REPLACEMENT_MAP_COL>>": COL_REPLACEMENT_MAP_FOR_PROMPT,
-            "<<REPLACEMENT_MAP>>": _jinja(COL_REPLACEMENT_MAP_FOR_PROMPT),
         },
     )
 
@@ -128,12 +120,18 @@ def _get_rewrite_prompt(privacy_goal: PrivacyGoal, data_summary: str | None = No
 
 @custom_column_generator(required_columns=[COL_SENSITIVITY_DISPOSITION])
 def _format_rewrite_disposition_block(row: dict[str, Any]) -> dict[str, Any]:
-    """Pre-filter and serialize protected entities (protection_method_suggestion != "leave_as_is") for the rewrite prompt."""
+    """Pre-filter and serialize protected entities for the rewrite prompt.
+
+    Excludes leave_as_is entities and replace entities (the latter are handled
+    programmatically by _apply_direct_replacements before the LLM sees the text).
+    """
     disposition = parse_sensitivity_disposition(row[COL_SENSITIVITY_DISPOSITION])
     block = []
     for e in disposition.sensitivity_disposition:
         if not e.needs_protection:
             continue
+        if e.protection_method_suggestion == "replace":
+            continue
         d = e.model_dump(mode="json")
         block.append(
             {
@@ -148,29 +146,59 @@ def _format_rewrite_disposition_block(row: dict[str, Any]) -> dict[str, Any]:
     return row
 
 
-@custom_column_generator(required_columns=[COL_REPLACEMENT_MAP, COL_REWRITE_DISPOSITION_BLOCK])
-def _filter_replacement_map_for_prompt(row: dict[str, Any]) -> dict[str, Any]:
-    """Keep only replacement entries for entities with protection_method_suggestion='replace'."""
-    disposition_block: list[dict] = row.get(COL_REWRITE_DISPOSITION_BLOCK, [])
+def _get_replace_pairs(row: dict[str, Any]) -> list[tuple[str, str]]:
+    """Return (original, synthetic) pairs for entities with protection_method='replace'."""
+    disposition = parse_sensitivity_disposition(row[COL_SENSITIVITY_DISPOSITION])
     replace_values = {
-        e["entity_value"] for e in disposition_block if e.get("protection_method_suggestion") == "replace"
+        e.entity_value for e in disposition.sensitivity_disposition if e.protection_method_suggestion == "replace"
     }
     raw_map = row.get(COL_REPLACEMENT_MAP)
-    if raw_map is None:
-        if replace_values:
-            logger.warning(
-                "COL_REPLACEMENT_MAP is None but entities require replacement; prompt will have no replacements."
-            )
-        row[COL_REPLACEMENT_MAP_FOR_PROMPT] = {"replacements": []}
-        return row
+    if not raw_map or not replace_values:
+        return []
     raw_map = normalize_payload(raw_map)
     if hasattr(raw_map, "model_dump"):
         raw_map = raw_map.model_dump(mode="python")
     parsed_map = EntityReplacementMapSchema.model_validate(raw_map)
-    filtered = [
-        replacement.model_dump() for replacement in parsed_map.replacements if replacement.original in replace_values
-    ]
-    row[COL_REPLACEMENT_MAP_FOR_PROMPT] = {"replacements": filtered}
+    pairs = [(r.original, r.synthetic) for r in parsed_map.replacements if r.original in replace_values]
+    unmatched = replace_values - {original for original, _ in pairs}
+    if unmatched:
+        logger.warning(
+            "Replace entities have no entry in the replacement map and will pass through unprotected: %s",
+            sorted(unmatched),
+        )
+    return pairs
+
+
+@custom_column_generator(
+    required_columns=[COL_SENSITIVITY_DISPOSITION, COL_REPLACEMENT_MAP, COL_TEXT, COL_TAGGED_TEXT],
+    side_effect_columns=[COL_PREREPLACE_TAGGED_TEXT],
+)
+def _apply_direct_replacements(row: dict[str, Any]) -> dict[str, Any]:
+    """Programmatically replace direct identifier entities before the rewrite LLM call.
+
+    Applies each (original → synthetic) pair as a global string replacement on both
+    the plain text (COL_TEXT → COL_PREREPLACE_TEXT) and the tagged text
+    (COL_TAGGED_TEXT → COL_PREREPLACE_TAGGED_TEXT). Using str.replace() ensures all
+    occurrences are substituted, not just detected spans.
+    """
+    pairs = _get_replace_pairs(row)
+    plain_text = str(row.get(COL_TEXT, ""))
+    tagged_text = str(row.get(COL_TAGGED_TEXT, ""))
+    if not pairs:
+        disposition = parse_sensitivity_disposition(row[COL_SENSITIVITY_DISPOSITION])
+        has_replace_entities = any(
+            e.protection_method_suggestion == "replace" for e in disposition.sensitivity_disposition
+        )
+        if has_replace_entities and not row.get(COL_REPLACEMENT_MAP):
+            logger.warning("COL_REPLACEMENT_MAP is None but entities require replacement; no replacements applied.")
+        row[COL_PREREPLACE_TEXT] = plain_text
+        row[COL_PREREPLACE_TAGGED_TEXT] = tagged_text
+        return row
+    for original, synthetic in sorted(pairs, key=lambda p: len(p[0]), reverse=True):
+        plain_text = plain_text.replace(original, synthetic)
+        tagged_text = tagged_text.replace(original, synthetic)
+    row[COL_PREREPLACE_TEXT] = plain_text
+    row[COL_PREREPLACE_TAGGED_TEXT] = tagged_text
     return row
 
 
@@ -226,8 +254,8 @@ def columns(
                 generator_function=_format_rewrite_disposition_block,
             ),
             CustomColumnConfig(
-                name=COL_REPLACEMENT_MAP_FOR_PROMPT,
-                generator_function=_filter_replacement_map_for_prompt,
+                name=COL_PREREPLACE_TEXT,
+                generator_function=_apply_direct_replacements,
             ),
             LLMStructuredColumnConfig(
                 name=COL_FULL_REWRITE,

src/anonymizer/engine/schemas/__init__.py

@@ -34,7 +34,6 @@
     EntityCategory,
     EntityDispositionSchema,
     EntitySource,
-    MeaningUnitAspect,
     MeaningUnitImportance,
     MeaningUnitSchema,
     MeaningUnitsSchema,
@@ -90,7 +89,6 @@
     "EntityCategory",
     "EntityDispositionSchema",
     "EntitySource",
-    "MeaningUnitAspect",
     "MeaningUnitImportance",
     "MeaningUnitSchema",
     "MeaningUnitsSchema",

src/anonymizer/engine/schemas/detection.py

@@ -157,7 +157,7 @@ class LatentEntitySchema(BaseModel):
     )
     rationale: str = Field(
         min_length=20,
-        max_length=150,
+        max_length=250,
         description="One sentence explaining the inference without adding new facts",
     )
 

src/anonymizer/engine/schemas/rewrite.py

@@ -33,7 +33,7 @@
         Uses LLMJudgeColumnConfig with Score rubrics (no custom schema needed)
 
 Supporting enums: Domain, EntitySource, EntityCategory, SensitivityLevel,
-                  ProtectionMethod, CombinedRiskLevel, MeaningUnitAspect, PrivacyAnswer
+                  ProtectionMethod, CombinedRiskLevel, PrivacyAnswer
 """
 
 from __future__ import annotations
@@ -249,33 +249,14 @@ class StrictSensitivityDispositionSchema(SensitivityDispositionSchema):
 # ---------------------------------------------------------------------------
 
 
-class MeaningUnitAspect(str, Enum):
-    ROLE = "role"
-    PROCESS = "process"
-    RELATIONSHIP = "relationship"
-    ENVIRONMENT = "environment"
-    ROUTINE = "routine"
-    CREATIVE_OUTPUT = "creative_output"
-    VALUE = "value"
-    MOTIVATION = "motivation"
-    INFLUENCE = "influence"
-    AUDIENCE = "audience"
-    LEGAL_BASIS = "legal_basis"
-    INSTITUTION = "institution"
-    JUSTIFICATION = "justification"
-    PROCEDURAL_STATUS = "procedural_status"
-    TEMPORAL_SEQUENCE = "temporal_sequence"
-    RIGHTS_IMPACT = "rights_impact"
-
-
 class MeaningUnitImportance(str, Enum):
     critical = "critical"
     important = "important"
 
 
 class MeaningUnitSchema(BaseModel):
     id: int = Field(ge=1)
-    aspect: MeaningUnitAspect
+    aspect: str
     unit: str = Field(min_length=1)
     importance: MeaningUnitImportance
 

tests/engine/test_qa_generation.py

@@ -34,7 +34,6 @@
     EntityCategory,
     EntityDispositionSchema,
     EntitySource,
-    MeaningUnitAspect,
     MeaningUnitImportance,
     MeaningUnitSchema,
     MeaningUnitsSchema,
@@ -75,13 +74,13 @@
     units=[
         MeaningUnitSchema(
             id=1,
-            aspect=MeaningUnitAspect.ROLE,
+            aspect="role",
             unit="An individual works as a software engineer.",
             importance=MeaningUnitImportance.critical,
         ),
         MeaningUnitSchema(
             id=2,
-            aspect=MeaningUnitAspect.ENVIRONMENT,
+            aspect="environment",
             unit="The individual works remotely.",
             importance=MeaningUnitImportance.important,
         ),