SCRUM-1068-explore profile flags a boolean-shaped column with >2 values

CHANGELOG.md

@@ -11,6 +11,28 @@ tag releases both in lockstep, so entries below are keyed by the engine version.
 
 ### Added
 
+- **`explore profile` flags a boolean-shaped column with more than two
+  values** ([#218]). A column named like a two-valued flag (`is_*`, `has_*`,
+  `*_flag`, `*_yn`, `*_ind`) whose content holds more than two distinct
+  non-null values is a mixed-encoding defect, and it went unreported: on the
+  public ADE-bench `helixops_saas.duckdb`, `raw_workspaces.primary_ws_yn`
+  holds five distinct values under a name that promises two. Every
+  `where flag = 'Y'` written against a column like that is quietly wrong for
+  some share of rows, and the share was invisible without asking for the
+  domain directly.
+
+  A data-quality observation now fires when a boolean-ish name and a
+  non-null distinct count above two coincide, naming the encodings and their
+  counts wherever the value domain ([#203]) is already known for that
+  column, and falling back to the count alone where it is not (a column that
+  failed one of that feature's own eligibility gates is not evidence there
+  are only two values, only that the tool cannot name them). A genuinely
+  `BOOLEAN`-typed column is excluded outright, whatever its name: it cannot
+  hold more than two values by construction. The check does not, and is not
+  asked to, tell a data-quality defect apart from a genuine third state; the
+  tool cannot make that call, and naming what it found is what lets the
+  caller make it instead.
+
 - **A PII refusal names the exact override entry that would clear the
   column** ([#217]). The firewall's refusal was correct, and so is a name
   detector flagging `account_name` on a table of companies or a team name in

packages/dex-core/src/exmergo_dex_core/explore/profile.py

@@ -145,6 +145,13 @@
     r"(^|_)(comments?|notes?|message|body|feedback|review_text|bio|about)(_|$)"
 )
 
+# A column name that promises exactly two states (issue #218): the `is_`/`has_`
+# prefixes and `_flag`/`_yn`/`_ind` suffixes a data-quality check reads as
+# "boolean, by convention" even on a connector with no native BOOLEAN type.
+# Matched as a whole underscore-delimited token, same discipline as the PII
+# patterns above, so "is_active" matches but "history"/"island" do not.
+_BOOLEAN_NAME = re.compile(r"(^|_)(is|has|flag|yn|ind)(_|$)")
+
 # "FIXED" is Snowflake's SHOW COLUMNS token for every integer and NUMBER type
 # (surfaced by snowflake._render_type); without it no Snowflake integer column
 # reads as numeric, and the type-aware PII gates below would be inert there.
@@ -549,6 +556,56 @@ def _heterogeneous_key_note(col_name: str, agg: ColumnAggregate | None) -> str |
     )
 
 
+def _boolean_shaped_flag_note(
+    col_name: str,
+    data_type: str,
+    agg: ColumnAggregate | None,
+    domain: ValueDomain | None,
+) -> str | None:
+    """A column named like a two-valued flag (``is_*``, ``has_*``, ``*_flag``,
+    ``*_yn``, ``*_ind``) whose non-null content holds more than two distinct
+    values -- a mixed-encoding defect invisible to anyone who does not ask for
+    the domain (issue #218). Firing does not require telling a data-quality
+    defect apart from a genuine third state; the tool cannot make that call,
+    and naming the count (and the values, when known) is what lets the caller
+    make it instead.
+
+    A genuinely ``BOOLEAN``-typed column can only ever hold two values (plus
+    null) by construction and is excluded outright, whatever its name.
+
+    Where the value domain (#203) is available, the note names the
+    encodings and their counts. Where it is not, because the column failed
+    one of that feature's own eligibility gates (PII-flagged, a candidate
+    key, or too many distinct values to report), the note still fires on the
+    count alone: the tool not being able to name the values is not evidence
+    there are only two of them.
+    """
+
+    if not _BOOLEAN_NAME.search(_normalize(col_name)):
+        return None
+    if any(hint in data_type.upper() for hint in _BOOLEAN_HINTS):
+        return None
+    if agg is None or agg.distinct_count is None or agg.distinct_count <= 2:
+        return None
+
+    if domain is not None and domain.values:
+        total = len(domain.values) + domain.elided
+        encodings = ", ".join(f"{v.value}={v.count}" for v in domain.values)
+        if domain.elided:
+            encodings += f", +{domain.elided} more"
+        return (
+            f"{col_name} looks boolean-shaped by name but holds {total} "
+            f"distinct non-null values: {encodings}; a two-valued read of "
+            "it will misclassify some rows"
+        )
+    marker = "" if agg.distinct_count_exact else "~"
+    return (
+        f"{col_name} looks boolean-shaped by name but holds "
+        f"{marker}{agg.distinct_count} distinct non-null values; a "
+        "two-valued read of it will misclassify some rows"
+    )
+
+
 def _parse_temporal(value: object) -> date | datetime:
     """Normalize a temporal min/max back to a comparable object. Adapters
     disagree about when they call ``json_safe`` on these two fields (see the
@@ -801,6 +858,11 @@ def profile(
                     key_note = _heterogeneous_key_note(col.name, agg)
                     if key_note is not None:
                         data_quality.append(key_note)
+                flag_note = _boolean_shaped_flag_note(
+                    col.name, col.data_type, agg, value_domains.get(col.name)
+                )
+                if flag_note is not None:
+                    data_quality.append(flag_note)
             continuity = (
                 _temporal_continuity(agg) if col.name in temporal_stats else None
             )

packages/dex-core/tests/explore/test_profile.py

@@ -1036,6 +1036,149 @@ def test_heterogeneous_key_end_to_end_real_duckdb(tmp_path: Path):
     assert "comments" not in notes
 
 
+# --- boolean-shaped flag with more than two values (#218) ----------------------
+
+
+def test_boolean_shaped_flag_note_names_the_domain_when_known():
+    """The issue's own worked example: a *_yn column with five values, named
+    with counts once the domain (#203) is known."""
+
+    from exmergo_dex_core.cache import ValueCount, ValueDomain
+    from exmergo_dex_core.explore.profile import _boolean_shaped_flag_note
+
+    agg = _aggregate(name="primary_ws_yn", distinct_count=5, distinct_count_exact=True)
+    domain = ValueDomain(
+        values=[
+            ValueCount(value="Y", count=40),
+            ValueCount(value="N", count=38),
+            ValueCount(value="Yes", count=10),
+            ValueCount(value="No", count=8),
+            ValueCount(value="1", count=4),
+        ]
+    )
+    note = _boolean_shaped_flag_note("primary_ws_yn", "VARCHAR", agg, domain)
+    assert note is not None
+    assert "primary_ws_yn" in note and "5 distinct non-null values" in note
+    assert "Y=40" in note and "No=8" in note
+
+
+@pytest.mark.parametrize(
+    ("col_name", "data_type", "agg_kwargs", "expect_note"),
+    [
+        # A *_yn column with more than two values but no domain known (it
+        # failed one of #203's own eligibility gates) still fires, on the
+        # count alone: not being able to name the values is not evidence
+        # there are only two of them.
+        (
+            "primary_ws_yn",
+            "VARCHAR",
+            {"distinct_count": 5, "distinct_count_exact": True},
+            True,
+        ),
+        # A clean two-value flag: no note.
+        (
+            "has_discount",
+            "VARCHAR",
+            {"distinct_count": 2, "distinct_count_exact": True},
+            False,
+        ),
+        # A three-valued flag: still fires, even though a third value could
+        # be a genuine third state rather than a defect -- the tool cannot
+        # tell those apart, and the acceptance criteria says it should not
+        # try.
+        (
+            "is_active",
+            "VARCHAR",
+            {"distinct_count": 3, "distinct_count_exact": True},
+            True,
+        ),
+        # A genuinely BOOLEAN-typed column: excluded outright, whatever its
+        # distinct count claims (it cannot really exceed two).
+        (
+            "is_active",
+            "BOOLEAN",
+            {"distinct_count": 5},
+            False,
+        ),
+        # A column whose name does not match any boolean-ish pattern: no
+        # note, however many values it holds.
+        (
+            "ws_stat_cd",
+            "VARCHAR",
+            {"distinct_count": 6},
+            False,
+        ),
+        # An approximate (not yet escalated) distinct count still fires,
+        # marked accordingly by the caller-visible "~".
+        (
+            "has_flag",
+            "VARCHAR",
+            {"distinct_count": 4, "distinct_count_exact": False},
+            True,
+        ),
+    ],
+)
+def test_boolean_shaped_flag_note_decisions(
+    col_name, data_type, agg_kwargs, expect_note
+):
+    from exmergo_dex_core.explore.profile import _boolean_shaped_flag_note
+
+    agg = _aggregate(name=col_name, **agg_kwargs)
+    note = _boolean_shaped_flag_note(col_name, data_type, agg, None)
+    if expect_note:
+        assert note is not None and col_name in note
+    else:
+        assert note is None
+
+
+def test_boolean_shaped_flag_note_absent_without_aggregate():
+    from exmergo_dex_core.explore.profile import _boolean_shaped_flag_note
+
+    assert _boolean_shaped_flag_note("is_active", "VARCHAR", None, None) is None
+
+
+def test_boolean_shaped_flag_end_to_end_real_duckdb(tmp_path: Path):
+    """The issue's own four acceptance scenarios, in one table: a *_yn column
+    with five values names them; a clean two-value flag and a genuinely
+    BOOLEAN column stay silent; a three-valued flag still fires."""
+
+    import duckdb
+
+    from exmergo_dex_core.adapters.duckdb import DuckDBAdapter
+    from exmergo_dex_core.explore.profile import profile as profile_fn
+
+    db_path = tmp_path / "boolean_flag.duckdb"
+    conn = duckdb.connect(str(db_path))
+    conn.execute(
+        "CREATE TABLE raw_workspaces ("
+        "primary_ws_yn VARCHAR, has_discount VARCHAR, is_native_bool BOOLEAN, "
+        "onboarding_stage_flag VARCHAR)"
+    )
+    yn_values = ["Y", "N", "Yes", "No", "1"]
+    stage_values = ["new", "active", "churned"]
+    rows = [
+        (yn_values[i % 5], "Y" if i % 2 == 0 else "N", True, stage_values[i % 3])
+        for i in range(100)
+    ]
+    conn.executemany("INSERT INTO raw_workspaces VALUES (?, ?, ?, ?)", rows)
+    conn.close()
+
+    adapter = DuckDBAdapter(path=db_path)
+    try:
+        (dataset,) = profile_fn(adapter, ["boolean_flag.main.raw_workspaces"])
+    finally:
+        adapter.close()
+
+    notes = " ".join(dataset.data_quality)
+    assert "primary_ws_yn looks boolean-shaped by name" in notes
+    assert "5 distinct non-null values" in notes
+    assert "Y=" in notes and "Yes=" in notes
+    assert "onboarding_stage_flag looks boolean-shaped by name" in notes
+    assert "3 distinct non-null values" in notes
+    assert "has_discount" not in notes
+    assert "is_native_bool" not in notes
+
+
 # --- pii_overrides: the durable human decision ---------------------------------