From b164047834a8df2df4a63d65ff40b1e01b7799de Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:10:02 +0100 Subject: [PATCH 1/5] feat: add unique coverage and case-level sufficiency diagnostics --- src/setqca/analysis/__init__.py | 14 ++ src/setqca/analysis/sufficiency.py | 366 +++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 src/setqca/analysis/sufficiency.py diff --git a/src/setqca/analysis/__init__.py b/src/setqca/analysis/__init__.py index f85543d..9c09162 100644 --- a/src/setqca/analysis/__init__.py +++ b/src/setqca/analysis/__init__.py @@ -16,9 +16,23 @@ NecessityCandidate, necessity_analysis, ) +from .sufficiency import ( + CaseDiagnostic, + CaseRole, + SolutionDiagnostics, + TermDiagnostics, + classify_case, + sufficiency_diagnostics, +) __all__ = [ + "CaseDiagnostic", + "CaseRole", "NecessityAnalysis", "NecessityCandidate", + "SolutionDiagnostics", + "TermDiagnostics", + "classify_case", "necessity_analysis", + "sufficiency_diagnostics", ] diff --git a/src/setqca/analysis/sufficiency.py b/src/setqca/analysis/sufficiency.py new file mode 100644 index 0000000..ae65ac1 --- /dev/null +++ b/src/setqca/analysis/sufficiency.py @@ -0,0 +1,366 @@ +"""Case-level diagnostics for a sufficiency solution. + +Parameters of fit summarise a solution in a few numbers. They do not say which +cases produced those numbers, and that is usually the question a researcher +actually has: which cases support this path, which contradict it, and which +outcomes does it fail to explain. + +Case typology +------------- + +For a term ``X`` and outcome ``Y``, with the crossover at 0.5 +(Schneider and Rohlfing 2013): + +- ``X > 0.5``, ``Y > 0.5``, ``X <= Y``: **typical**, supporting the claim. +- ``X > 0.5``, ``Y > 0.5``, ``X > Y``: **deviant for consistency in degree**, + the right corner at the wrong magnitude. +- ``X > 0.5``, ``Y <= 0.5``: **deviant for consistency in kind**. The term holds + and the outcome does not; this is the case-level contradiction. +- ``X <= 0.5``, ``Y > 0.5``: **deviant for coverage**, an outcome this term + does not explain. +- ``X <= 0.5``, ``Y <= 0.5``: **individually irrelevant**, outside both sets. + +Unique coverage +--------------- + +Raw coverage counts outcome membership a term accounts for. Unique coverage +counts only what *no other term* accounts for:: + + covU_i = [ sum(min(Xi, Y)) - sum(min(Xi, max_over_others(Xj), Y)) ] / sum(Y) + +A term with substantial raw coverage but near-zero unique coverage is +redundant in practice: drop it and the solution still explains the same cases. + +References +---------- +Schneider, C. Q. and Rohlfing, I. (2013). Combining QCA and process tracing in +set-theoretic multi-method research. *Sociological Methods & Research* 42(4), +559-597. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd + +from setqca._validation import validate_columns, validate_membership +from setqca.expressions import evaluate_expression, parse_set_expression +from setqca.metrics import SufficiencyFit, sufficiency + +if TYPE_CHECKING: # pragma: no cover - imported for type checking only + from setqca._validation import FloatArray + from setqca.sets import SetExpression + +CROSSOVER = 0.5 + + +class CaseRole(Enum): + """Where a case sits relative to one sufficiency claim.""" + + TYPICAL = "typical" + DEVIANT_CONSISTENCY_IN_DEGREE = "deviant consistency (degree)" + DEVIANT_CONSISTENCY_IN_KIND = "deviant consistency (kind)" + DEVIANT_COVERAGE = "deviant coverage" + INDIVIDUALLY_IRRELEVANT = "individually irrelevant" + + @property + def contradicts_sufficiency(self) -> bool: + """Return whether this role counts against the sufficiency claim.""" + return self in { + CaseRole.DEVIANT_CONSISTENCY_IN_DEGREE, + CaseRole.DEVIANT_CONSISTENCY_IN_KIND, + } + + +def classify_case(term_membership: float, outcome_membership: float) -> CaseRole: + """Classify one case against one sufficiency claim. + + Parameters + ---------- + term_membership : float + Membership of the case in the term. + outcome_membership : float + Membership of the case in the outcome. + + Returns + ------- + CaseRole + The case's role, per the typology in the module docstring. + """ + in_term = term_membership > CROSSOVER + in_outcome = outcome_membership > CROSSOVER + + if in_term and in_outcome: + if term_membership <= outcome_membership: + return CaseRole.TYPICAL + return CaseRole.DEVIANT_CONSISTENCY_IN_DEGREE + if in_term: + return CaseRole.DEVIANT_CONSISTENCY_IN_KIND + if in_outcome: + return CaseRole.DEVIANT_COVERAGE + return CaseRole.INDIVIDUALLY_IRRELEVANT + + +@dataclass(frozen=True, slots=True) +class CaseDiagnostic: + """One case, judged against one term.""" + + case: str + term_membership: float + outcome_membership: float + role: CaseRole + uniquely_covered: bool + + +@dataclass(frozen=True, slots=True) +class TermDiagnostics: + """One solution term, its fit, and every case's relation to it.""" + + expression: str + fit: SufficiencyFit + unique_coverage: float + frequency: int + cases: tuple[CaseDiagnostic, ...] + + def by_role(self, role: CaseRole) -> tuple[str, ...]: + """Return the labels of cases in one role.""" + return tuple(item.case for item in self.cases if item.role is role) + + @property + def typical(self) -> tuple[str, ...]: + """Return cases supporting the claim.""" + return self.by_role(CaseRole.TYPICAL) + + @property + def deviant_consistency(self) -> tuple[str, ...]: + """Return cases contradicting the claim, in kind or in degree.""" + return tuple(item.case for item in self.cases if item.role.contradicts_sufficiency) + + @property + def contradictory(self) -> tuple[str, ...]: + """Return cases where the term holds but the outcome does not.""" + return self.by_role(CaseRole.DEVIANT_CONSISTENCY_IN_KIND) + + @property + def deviant_coverage(self) -> tuple[str, ...]: + """Return outcome cases this term does not reach.""" + return self.by_role(CaseRole.DEVIANT_COVERAGE) + + @property + def uniquely_covered(self) -> tuple[str, ...]: + """Return cases this term covers that no other term does.""" + return tuple(item.case for item in self.cases if item.uniquely_covered) + + @property + def redundant(self) -> bool: + """Return whether the term adds no coverage another term does not already give.""" + return self.unique_coverage <= 0.0 + + +@dataclass(frozen=True, slots=True) +class SolutionDiagnostics: + """Diagnostics for a whole disjunctive solution.""" + + outcome: str + terms: tuple[TermDiagnostics, ...] + fit: SufficiencyFit + + @property + def redundant_terms(self) -> tuple[TermDiagnostics, ...]: + """Return terms contributing no unique coverage.""" + return tuple(term for term in self.terms if term.redundant) + + def to_frame(self) -> pd.DataFrame: + """Return one row per term, with fit and case counts. + + Returns + ------- + pandas.DataFrame + Columns ``term``, ``consistency``, ``PRI``, ``raw_coverage``, + ``unique_coverage``, ``n``, and one count per case role. + """ + return pd.DataFrame( + { + "term": [term.expression for term in self.terms], + "consistency": [term.fit.consistency for term in self.terms], + "PRI": [term.fit.pri for term in self.terms], + "raw_coverage": [term.fit.coverage for term in self.terms], + "unique_coverage": [term.unique_coverage for term in self.terms], + "n": [term.frequency for term in self.terms], + **{ + role.value: [len(term.by_role(role)) for term in self.terms] + for role in CaseRole + }, + } + ) + + def cases_frame(self) -> pd.DataFrame: + """Return one row per case per term, for case-oriented work. + + Returns + ------- + pandas.DataFrame + Columns ``term``, ``case``, ``term_membership``, + ``outcome_membership``, ``role`` and ``uniquely_covered``. + """ + records = [ + { + "term": term.expression, + "case": item.case, + "term_membership": item.term_membership, + "outcome_membership": item.outcome_membership, + "role": item.role.value, + "uniquely_covered": item.uniquely_covered, + } + for term in self.terms + for item in term.cases + ] + return pd.DataFrame.from_records( + records, + columns=[ + "term", + "case", + "term_membership", + "outcome_membership", + "role", + "uniquely_covered", + ], + ) + + def __str__(self) -> str: + lines = [f"Sufficiency diagnostics for {self.outcome}", ""] + for term in self.terms: + lines.append( + f"{term.expression} " + f"[cons={term.fit.consistency:.3f}, cov={term.fit.coverage:.3f}, " + f"uniq={term.unique_coverage:.3f}, n={term.frequency}]" + ) + if term.typical: + lines.append(f" typical: {', '.join(term.typical)}") + if term.contradictory: + lines.append(f" contradictory: {', '.join(term.contradictory)}") + if term.deviant_coverage: + lines.append(f" unexplained outcomes: {', '.join(term.deviant_coverage)}") + if term.redundant: + lines.append(" redundant: adds no coverage beyond the other terms") + return "\n".join(lines) + + +def _memberships( + data: pd.DataFrame, terms: Sequence[str | SetExpression] +) -> list[tuple[str, FloatArray]]: + resolved: list[tuple[str, FloatArray]] = [] + for term in terms: + node = parse_set_expression(term) if isinstance(term, str) else term + resolved.append((str(node), evaluate_expression(node, data))) + return resolved + + +def _unique_coverage( + membership: FloatArray, others: list[FloatArray], outcome: FloatArray +) -> float: + """Return coverage of the outcome that no other term accounts for.""" + total = float(outcome.sum()) + if total == 0.0: + return 0.0 + own = float(np.minimum(membership, outcome).sum()) + if not others: + return own / total + overlap_membership = np.maximum.reduce(others) + shared = float(np.minimum(np.minimum(membership, overlap_membership), outcome).sum()) + return (own - shared) / total + + +def sufficiency_diagnostics( + data: pd.DataFrame, + *, + outcome: str, + terms: Sequence[str | SetExpression], + case_id: str | None = None, +) -> SolutionDiagnostics: + """Diagnose a disjunctive sufficiency solution case by case. + + Parameters + ---------- + data : pandas.DataFrame + Calibrated memberships in ``[0, 1]``. + outcome : str + Name of the outcome column. + terms : sequence of str or SetExpression + The solution's terms. Strings are parsed, so + ``["DEV*URB", "LIT*~IND"]`` works directly. + case_id : str, optional + Column holding case labels. Defaults to the frame index, so no + particular schema is assumed. + + Returns + ------- + SolutionDiagnostics + Per-term fit including unique coverage, and every case's role. + + Raises + ------ + ValueError + If no terms are given, or the data are not calibrated. + KeyError + If a named column is absent. + + Examples + -------- + >>> diagnostics = sufficiency_diagnostics( # doctest: +SKIP + ... data, outcome="SURV", terms=["DEV*URB*LIT*IND*STB"] + ... ) + >>> diagnostics.terms[0].typical # doctest: +SKIP + ('BE', 'CZ', 'NL', 'UK') + """ + if not terms: + raise ValueError("At least one term is required.") + validate_columns(data, [outcome]) + y = validate_membership(data[outcome].to_numpy(), name=outcome) + + if case_id is None: + labels = [str(index) for index in data.index] + else: + validate_columns(data, [case_id]) + labels = [str(value) for value in data[case_id]] + + resolved = _memberships(data, terms) + all_memberships = [membership for _, membership in resolved] + + diagnostics: list[TermDiagnostics] = [] + for position, (expression, membership) in enumerate(resolved): + others = [other for index, other in enumerate(all_memberships) if index != position] + covered_elsewhere = np.maximum.reduce(others) if others else np.zeros_like(membership) + cases = tuple( + CaseDiagnostic( + case=label, + term_membership=float(term_value), + outcome_membership=float(outcome_value), + role=classify_case(float(term_value), float(outcome_value)), + uniquely_covered=bool(term_value > CROSSOVER and other_value <= CROSSOVER), + ) + for label, term_value, outcome_value, other_value in zip( + labels, membership, y, covered_elsewhere, strict=True + ) + ) + diagnostics.append( + TermDiagnostics( + expression=expression, + fit=sufficiency(membership, y), + unique_coverage=_unique_coverage(membership, others, y), + frequency=int(np.sum(membership > CROSSOVER)), + cases=cases, + ) + ) + + overall = np.maximum.reduce(all_memberships) + return SolutionDiagnostics( + outcome=outcome, + terms=tuple(diagnostics), + fit=sufficiency(overall, y), + ) From 93ea28cc1f53eb2706e991cfeff744ad41604531 Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:10:02 +0100 Subject: [PATCH 2/5] feat: export sufficiency diagnostics from the package root --- src/setqca/__init__.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/setqca/__init__.py b/src/setqca/__init__.py index 07b88e9..120cc04 100644 --- a/src/setqca/__init__.py +++ b/src/setqca/__init__.py @@ -9,7 +9,16 @@ from importlib.metadata import PackageNotFoundError from importlib.metadata import version as _version -from .analysis import NecessityAnalysis, NecessityCandidate, necessity_analysis +from .analysis import ( + CaseDiagnostic, + CaseRole, + NecessityAnalysis, + NecessityCandidate, + SolutionDiagnostics, + TermDiagnostics, + necessity_analysis, + sufficiency_diagnostics, +) from .calibration import DirectCalibration, calibrate_crisp, calibrate_direct from .counterfactuals import ( CounterfactualAnalysis, @@ -50,6 +59,8 @@ "CSQCA", "FSQCA", "BooleanSolution", + "CaseDiagnostic", + "CaseRole", "Condition", "Configuration", "CounterfactualAnalysis", @@ -71,7 +82,9 @@ "PrimeImplicantChart", "QCAResult", "SetExpression", + "SolutionDiagnostics", "SufficiencyFit", + "TermDiagnostics", "TruthCode", "TruthTable", "TruthTableRow", @@ -90,4 +103,5 @@ "parse_expression", "simplify_expression", "sufficiency", + "sufficiency_diagnostics", ] From 53049544aad53ae2b3f5527f5f312f06eef886ce Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:10:03 +0100 Subject: [PATCH 3/5] test: add R fixtures for per-term fit and unique coverage --- validation/fixtures/r_qca.json | 107 +++++++++++++++++++++++++++++++ validation/r/generate_fixtures.R | 43 ++++++++++++- 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/validation/fixtures/r_qca.json b/validation/fixtures/r_qca.json index 205364f..2f6c2dd 100644 --- a/validation/fixtures/r_qca.json +++ b/validation/fixtures/r_qca.json @@ -373,6 +373,44 @@ ], "parsimonious": [ ["DEV*~IND", "URB*STB"] + ], + "term_fits": [ + { + "family": "conservative", + "term": "DEV*URB*LIT*IND*STB", + "consistency": 0.9042056074766355, + "pri": 0.8857938718662952, + "raw_coverage": 0.454225352112676, + "unique_coverage": 0.3931924882629108, + "cases": ["BE", "CZ", "NL", "UK"] + }, + { + "family": "conservative", + "term": "DEV*~URB*LIT*~IND*STB", + "consistency": 0.8042704626334523, + "pri": 0.7193877551020412, + "raw_coverage": 0.2652582159624414, + "unique_coverage": 0.2042253521126761, + "cases": ["FI", "IE"] + }, + { + "family": "parsimonious", + "term": "DEV*~IND", + "consistency": 0.8148148148148149, + "pri": 0.7208121827411167, + "raw_coverage": 0.2840375586854459, + "unique_coverage": 0.1936619718309858, + "cases": ["FI", "IE"] + }, + { + "family": "parsimonious", + "term": "URB*STB", + "consistency": 0.8737672583826429, + "pri": 0.8454106280193235, + "raw_coverage": 0.5199530516431925, + "unique_coverage": 0.4295774647887324, + "cases": ["BE", "CZ", "NL", "UK"] + } ] }, { @@ -654,6 +692,26 @@ ], "parsimonious": [ "URB*STB" + ], + "term_fits": [ + { + "family": "conservative", + "term": "DEV*URB*LIT*IND*STB", + "consistency": 0.9042056074766355, + "pri": 0.8857938718662952, + "raw_coverage": 0.454225352112676, + "unique_coverage": null, + "cases": ["BE", "CZ", "NL", "UK"] + }, + { + "family": "parsimonious", + "term": "URB*STB", + "consistency": 0.8737672583826429, + "pri": 0.8454106280193235, + "raw_coverage": 0.5199530516431925, + "unique_coverage": null, + "cases": ["BE", "CZ", "NL", "UK"] + } ] }, { @@ -935,6 +993,35 @@ ], "parsimonious": [ "DEV*STB" + ], + "term_fits": [ + { + "family": "conservative", + "term": "DEV*~URB*LIT*STB", + "consistency": 1, + "pri": 1, + "raw_coverage": 0.5, + "unique_coverage": 0.25, + "cases": ["FI", "IE", "FR", "SE"] + }, + { + "family": "conservative", + "term": "DEV*LIT*IND*STB", + "consistency": 1, + "pri": 1, + "raw_coverage": 0.75, + "unique_coverage": 0.5, + "cases": ["FR", "SE", "BE", "CZ", "NL", "UK"] + }, + { + "family": "parsimonious", + "term": "DEV*STB", + "consistency": 1, + "pri": 1, + "raw_coverage": 1, + "unique_coverage": null, + "cases": ["FI", "IE", "FR", "SE", "BE", "CZ", "NL", "UK"] + } ] }, { @@ -1022,6 +1109,26 @@ ], "parsimonious": [ "DEV*STB" + ], + "term_fits": [ + { + "family": "conservative", + "term": "DEV*LIT*STB", + "consistency": 0.868811881188119, + "pri": 0.8481375358166192, + "raw_coverage": 0.8239436619718311, + "unique_coverage": null, + "cases": ["BE", "CZ", "FI", "FR", "IE", "NL", "SE", "UK"] + }, + { + "family": "parsimonious", + "term": "DEV*STB", + "consistency": 0.868811881188119, + "pri": 0.8481375358166192, + "raw_coverage": 0.8239436619718311, + "unique_coverage": null, + "cases": ["BE", "CZ", "FI", "FR", "IE", "NL", "SE", "UK"] + } ] } ], diff --git a/validation/r/generate_fixtures.R b/validation/r/generate_fixtures.R index fe376e2..2d969c1 100644 --- a/validation/r/generate_fixtures.R +++ b/validation/r/generate_fixtures.R @@ -51,6 +51,46 @@ solution_records <- function(model) { lapply(model$solution, function(solution) as.character(solution)) } +# Per-term fit for both solution families, including unique coverage, which R +# reports as covU and which is the one fit measure setqca did not previously +# compute at all. +term_fit_records <- function(tt) { + records <- list() + for (label in c("conservative", "parsimonious")) { + include <- if (label == "conservative") "" else "?" + model <- tryCatch(minimize(tt, include = include, details = TRUE), error = function(e) NULL) + if (is.null(model) || is.null(model$IC$incl.cov)) { + next + } + frame <- model$IC$incl.cov + terms <- rownames(frame) + for (index in seq_along(terms)) { + if (identical(terms[index], "expression")) { + next + } + # R separates cases from different truth-table rows with ";" and cases + # within a row with ",". Both are just case labels here. + labels <- trimws(unlist(strsplit(as.character(frame[index, "cases"]), "[,;]"))) + labels <- labels[nzchar(labels)] + + # covU is NA for a single-term solution: there is no other term for the + # coverage to be unique against. + covu <- as.numeric(frame[index, "covU"]) + + records[[length(records) + 1]] <- list( + family = label, + term = terms[index], + consistency = as.numeric(frame[index, "inclS"]), + pri = as.numeric(frame[index, "PRI"]), + raw_coverage = as.numeric(frame[index, "covS"]), + unique_coverage = if (is.na(covu)) NULL else covu, + cases = labels + ) + } + } + records +} + safe_minimize <- function(tt, include) { result <- tryCatch( minimize(tt, include = include, details = TRUE), @@ -124,7 +164,8 @@ build_analysis <- function(id, dataset_name, outcome, conditions, incl_cut, n_cu case_ids = rownames(frame), truth_table = truth_table_records(tt, conditions), conservative = safe_minimize(tt, include = ""), - parsimonious = safe_minimize(tt, include = "?") + parsimonious = safe_minimize(tt, include = "?"), + term_fits = term_fit_records(tt) ) } From 4f1c3fa924c49f062aeb4b16b9601247e923a99e Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:10:03 +0100 Subject: [PATCH 4/5] test: cover the case typology and unique coverage --- tests/test_parity.py | 73 +++++++++ tests/test_sufficiency_diagnostics.py | 225 ++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 tests/test_sufficiency_diagnostics.py diff --git a/tests/test_parity.py b/tests/test_parity.py index f45f6cc..ca20a67 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -31,6 +31,7 @@ necessity, necessity_analysis, sufficiency, + sufficiency_diagnostics, ) pytestmark = pytest.mark.parity @@ -59,6 +60,15 @@ def _as_list(value: str | list[str]) -> list[str]: return list(value) +def _as_str_list(value: str | list[str] | None) -> list[str]: + """Normalise a jsonlite string field that may have been auto-unboxed.""" + if value is None: + return [] + if isinstance(value, str): + return [value] + return list(value) + + def _as_int_list(value: int | list[int] | None) -> list[int]: """Normalise a jsonlite numeric field that may have been auto-unboxed.""" if value is None: @@ -285,6 +295,69 @@ def test_parsimonious_solution_matches_r(analysis: dict[str, Any]) -> None: assert obtained == _canonical_solutions(analysis["parsimonious"]) +# --------------------------------------------------------------------------- +# Per-term fit, including unique coverage +# --------------------------------------------------------------------------- + +_TERM_FITS = [ + (analysis, record) + for analysis in FIXTURE["analyses"] + for record in analysis.get("term_fits", []) +] + + +@pytest.mark.parametrize( + ("analysis", "record"), + _TERM_FITS, + ids=[f"{a['id']}-{r['family']}-{r['term']}" for a, r in _TERM_FITS], +) +def test_term_fit_matches_r(analysis: dict[str, Any], record: dict[str, Any]) -> None: + """Per-term consistency, PRI, raw coverage and unique coverage.""" + frame = _frame(analysis) + family_terms = [ + item["term"] for item in analysis["term_fits"] if item["family"] == record["family"] + ] + diagnostics = sufficiency_diagnostics(frame, outcome=analysis["outcome"], terms=family_terms) + term = next(item for item in diagnostics.terms if item.expression == record["term"]) + + assert term.fit.consistency == pytest.approx(record["consistency"], abs=TOLERANCE) + assert term.fit.pri == pytest.approx(record["pri"], abs=TOLERANCE) + assert term.fit.coverage == pytest.approx(record["raw_coverage"], abs=TOLERANCE) + + if record.get("unique_coverage") is None: + # R leaves covU undefined for a one-term solution. setqca reports the + # raw coverage instead: with nothing to share with, everything the term + # covers is uniquely covered by it. + assert len(family_terms) == 1 + assert term.unique_coverage == pytest.approx(term.fit.coverage) + else: + assert term.unique_coverage == pytest.approx(record["unique_coverage"], abs=TOLERANCE) + + +@pytest.mark.parametrize( + ("analysis", "record"), + _TERM_FITS, + ids=[f"{a['id']}-{r['family']}-{r['term']}" for a, r in _TERM_FITS], +) +def test_cases_in_a_term_match_r(analysis: dict[str, Any], record: dict[str, Any]) -> None: + """R's ``cases`` column lists membership in the term, above the crossover. + + The case typology splits those further into typical and deviant-in-degree, + so the union of those two roles is what corresponds to R's list. + """ + frame = _frame(analysis) + family_terms = [ + item["term"] for item in analysis["term_fits"] if item["family"] == record["family"] + ] + diagnostics = sufficiency_diagnostics(frame, outcome=analysis["outcome"], terms=family_terms) + term = next(item for item in diagnostics.terms if item.expression == record["term"]) + + in_term = {item.case for item in term.cases if item.term_membership > 0.5} + assert in_term == set(_as_str_list(record["cases"])) + assert term.frequency == len(in_term) + assert set(term.typical) <= in_term + + # --------------------------------------------------------------------------- # Systematic necessity screening # --------------------------------------------------------------------------- diff --git a/tests/test_sufficiency_diagnostics.py b/tests/test_sufficiency_diagnostics.py new file mode 100644 index 0000000..b1e7ed3 --- /dev/null +++ b/tests/test_sufficiency_diagnostics.py @@ -0,0 +1,225 @@ +"""Tests for case-level sufficiency diagnostics. + +The dataset below is built so every case's role is known by hand before the +code runs. +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from setqca import CaseRole, sufficiency_diagnostics +from setqca.analysis.sufficiency import classify_case + +# X is the term membership, Y the outcome. Roles, in order: +# t1 X=0.9 Y=0.9 in both, X <= Y -> typical +# t2 X=0.9 Y=0.7 in both, X > Y -> deviant consistency (degree) +# t3 X=0.8 Y=0.2 in term, outcome absent -> deviant consistency (kind) +# t4 X=0.2 Y=0.8 outcome without the term -> deviant coverage +# t5 X=0.1 Y=0.1 outside both -> individually irrelevant +KNOWN = pd.DataFrame( + { + "X": [0.9, 0.9, 0.8, 0.2, 0.1], + "Y": [0.9, 0.7, 0.2, 0.8, 0.1], + }, + index=["t1", "t2", "t3", "t4", "t5"], +) + + +class TestCaseTypology: + @pytest.mark.parametrize( + ("term", "outcome", "expected"), + [ + (0.9, 0.9, CaseRole.TYPICAL), + (0.6, 1.0, CaseRole.TYPICAL), + (1.0, 1.0, CaseRole.TYPICAL), + (0.9, 0.7, CaseRole.DEVIANT_CONSISTENCY_IN_DEGREE), + (0.8, 0.2, CaseRole.DEVIANT_CONSISTENCY_IN_KIND), + (0.2, 0.8, CaseRole.DEVIANT_COVERAGE), + (0.1, 0.1, CaseRole.INDIVIDUALLY_IRRELEVANT), + ], + ) + def test_classification_matches_the_typology( + self, term: float, outcome: float, expected: CaseRole + ) -> None: + assert classify_case(term, outcome) is expected + + def test_equal_memberships_above_the_crossover_are_typical(self) -> None: + """X <= Y is the consistency condition, so equality is consistent.""" + assert classify_case(0.8, 0.8) is CaseRole.TYPICAL + + def test_the_crossover_itself_is_outside_both_sets(self) -> None: + """Membership of exactly 0.5 is not 'more in than out'.""" + assert classify_case(0.5, 0.5) is CaseRole.INDIVIDUALLY_IRRELEVANT + assert classify_case(0.5, 0.9) is CaseRole.DEVIANT_COVERAGE + + def test_only_consistency_deviance_counts_against_the_claim(self) -> None: + assert CaseRole.DEVIANT_CONSISTENCY_IN_KIND.contradicts_sufficiency + assert CaseRole.DEVIANT_CONSISTENCY_IN_DEGREE.contradicts_sufficiency + assert not CaseRole.DEVIANT_COVERAGE.contradicts_sufficiency + assert not CaseRole.TYPICAL.contradicts_sufficiency + assert not CaseRole.INDIVIDUALLY_IRRELEVANT.contradicts_sufficiency + + +class TestKnownRoles: + def test_every_case_lands_in_its_hand_computed_role(self) -> None: + result = sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"]) + roles = {item.case: item.role for item in result.terms[0].cases} + assert roles == { + "t1": CaseRole.TYPICAL, + "t2": CaseRole.DEVIANT_CONSISTENCY_IN_DEGREE, + "t3": CaseRole.DEVIANT_CONSISTENCY_IN_KIND, + "t4": CaseRole.DEVIANT_COVERAGE, + "t5": CaseRole.INDIVIDUALLY_IRRELEVANT, + } + + def test_the_named_accessors_agree_with_the_roles(self) -> None: + term = sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"]).terms[0] + assert term.typical == ("t1",) + assert term.contradictory == ("t3",) + assert term.deviant_coverage == ("t4",) + assert set(term.deviant_consistency) == {"t2", "t3"} + + def test_frequency_counts_cases_in_the_term(self) -> None: + term = sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"]).terms[0] + assert term.frequency == 3 # t1, t2, t3 + + def test_case_labels_come_from_the_index_by_default(self) -> None: + term = sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"]).terms[0] + assert [item.case for item in term.cases] == ["t1", "t2", "t3", "t4", "t5"] + + def test_a_case_id_column_can_be_used_instead(self) -> None: + frame = KNOWN.reset_index(names="country") + term = sufficiency_diagnostics(frame, outcome="Y", terms=["X"], case_id="country").terms[0] + assert term.typical == ("t1",) + + +class TestUniqueCoverage: + frame = pd.DataFrame( + { + "A": [0.9, 0.1, 0.9], + "B": [0.1, 0.9, 0.9], + "Y": [0.9, 0.9, 0.9], + }, + index=["a-only", "b-only", "both"], + ) + + def test_a_single_term_uniquely_covers_everything_it_covers(self) -> None: + result = sufficiency_diagnostics(self.frame, outcome="Y", terms=["A"]) + term = result.terms[0] + assert term.unique_coverage == pytest.approx(term.fit.coverage) + + def test_overlap_is_removed_from_unique_coverage(self) -> None: + result = sufficiency_diagnostics(self.frame, outcome="Y", terms=["A", "B"]) + for term in result.terms: + assert term.unique_coverage < term.fit.coverage + + def test_a_duplicated_term_has_no_unique_coverage(self) -> None: + """Two identical terms each explain nothing the other does not.""" + result = sufficiency_diagnostics(self.frame, outcome="Y", terms=["A", "A"]) + for term in result.terms: + assert term.unique_coverage == pytest.approx(0.0) + assert term.redundant is True + + def test_a_case_covered_by_two_terms_is_uniquely_covered_by_neither(self) -> None: + result = sufficiency_diagnostics(self.frame, outcome="Y", terms=["A", "B"]) + by_term = {term.expression: term.uniquely_covered for term in result.terms} + assert by_term["A"] == ("a-only",) + assert by_term["B"] == ("b-only",) + + def test_an_empty_outcome_yields_zero_rather_than_dividing_by_zero(self) -> None: + frame = pd.DataFrame({"A": [0.9, 0.8], "Y": [0.0, 0.0]}) + result = sufficiency_diagnostics(frame, outcome="Y", terms=["A"]) + assert result.terms[0].unique_coverage == 0.0 + + +class TestSolutionLevel: + def test_solution_fit_uses_the_union_of_the_terms(self) -> None: + from setqca import sufficiency + + frame = TestUniqueCoverage.frame + result = sufficiency_diagnostics(frame, outcome="Y", terms=["A", "B"]) + union = frame[["A", "B"]].max(axis=1) + assert result.fit.consistency == pytest.approx(sufficiency(union, frame["Y"]).consistency) + + def test_redundant_terms_are_reported(self) -> None: + result = sufficiency_diagnostics(TestUniqueCoverage.frame, outcome="Y", terms=["A", "A"]) + assert len(result.redundant_terms) == 2 + + def test_terms_may_be_given_as_expressions(self) -> None: + from setqca import Condition + + result = sufficiency_diagnostics(KNOWN, outcome="Y", terms=[Condition("X")]) + assert result.terms[0].expression == "X" + + def test_term_strings_are_parsed(self) -> None: + frame = pd.DataFrame({"A": [0.9, 0.1], "B": [0.8, 0.2], "Y": [0.9, 0.1]}) + result = sufficiency_diagnostics(frame, outcome="Y", terms=["A*~B"]) + assert result.terms[0].expression == "A*~B" + + +class TestExport: + def test_the_term_frame_has_a_column_per_role(self) -> None: + frame = sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"]).to_frame() + for role in CaseRole: + assert role.value in frame.columns + assert frame.loc[0, "n"] == 3 + + def test_the_case_frame_has_one_row_per_case_per_term(self) -> None: + result = sufficiency_diagnostics(TestUniqueCoverage.frame, outcome="Y", terms=["A", "B"]) + frame = result.cases_frame() + assert len(frame) == 2 * 3 + assert list(frame.columns) == [ + "term", + "case", + "term_membership", + "outcome_membership", + "role", + "uniquely_covered", + ] + + def test_the_report_names_the_awkward_cases(self) -> None: + text = str(sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"])) + assert "typical: t1" in text + assert "contradictory: t3" in text + assert "unexplained outcomes: t4" in text + + def test_the_report_omits_sections_with_nothing_to_say(self) -> None: + """A term with no awkward cases gets no awkward-case lines.""" + clean = pd.DataFrame({"X": [0.9, 0.1], "Y": [0.95, 0.05]}, index=["a", "b"]) + text = str(sufficiency_diagnostics(clean, outcome="Y", terms=["X"])) + assert "typical: a" in text + assert "contradictory" not in text + assert "unexplained outcomes" not in text + assert "redundant" not in text + + def test_the_report_names_terms_with_no_typical_cases(self) -> None: + """A term supported by nothing still appears, without a typical line.""" + awkward = pd.DataFrame({"X": [0.9, 0.8], "Y": [0.2, 0.1]}, index=["a", "b"]) + text = str(sufficiency_diagnostics(awkward, outcome="Y", terms=["X"])) + assert "typical:" not in text + assert "contradictory: a, b" in text + + def test_the_report_flags_redundancy(self) -> None: + text = str(sufficiency_diagnostics(TestUniqueCoverage.frame, outcome="Y", terms=["A", "A"])) + assert "redundant" in text + + +class TestGuards: + def test_at_least_one_term_is_required(self) -> None: + with pytest.raises(ValueError, match="At least one term"): + sufficiency_diagnostics(KNOWN, outcome="Y", terms=[]) + + def test_an_unknown_outcome_is_rejected(self) -> None: + with pytest.raises(KeyError, match="Missing columns"): + sufficiency_diagnostics(KNOWN, outcome="Z", terms=["X"]) + + def test_an_unknown_case_column_is_rejected(self) -> None: + with pytest.raises(KeyError, match="Missing columns"): + sufficiency_diagnostics(KNOWN, outcome="Y", terms=["X"], case_id="nope") + + def test_uncalibrated_data_is_rejected(self) -> None: + frame = pd.DataFrame({"X": [0.5, 0.5], "Y": [1.5, 0.1]}) + with pytest.raises(ValueError, match=r"\[0, 1\]"): + sufficiency_diagnostics(frame, outcome="Y", terms=["X"]) From 23319edc756158bd8541dce90a1a1b9fbf5da829 Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:10:04 +0100 Subject: [PATCH 5/5] docs: add sufficiency diagnostics guide --- docs/guide/sufficiency-diagnostics.md | 99 +++++++++++++++++++++++++++ docs/mathematical_validation.md | 11 +-- mkdocs.yml | 1 + 3 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 docs/guide/sufficiency-diagnostics.md diff --git a/docs/guide/sufficiency-diagnostics.md b/docs/guide/sufficiency-diagnostics.md new file mode 100644 index 0000000..a02bfbb --- /dev/null +++ b/docs/guide/sufficiency-diagnostics.md @@ -0,0 +1,99 @@ +# Sufficiency diagnostics + +Parameters of fit summarise a solution in a few numbers. They do not say which +cases produced those numbers — and that is usually the question you actually +have. Which cases support this path? Which contradict it? Which outcomes does it +fail to explain? + +```python +from setqca import sufficiency_diagnostics + +diagnostics = sufficiency_diagnostics( + data, + outcome="SURV", + terms=["DEV*URB*LIT*IND*STB", "DEV*~URB*LIT*~IND*STB"], +) +print(diagnostics) +print(diagnostics.to_frame()) +print(diagnostics.cases_frame()) +``` + +Terms are given as expression strings and parsed, so a solution can be pasted +straight in. Case labels come from the frame index by default, or from a column +you name — no particular schema is assumed. + +## The case typology + +For a term `X` and outcome `Y`, with the crossover at 0.5: + +| Membership | Role | What it means | +| --- | --- | --- | +| `X > 0.5`, `Y > 0.5`, `X ≤ Y` | **typical** | Supports the claim. These are the cases to study for the mechanism. | +| `X > 0.5`, `Y > 0.5`, `X > Y` | **deviant consistency (degree)** | Right corner, wrong magnitude — more in the term than in the outcome. | +| `X > 0.5`, `Y ≤ 0.5` | **deviant consistency (kind)** | The term holds and the outcome does not. This is the case-level contradiction. | +| `X ≤ 0.5`, `Y > 0.5` | **deviant coverage** | An outcome this term does not explain. | +| `X ≤ 0.5`, `Y ≤ 0.5` | **individually irrelevant** | Outside both sets. | + +```python +term = diagnostics.terms[0] +term.typical # ('BE', 'CZ', 'NL') +term.contradictory # cases where the term holds but the outcome does not +term.deviant_coverage # outcomes this term misses +term.deviant_consistency # both kinds of consistency deviance +term.uniquely_covered # cases no other term reaches +``` + +!!! note "Only consistency deviance counts against the claim" + A deviant-coverage case is not evidence against sufficiency. It says the + outcome occurred through some other path, which is exactly what a + disjunctive solution expects. `CaseRole.contradicts_sufficiency` encodes + the distinction. + +## Unique coverage + +Raw coverage counts the outcome membership a term accounts for. **Unique** +coverage counts only what no other term accounts for: + +```text +covU_i = [ Σ min(Xᵢ, Y) − Σ min(Xᵢ, max_{j≠i} Xⱼ, Y) ] / Σ Y +``` + +A term with substantial raw coverage but near-zero unique coverage is redundant +in practice — drop it and the same cases are still explained: + +```python +diagnostics.redundant_terms +``` + +Two identical terms each have unique coverage of exactly zero, which is the +degenerate case the property makes obvious. + +!!! info "A small divergence from R" + R reports `covU` as `NA` for a single-term solution, since there is no other + term to be unique against. `setqca` reports the raw coverage instead: with + nothing to share with, everything the term covers is uniquely covered by it. + Verified against R for every multi-term solution on the Lipset data. + +## Reading R's `cases` column + +R's per-term `cases` column lists cases whose membership in the term exceeds the +crossover. The typology splits that same set further, so R's list corresponds to +**typical plus deviant-in-degree**, not to typical alone. + +On the Lipset conservative solution R lists `BE, CZ, NL, UK` for the first term. +`setqca` agrees on all four being in the term, and additionally reports that UK +is deviant in degree — its membership in the term exceeds its membership in the +outcome. That distinction is the point of the typology, and it is not visible +from the `cases` column alone. + +## Choosing cases to study + +The typology exists to support case selection in multi-method work: + +- **Typical** cases are where the proposed mechanism should be visible. +- **Deviant consistency** cases are where it should be visible and is not — the + most informative cases for revising the theory. +- **Deviant coverage** cases point at paths the solution is missing. +- **Uniquely covered** cases are the ones that justify keeping a term at all. + +::: setqca.analysis.sufficiency diff --git a/docs/mathematical_validation.md b/docs/mathematical_validation.md index b8f53c0..5debf23 100644 --- a/docs/mathematical_validation.md +++ b/docs/mathematical_validation.md @@ -33,7 +33,8 @@ run over all cases. | Necessity consistency | `Σ min(X,Y) / Σ Y` | `metrics.necessity` | `TestNecessity` | `1e-9` vs R | ✅ Verified | | Necessity coverage | `Σ min(X,Y) / Σ X` | `metrics.necessity` | `TestNecessity` | `1e-9` vs R | ✅ Verified | | Relevance of necessity | `Σ (1−X) / Σ (1 − min(X,Y))` | `metrics.necessity` | `TestNecessity` | `1e-9` vs R | ✅ Verified | -| Unique coverage | `cov(Tᵢ) − cov(⋃ⱼ≠ᵢ Tⱼ)` | — | — | — | ❌ Not implemented | +| Unique coverage | `[Σ min(Xᵢ,Y) − Σ min(Xᵢ, max_{j≠i} Xⱼ, Y)] / Σ Y` | `analysis.sufficiency` | `test_sufficiency_diagnostics`, parity | `1e-9` vs R | ✅ Verified | +| Case typology | crossover comparison of `X` and `Y` | `analysis.sufficiency` | `test_sufficiency_diagnostics` | exact | ✅ Tested | | Trivial necessity | `RoN` below threshold with high consistency | `analysis.necessity` | `test_necessity`, parity | `1e-9` vs R | ✅ Verified | | SUIN disjunction | `consistency(A+B) ≥ max over parts` | `analysis.necessity` | `test_necessity` | `1e-12` | ✅ Tested | | Direct calibration, logistic | see below | `calibration.DirectCalibration` | `TestCalibration`, parity | `1e-9` vs R | ✅ Verified | @@ -111,10 +112,10 @@ poison downstream aggregation. ## Findings -1. **Unique coverage is absent.** Raw coverage is implemented and verified; the - per-term unique coverage reported by other QCA software is not. Solutions - currently expose overall and per-term fit through `FittedSolution.term_fits`, - which is raw coverage per term. This is a gap, not a divergence. +1. **Unique coverage is implemented and verified**, closing the gap this audit + first recorded. It matches R's `covU` for every multi-term solution on the + Lipset data. R leaves `covU` undefined for a one-term solution; setqca + reports the raw coverage there, since there is no other term to share with. 2. **Intermediate solutions now follow the standard algorithm.** Simplifying assumptions are derived from the parsimonious solution and split into easy and difficult counterfactuals, matching R `QCA` on the Lipset data. diff --git a/mkdocs.yml b/mkdocs.yml index 08fd284..ff7e776 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - Calibration: guide/calibration.md - Expressions: guide/expressions.md - Necessity: guide/necessity.md + - Sufficiency diagnostics: guide/sufficiency-diagnostics.md - Truth tables: guide/truth-tables.md - Minimisation: guide/minimisation.md - Methodology: METHODOLOGY.md