From a6f9ad7aa88b6452ce4eae77f323f25d4ccbe6ce Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:45:39 +0100 Subject: [PATCH 1/4] feat: add calibration specs, indirect method and diagnostics --- src/setqca/calibration/__init__.py | 81 +++++ src/setqca/calibration/_diagnostics.py | 304 ++++++++++++++++++ .../_direct.py} | 2 +- src/setqca/calibration/_spec.py | 271 ++++++++++++++++ 4 files changed, 657 insertions(+), 1 deletion(-) create mode 100644 src/setqca/calibration/__init__.py create mode 100644 src/setqca/calibration/_diagnostics.py rename src/setqca/{calibration.py => calibration/_direct.py} (99%) create mode 100644 src/setqca/calibration/_spec.py diff --git a/src/setqca/calibration/__init__.py b/src/setqca/calibration/__init__.py new file mode 100644 index 0000000..e5b76a6 --- /dev/null +++ b/src/setqca/calibration/__init__.py @@ -0,0 +1,81 @@ +"""Calibration: turning raw measures into set memberships. + +Calibration is where substantive knowledge enters a QCA, and where a result is +most easily manufactured. The primitives are here, along with reproducible +specifications, diagnostics for the failures that spoil a truth table, and +quantile helpers that are explicitly *not* a calibration. + +Examples +-------- +>>> from setqca.calibration import calibrate, direct_spec +>>> spec = direct_spec("innovation", full_out=20, crossover=50, full_in=80) +>>> result = calibrate([10, 50, 90], spec) # doctest: +SKIP +>>> result.diagnostics.warnings # doctest: +SKIP +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ._diagnostics import ( + AnchorSuggestion, + CalibrationDiagnostics, + CalibrationResult, + diagnose_calibration, + diagnose_frame, + suggest_anchors, +) +from ._direct import DirectCalibration, calibrate_crisp, calibrate_direct +from ._spec import ( + CalibrationMethod, + CalibrationSpec, + crisp_spec, + direct_spec, + indirect_spec, +) + +if TYPE_CHECKING: # pragma: no cover - imported for type checking only + import numpy.typing as npt + +__all__ = [ + "AnchorSuggestion", + "CalibrationDiagnostics", + "CalibrationMethod", + "CalibrationResult", + "CalibrationSpec", + "DirectCalibration", + "calibrate", + "calibrate_crisp", + "calibrate_direct", + "crisp_spec", + "diagnose_calibration", + "diagnose_frame", + "direct_spec", + "indirect_spec", + "suggest_anchors", +] + + +def calibrate(values: npt.ArrayLike, spec: CalibrationSpec) -> CalibrationResult: + """Apply a specification and diagnose the result in one step. + + Parameters + ---------- + values : array_like + Raw values, or calibrated ones for the identity method. + spec : CalibrationSpec + The calibration to apply. + + Returns + ------- + CalibrationResult + The calibrated values, the specification that produced them, and + diagnostics. Keeping the three together is what makes a calibration + reproducible rather than a number that appeared once. + """ + calibrated = spec.apply(values) + return CalibrationResult( + spec=spec, + values=calibrated, + diagnostics=diagnose_calibration(calibrated, name=spec.condition), + ) diff --git a/src/setqca/calibration/_diagnostics.py b/src/setqca/calibration/_diagnostics.py new file mode 100644 index 0000000..0feac1f --- /dev/null +++ b/src/setqca/calibration/_diagnostics.py @@ -0,0 +1,304 @@ +"""Diagnostics for calibrated memberships, and quantile helpers for anchors. + +A calibration can be arithmetically valid and analytically useless. These +checks catch the failures that quietly ruin a truth table: everything piled at +the crossover, everything crushed to 0 and 1, or no variation at all. + +None of these is fatal by itself. They are reported so the researcher can +decide, not enforced. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd + +from setqca._validation import as_float_array, validate_membership + +if TYPE_CHECKING: # pragma: no cover - imported for type checking only + import numpy.typing as npt + + from setqca._validation import FloatArray + +CROSSOVER_BAND = 0.05 +EXTREME_BAND = 0.05 +PILE_UP_SHARE = 0.25 +COMPRESSION_SHARE = 0.90 +LOW_VARIANCE = 0.05 + + +@dataclass(frozen=True, slots=True) +class CalibrationDiagnostics: + """What a calibrated vector looks like, and what is worrying about it. + + Attributes + ---------- + n + Number of cases. + at_crossover + Cases with membership exactly 0.5. Their truth-table corner is + undefined, so these block analysis rather than merely warning. + near_crossover + Cases within ``CROSSOVER_BAND`` of 0.5. + extreme + Cases within ``EXTREME_BAND`` of 0 or 1. + minimum, maximum, mean, standard_deviation + Summary statistics of the calibrated values. + above_crossover + Cases that will be assigned the "present" corner. + warnings + Human-readable descriptions of every issue found. + """ + + n: int + at_crossover: int + near_crossover: int + extreme: int + minimum: float + maximum: float + mean: float + standard_deviation: float + above_crossover: int + warnings: tuple[str, ...] + + @property + def usable(self) -> bool: + """Return whether the vector can be used for a truth table at all. + + Only an exact crossover membership makes a vector unusable; everything + else is a judgement call left to the researcher. + """ + return self.at_crossover == 0 + + @property + def pile_up_share(self) -> float: + """Return the proportion of cases bunched around the crossover.""" + return self.near_crossover / self.n if self.n else 0.0 + + @property + def compression_share(self) -> float: + """Return the proportion of cases pushed to the extremes.""" + return self.extreme / self.n if self.n else 0.0 + + def __str__(self) -> str: + lines = [ + f"n={self.n}, mean={self.mean:.3f}, sd={self.standard_deviation:.3f}, " + f"range=[{self.minimum:.3f}, {self.maximum:.3f}]", + f"above crossover: {self.above_crossover}/{self.n}", + ] + lines.extend(f" warning: {warning}" for warning in self.warnings) + if not self.warnings: + lines.append(" no issues found") + return "\n".join(lines) + + +def diagnose_calibration(values: npt.ArrayLike, *, name: str = "values") -> CalibrationDiagnostics: + """Inspect a calibrated vector for the failures that spoil a truth table. + + Parameters + ---------- + values : array_like + Calibrated memberships in ``[0, 1]``. + name : str, default "values" + Name used in the warning messages. + + Returns + ------- + CalibrationDiagnostics + Summary statistics and a list of warnings. + + Raises + ------ + ValueError + If the values are not calibrated memberships. + """ + membership = validate_membership(values, name=name) + n = int(membership.size) + + at_crossover = int(np.sum(np.isclose(membership, 0.5, atol=1e-12))) + near_crossover = int(np.sum(np.abs(membership - 0.5) < CROSSOVER_BAND)) + extreme = int(np.sum((membership < EXTREME_BAND) | (membership > 1.0 - EXTREME_BAND))) + above = int(np.sum(membership >= 0.5)) + deviation = float(np.std(membership)) + + warnings: list[str] = [] + if at_crossover: + warnings.append( + f"{at_crossover} case(s) sit exactly at 0.5, where the truth-table " + "corner is undefined; resolve them or set allow_crossover_cases" + ) + if n and near_crossover / n > PILE_UP_SHARE: + warnings.append( + f"{near_crossover}/{n} cases lie within {CROSSOVER_BAND} of the crossover; " + "small anchor changes will move them between corners" + ) + if n and extreme / n > COMPRESSION_SHARE: + warnings.append( + f"{extreme}/{n} cases are at the extremes; the calibration is close to " + "crisp and the fuzzy detail has been squeezed out" + ) + if deviation < LOW_VARIANCE: + warnings.append( + f"standard deviation is {deviation:.3f}; the condition barely varies and " + "will carry little information" + ) + if above == 0: + warnings.append("no case is above the crossover; the condition is never present") + elif above == n: + warnings.append("every case is above the crossover; the condition is never absent") + + return CalibrationDiagnostics( + n=n, + at_crossover=at_crossover, + near_crossover=near_crossover, + extreme=extreme, + minimum=float(np.min(membership)), + maximum=float(np.max(membership)), + mean=float(np.mean(membership)), + standard_deviation=deviation, + above_crossover=above, + warnings=tuple(warnings), + ) + + +@dataclass(frozen=True, slots=True) +class AnchorSuggestion: + """Quantiles of a raw variable, offered as a starting point only. + + Attributes + ---------- + quantiles + The probabilities used. + values + The corresponding raw values, in ascending order. + caveat + A standing reminder that these are not a calibration. + """ + + quantiles: tuple[float, float, float] + values: tuple[float, float, float] + caveat: str = ( + "Quantiles describe the sample, not the concept. Anchors must be " + "justified substantively; these are a starting point for that argument, " + "not a substitute for it." + ) + + @property + def anchors(self) -> tuple[float, float, float]: + """Return the suggested ``(full_out, crossover, full_in)``.""" + return self.values + + def __str__(self) -> str: + low, mid, high = self.values + return ( + f"Suggested from quantiles {self.quantiles}: " + f"full_out={low:g}, crossover={mid:g}, full_in={high:g}\n" + f" {self.caveat}" + ) + + +def suggest_anchors( + values: npt.ArrayLike, + *, + quantiles: tuple[float, float, float] = (0.05, 0.50, 0.95), +) -> AnchorSuggestion: + """Report quantiles of a raw variable to inform an anchor decision. + + This is a **diagnostic**, not a calibration. Data-driven anchors describe + the sample rather than the concept, and a set defined by its own + distribution cannot support a claim about set membership. Use the output to + see where your cases actually lie, then argue for anchors on substantive + grounds. + + Parameters + ---------- + values : array_like + Raw, uncalibrated measures. + quantiles : tuple of float, default (0.05, 0.50, 0.95) + Probabilities for the exclusion, crossover and inclusion anchors. + + Returns + ------- + AnchorSuggestion + The quantile values, with the caveat attached. + + Raises + ------ + ValueError + If the quantiles are not strictly increasing and inside ``[0, 1]``. + """ + if not all(0.0 <= q <= 1.0 for q in quantiles): + raise ValueError("Quantiles must lie in [0, 1].") + if not quantiles[0] < quantiles[1] < quantiles[2]: + raise ValueError("Quantiles must be strictly increasing.") + + raw = as_float_array(values, name="values") + low, mid, high = (float(value) for value in np.quantile(raw, quantiles)) + if not low < mid < high: + raise ValueError( + "The requested quantiles are not distinct in this sample, so they " + "cannot serve as anchors; the variable may be too concentrated." + ) + return AnchorSuggestion(quantiles=quantiles, values=(low, mid, high)) + + +def diagnose_frame( + data: pd.DataFrame, columns: list[str] | tuple[str, ...] | None = None +) -> pd.DataFrame: + """Diagnose every calibrated column and return a tidy summary. + + Parameters + ---------- + data : pandas.DataFrame + Calibrated memberships. + columns : list of str, optional + Columns to check. Defaults to every column. + + Returns + ------- + pandas.DataFrame + One row per column, with the summary statistics and a joined warning + string. + """ + names = list(data.columns) if columns is None else list(columns) + records = [] + for name in names: + diagnostics = diagnose_calibration(data[name].to_numpy(), name=name) + records.append( + { + "condition": name, + "n": diagnostics.n, + "mean": diagnostics.mean, + "sd": diagnostics.standard_deviation, + "min": diagnostics.minimum, + "max": diagnostics.maximum, + "at_crossover": diagnostics.at_crossover, + "near_crossover": diagnostics.near_crossover, + "extreme": diagnostics.extreme, + "above_crossover": diagnostics.above_crossover, + "usable": diagnostics.usable, + "warnings": "; ".join(diagnostics.warnings), + } + ) + return pd.DataFrame.from_records(records) + + +@dataclass(frozen=True, slots=True) +class CalibrationResult: + """Calibrated values together with the specification that produced them.""" + + spec: object + values: FloatArray + diagnostics: CalibrationDiagnostics + + def to_frame(self) -> pd.DataFrame: + """Return the calibrated values as a one-column frame.""" + condition = getattr(self.spec, "condition", "values") + return pd.DataFrame({condition: self.values}) + + def __str__(self) -> str: + condition = getattr(self.spec, "condition", "values") + return f"Calibration of {condition}\n{self.diagnostics}" diff --git a/src/setqca/calibration.py b/src/setqca/calibration/_direct.py similarity index 99% rename from src/setqca/calibration.py rename to src/setqca/calibration/_direct.py index b403622..8560bde 100644 --- a/src/setqca/calibration.py +++ b/src/setqca/calibration/_direct.py @@ -8,7 +8,7 @@ import numpy as np import numpy.typing as npt -from ._validation import FloatArray, as_float_array +from setqca._validation import FloatArray, as_float_array def _logistic_cdf(z: FloatArray) -> FloatArray: diff --git a/src/setqca/calibration/_spec.py b/src/setqca/calibration/_spec.py new file mode 100644 index 0000000..980a849 --- /dev/null +++ b/src/setqca/calibration/_spec.py @@ -0,0 +1,271 @@ +"""Reproducible, serialisable calibration specifications. + +Calibration is the step where substantive judgement enters an analysis, so it +is the step most worth recording. A :class:`CalibrationSpec` is a value: it can +be stored beside the results, shipped in a replication package, compared +against a colleague's, and applied unchanged to further cases. + +Four methods are supported: + +``direct`` + Three-anchor logistic or piecewise transformation. The standard approach. +``crisp`` + Threshold cuts into binary or ordered categories. +``indirect`` + An explicit monotone mapping from raw values to memberships. Use this when + theory dictates a shape the direct transformation cannot express. +``identity`` + The values are already calibrated and are passed through, validated. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, replace +from enum import Enum +from typing import TYPE_CHECKING, Any + +import numpy as np + +from setqca._validation import as_float_array, validate_membership + +from ._direct import DirectCalibration, calibrate_crisp + +if TYPE_CHECKING: # pragma: no cover - imported for type checking only + import numpy.typing as npt + + from setqca._validation import FloatArray + + +class CalibrationMethod(Enum): + """How raw values become set memberships.""" + + DIRECT = "direct" + CRISP = "crisp" + INDIRECT = "indirect" + IDENTITY = "identity" + + +@dataclass(frozen=True, slots=True) +class CalibrationSpec: + """A complete, replayable description of one condition's calibration. + + Parameters + ---------- + condition : str + Name of the condition this calibrates. + method : CalibrationMethod + Which transformation to apply. + anchors : tuple of float, optional + ``(full_out, crossover, full_in)`` for the direct method. + idm : float, default 0.95 + Membership at the inclusion anchor, for the direct logistic form. + logistic : bool, default True + Use the logistic rather than piecewise direct transformation. + below, above : float, default 1.0 + Piecewise shaping exponents. + thresholds : tuple of float, optional + Cut points for the crisp method. + mapping : tuple of (float, float), optional + ``(raw, membership)`` points for the indirect method. Interpolated + linearly between points and held flat outside them. + note : str, optional + Free text recording *why* these choices were made. Carried through + serialisation, because the reason is part of the specification. + """ + + condition: str + method: CalibrationMethod = CalibrationMethod.DIRECT + anchors: tuple[float, float, float] | None = None + idm: float = 0.95 + logistic: bool = True + below: float = 1.0 + above: float = 1.0 + thresholds: tuple[float, ...] | None = None + mapping: tuple[tuple[float, float], ...] | None = None + note: str = "" + + def __post_init__(self) -> None: + if self.method is CalibrationMethod.DIRECT: + if self.anchors is None: + raise ValueError("The direct method requires anchors.") + # Validate eagerly so a bad spec fails when it is written, not when + # it is applied to data much later. + DirectCalibration( + full_out=self.anchors[0], + crossover=self.anchors[1], + full_in=self.anchors[2], + idm=self.idm, + logistic=self.logistic, + below=self.below, + above=self.above, + ) + elif self.method is CalibrationMethod.CRISP: + if not self.thresholds: + raise ValueError("The crisp method requires thresholds.") + elif self.method is CalibrationMethod.INDIRECT: + if self.mapping is None or len(self.mapping) < 2: + raise ValueError("The indirect method requires at least two mapping points.") + raws = [raw for raw, _ in self.mapping] + memberships = [membership for _, membership in self.mapping] + if any(np.diff(raws) <= 0): + raise ValueError( + "Indirect mapping points must have strictly increasing raw values." + ) + if any(np.diff(memberships) < 0): + raise ValueError("Indirect mapping must be non-decreasing in membership.") + if not all(0.0 <= membership <= 1.0 for membership in memberships): + raise ValueError("Indirect mapping memberships must be in [0, 1].") + + def apply(self, values: npt.ArrayLike) -> FloatArray: + """Calibrate raw values according to this specification.""" + if self.method is CalibrationMethod.DIRECT: + assert self.anchors is not None + return DirectCalibration( + full_out=self.anchors[0], + crossover=self.anchors[1], + full_in=self.anchors[2], + idm=self.idm, + logistic=self.logistic, + below=self.below, + above=self.above, + ).transform(values) + if self.method is CalibrationMethod.CRISP: + assert self.thresholds is not None + categories = calibrate_crisp(values, self.thresholds) + # A single threshold yields a binary set; several yield ordered + # categories, rescaled onto [0, 1] so downstream code sees + # memberships rather than raw category indices. + top = len(self.thresholds) + return (categories / top).astype(np.float64) + if self.method is CalibrationMethod.INDIRECT: + assert self.mapping is not None + raw = as_float_array(values, name=self.condition) + points = np.asarray([point[0] for point in self.mapping], dtype=np.float64) + targets = np.asarray([point[1] for point in self.mapping], dtype=np.float64) + interpolated: FloatArray = np.interp(raw, points, targets).astype(np.float64) + return interpolated + return validate_membership(values, name=self.condition) + + def with_anchors(self, anchors: tuple[float, float, float]) -> CalibrationSpec: + """Return a copy with different anchors, for sensitivity work.""" + return replace(self, anchors=anchors) + + def to_dict(self) -> dict[str, Any]: + """Return a plain dictionary suitable for JSON or YAML.""" + payload: dict[str, Any] = { + "condition": self.condition, + "method": self.method.value, + "note": self.note, + } + if self.anchors is not None: + payload["anchors"] = list(self.anchors) + payload["idm"] = self.idm + payload["logistic"] = self.logistic + payload["below"] = self.below + payload["above"] = self.above + if self.thresholds is not None: + payload["thresholds"] = list(self.thresholds) + if self.mapping is not None: + payload["mapping"] = [list(point) for point in self.mapping] + return payload + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> CalibrationSpec: + """Rebuild a specification from :meth:`to_dict` output. + + Raises + ------ + KeyError + If the payload lacks a condition or method. + ValueError + If the method is unknown, or the specification is invalid. + """ + anchors = payload.get("anchors") + thresholds = payload.get("thresholds") + mapping = payload.get("mapping") + try: + method = CalibrationMethod(payload["method"]) + except ValueError as error: + raise ValueError(f"Unknown calibration method {payload['method']!r}.") from error + return cls( + condition=payload["condition"], + method=method, + anchors=None if anchors is None else (anchors[0], anchors[1], anchors[2]), + idm=payload.get("idm", 0.95), + logistic=payload.get("logistic", True), + below=payload.get("below", 1.0), + above=payload.get("above", 1.0), + thresholds=None if thresholds is None else tuple(thresholds), + mapping=None if mapping is None else tuple((p[0], p[1]) for p in mapping), + note=payload.get("note", ""), + ) + + def to_json(self, *, indent: int | None = None, sort_keys: bool = False) -> str: + """Serialise to a JSON string. + + Parameters + ---------- + indent : int, optional + Passed to :func:`json.dumps` for readable output. + sort_keys : bool, default False + Sort keys, which makes stored specifications diff cleanly. + """ + return json.dumps(self.to_dict(), indent=indent, sort_keys=sort_keys) + + @classmethod + def from_json(cls, text: str) -> CalibrationSpec: + """Rebuild a specification from JSON.""" + return cls.from_dict(json.loads(text)) + + +def direct_spec( + condition: str, + *, + full_out: float, + crossover: float, + full_in: float, + idm: float = 0.95, + logistic: bool = True, + below: float = 1.0, + above: float = 1.0, + note: str = "", +) -> CalibrationSpec: + """Build a three-anchor direct calibration specification.""" + return CalibrationSpec( + condition=condition, + method=CalibrationMethod.DIRECT, + anchors=(full_out, crossover, full_in), + idm=idm, + logistic=logistic, + below=below, + above=above, + note=note, + ) + + +def crisp_spec(condition: str, *, thresholds: tuple[float, ...], note: str = "") -> CalibrationSpec: + """Build a crisp threshold specification.""" + return CalibrationSpec( + condition=condition, + method=CalibrationMethod.CRISP, + thresholds=thresholds, + note=note, + ) + + +def indirect_spec( + condition: str, *, mapping: tuple[tuple[float, float], ...], note: str = "" +) -> CalibrationSpec: + """Build an explicit monotone mapping specification. + + Use this when theory dictates a shape the direct transformation cannot + express — a plateau, a step, an asymmetric ramp. Points are interpolated + linearly and held flat beyond the ends. + """ + return CalibrationSpec( + condition=condition, + method=CalibrationMethod.INDIRECT, + mapping=mapping, + note=note, + ) From abcf94bf774d24a7e52db87bd79d7f815bc1022d Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:45:40 +0100 Subject: [PATCH 2/4] feat: export the calibration framework from the package root --- src/setqca/__init__.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/setqca/__init__.py b/src/setqca/__init__.py index d4c1b3a..62ea289 100644 --- a/src/setqca/__init__.py +++ b/src/setqca/__init__.py @@ -22,7 +22,23 @@ robustness_analysis, sufficiency_diagnostics, ) -from .calibration import DirectCalibration, calibrate_crisp, calibrate_direct +from .calibration import ( + AnchorSuggestion, + CalibrationDiagnostics, + CalibrationMethod, + CalibrationResult, + CalibrationSpec, + DirectCalibration, + calibrate, + calibrate_crisp, + calibrate_direct, + crisp_spec, + diagnose_calibration, + diagnose_frame, + direct_spec, + indirect_spec, + suggest_anchors, +) from .counterfactuals import ( CounterfactualAnalysis, DirectionalExpectation, @@ -61,7 +77,12 @@ __all__ = [ "CSQCA", "FSQCA", + "AnchorSuggestion", "BooleanSolution", + "CalibrationDiagnostics", + "CalibrationMethod", + "CalibrationResult", + "CalibrationSpec", "CaseDiagnostic", "CaseRole", "Condition", @@ -97,10 +118,16 @@ "__version__", "build_chart", "build_truth_table", + "calibrate", "calibrate_crisp", "calibrate_direct", "classify_counterfactuals", + "crisp_spec", + "diagnose_calibration", + "diagnose_frame", + "direct_spec", "evaluate_expression", + "indirect_spec", "minimize", "minimize_chart", "necessity", @@ -110,4 +137,5 @@ "simplify_expression", "sufficiency", "sufficiency_diagnostics", + "suggest_anchors", ] From 38e0d7995f81b8c34ceea28e977bf69b492d2945 Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:45:41 +0100 Subject: [PATCH 3/4] test: cover specs, serialisation, diagnostics and anchor helpers --- tests/test_calibration_framework.py | 246 ++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 tests/test_calibration_framework.py diff --git a/tests/test_calibration_framework.py b/tests/test_calibration_framework.py new file mode 100644 index 0000000..7906fd5 --- /dev/null +++ b/tests/test_calibration_framework.py @@ -0,0 +1,246 @@ +"""Tests for calibration specifications, diagnostics and anchor helpers.""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from setqca.calibration import ( + CalibrationMethod, + CalibrationSpec, + calibrate, + crisp_spec, + diagnose_calibration, + diagnose_frame, + direct_spec, + indirect_spec, + suggest_anchors, +) + +RAW = [0.0, 10.0, 25.0, 50.0, 75.0, 90.0, 100.0] + + +class TestDirectSpec: + def test_a_direct_spec_reproduces_the_function(self) -> None: + from setqca import calibrate_direct + + spec = direct_spec("x", full_out=20, crossover=50, full_in=80) + assert spec.apply(RAW) == pytest.approx( + calibrate_direct(RAW, full_out=20, crossover=50, full_in=80) + ) + + def test_bad_anchors_fail_when_the_spec_is_written(self) -> None: + """A spec is validated eagerly, not when it eventually meets data.""" + with pytest.raises(ValueError, match="strictly ordered"): + direct_spec("x", full_out=50, crossover=50, full_in=80) + + def test_the_direct_method_requires_anchors(self) -> None: + with pytest.raises(ValueError, match="requires anchors"): + CalibrationSpec(condition="x", method=CalibrationMethod.DIRECT) + + def test_anchors_can_be_swapped_for_sensitivity_work(self) -> None: + spec = direct_spec("x", full_out=20, crossover=50, full_in=80) + moved = spec.with_anchors((10, 50, 90)) + assert moved.anchors == (10, 50, 90) + assert spec.anchors == (20, 50, 80), "the original is untouched" + + +class TestCrispSpec: + def test_one_threshold_gives_a_binary_set(self) -> None: + spec = crisp_spec("x", thresholds=(50.0,)) + assert spec.apply(RAW).tolist() == [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0] + + def test_several_thresholds_are_rescaled_onto_the_unit_interval(self) -> None: + """Downstream code expects memberships, not raw category indices.""" + spec = crisp_spec("x", thresholds=(25.0, 75.0)) + values = spec.apply(RAW) + assert set(np.unique(values)) <= {0.0, 0.5, 1.0} + assert values.max() == 1.0 + + def test_the_crisp_method_requires_thresholds(self) -> None: + with pytest.raises(ValueError, match="requires thresholds"): + CalibrationSpec(condition="x", method=CalibrationMethod.CRISP) + + +class TestIndirectSpec: + def test_points_are_interpolated_linearly(self) -> None: + spec = indirect_spec("x", mapping=((0.0, 0.0), (50.0, 0.5), (100.0, 1.0))) + assert spec.apply([0, 25, 50, 75, 100]) == pytest.approx([0.0, 0.25, 0.5, 0.75, 1.0]) + + def test_values_beyond_the_ends_are_held_flat(self) -> None: + spec = indirect_spec("x", mapping=((10.0, 0.0), (90.0, 1.0))) + assert spec.apply([-100, 200]) == pytest.approx([0.0, 1.0]) + + def test_a_plateau_the_direct_form_cannot_express(self) -> None: + """Theory sometimes says 'no change across this range'.""" + spec = indirect_spec("x", mapping=((0.0, 0.0), (30.0, 0.5), (70.0, 0.5), (100.0, 1.0))) + assert spec.apply([30, 50, 70]) == pytest.approx([0.5, 0.5, 0.5]) + + def test_at_least_two_points_are_required(self) -> None: + with pytest.raises(ValueError, match="at least two mapping points"): + indirect_spec("x", mapping=((0.0, 0.0),)) + + def test_raw_values_must_increase(self) -> None: + with pytest.raises(ValueError, match="strictly increasing"): + indirect_spec("x", mapping=((10.0, 0.0), (10.0, 1.0))) + + def test_the_mapping_may_not_decrease(self) -> None: + with pytest.raises(ValueError, match="non-decreasing"): + indirect_spec("x", mapping=((0.0, 1.0), (10.0, 0.0))) + + def test_memberships_must_be_calibrated(self) -> None: + with pytest.raises(ValueError, match=r"\[0, 1\]"): + indirect_spec("x", mapping=((0.0, 0.0), (10.0, 1.5))) + + +class TestIdentitySpec: + def test_already_calibrated_values_pass_through(self) -> None: + spec = CalibrationSpec(condition="x", method=CalibrationMethod.IDENTITY) + assert spec.apply([0.0, 0.5, 1.0]).tolist() == [0.0, 0.5, 1.0] + + def test_uncalibrated_values_are_still_rejected(self) -> None: + spec = CalibrationSpec(condition="x", method=CalibrationMethod.IDENTITY) + with pytest.raises(ValueError, match=r"\[0, 1\]"): + spec.apply([0.0, 1.5]) + + +class TestSerialisation: + @pytest.mark.parametrize( + ("spec", "sample"), + [ + (direct_spec("x", full_out=20, crossover=50, full_in=80, note="theory"), RAW), + ( + direct_spec("x", full_out=80, crossover=50, full_in=20, logistic=False, above=2.0), + RAW, + ), + (crisp_spec("x", thresholds=(25.0, 75.0)), RAW), + (indirect_spec("x", mapping=((0.0, 0.0), (50.0, 0.5), (100.0, 1.0))), RAW), + # The identity method takes memberships, not raw measures. + (CalibrationSpec(condition="x", method=CalibrationMethod.IDENTITY), [0.0, 0.4, 1.0]), + ], + ) + def test_specs_round_trip_through_json( + self, spec: CalibrationSpec, sample: list[float] + ) -> None: + restored = CalibrationSpec.from_json(spec.to_json()) + assert restored == spec + assert restored.apply(sample) == pytest.approx(spec.apply(sample)) + + def test_the_note_survives_the_round_trip(self) -> None: + """The reason for a choice is part of the specification.""" + spec = direct_spec("x", full_out=20, crossover=50, full_in=80, note="OECD threshold") + assert CalibrationSpec.from_json(spec.to_json()).note == "OECD threshold" + + def test_an_unknown_method_is_rejected(self) -> None: + with pytest.raises(ValueError, match="Unknown calibration method"): + CalibrationSpec.from_dict({"condition": "x", "method": "telepathy"}) + + def test_a_payload_without_a_condition_is_rejected(self) -> None: + with pytest.raises(KeyError): + CalibrationSpec.from_dict({"method": "identity"}) + + +class TestDiagnostics: + def test_a_healthy_vector_reports_no_issues(self) -> None: + values = [0.05, 0.2, 0.35, 0.65, 0.8, 0.95] + diagnostics = diagnose_calibration(values) + assert diagnostics.warnings == () + assert diagnostics.usable + + def test_exact_crossover_membership_makes_a_vector_unusable(self) -> None: + diagnostics = diagnose_calibration([0.5, 0.9, 0.1]) + assert diagnostics.at_crossover == 1 + assert not diagnostics.usable + assert any("exactly at 0.5" in warning for warning in diagnostics.warnings) + + def test_pile_up_at_the_crossover_is_reported(self) -> None: + diagnostics = diagnose_calibration([0.48, 0.49, 0.51, 0.52, 0.9, 0.1]) + assert diagnostics.pile_up_share > 0.25 + assert any("of the crossover" in warning for warning in diagnostics.warnings) + + def test_compression_to_the_extremes_is_reported(self) -> None: + diagnostics = diagnose_calibration([0.0, 0.01, 0.99, 1.0, 1.0, 0.0]) + assert diagnostics.compression_share > 0.9 + assert any("close to" in warning for warning in diagnostics.warnings) + + def test_low_variance_is_reported(self) -> None: + diagnostics = diagnose_calibration([0.60, 0.61, 0.62, 0.60, 0.61]) + assert any("barely varies" in warning for warning in diagnostics.warnings) + + def test_a_condition_that_is_never_present_is_reported(self) -> None: + diagnostics = diagnose_calibration([0.1, 0.2, 0.3, 0.05]) + assert diagnostics.above_crossover == 0 + assert any("never present" in warning for warning in diagnostics.warnings) + + def test_a_condition_that_is_always_present_is_reported(self) -> None: + diagnostics = diagnose_calibration([0.9, 0.8, 0.95, 0.85]) + assert any("never absent" in warning for warning in diagnostics.warnings) + + def test_uncalibrated_input_is_rejected(self) -> None: + with pytest.raises(ValueError, match=r"\[0, 1\]"): + diagnose_calibration([0.5, 1.5]) + + def test_the_report_lists_warnings_or_says_there_are_none(self) -> None: + assert "no issues found" in str(diagnose_calibration([0.05, 0.3, 0.7, 0.95])) + assert "warning:" in str(diagnose_calibration([0.5, 0.9, 0.1])) + + def test_a_whole_frame_can_be_diagnosed(self) -> None: + frame = pd.DataFrame({"A": [0.1, 0.9, 0.2], "B": [0.5, 0.5, 0.5]}) + summary = diagnose_frame(frame) + assert list(summary["condition"]) == ["A", "B"] + assert not summary.loc[1, "usable"] + + def test_specific_columns_can_be_selected(self) -> None: + frame = pd.DataFrame({"A": [0.1, 0.9], "B": [0.2, 0.8], "Y": [0.3, 0.7]}) + assert list(diagnose_frame(frame, ["A", "B"])["condition"]) == ["A", "B"] + + +class TestAnchorSuggestions: + def test_quantiles_are_reported_with_a_caveat(self) -> None: + suggestion = suggest_anchors(list(range(101))) + assert suggestion.anchors[0] < suggestion.anchors[1] < suggestion.anchors[2] + assert "not a substitute" in suggestion.caveat + assert "not a substitute" in str(suggestion) + + def test_the_quantiles_are_adjustable(self) -> None: + suggestion = suggest_anchors(list(range(101)), quantiles=(0.1, 0.5, 0.9)) + assert suggestion.quantiles == (0.1, 0.5, 0.9) + assert suggestion.values == pytest.approx((10.0, 50.0, 90.0)) + + def test_quantiles_must_increase(self) -> None: + with pytest.raises(ValueError, match="strictly increasing"): + suggest_anchors([1, 2, 3], quantiles=(0.5, 0.5, 0.9)) + + def test_quantiles_must_be_probabilities(self) -> None: + with pytest.raises(ValueError, match=r"\[0, 1\]"): + suggest_anchors([1, 2, 3], quantiles=(-0.1, 0.5, 0.9)) + + def test_a_concentrated_variable_cannot_yield_distinct_anchors(self) -> None: + with pytest.raises(ValueError, match="not distinct"): + suggest_anchors([5.0] * 20) + + +class TestCalibrateEntryPoint: + def test_values_spec_and_diagnostics_come_back_together(self) -> None: + spec = direct_spec("innovation", full_out=20, crossover=50, full_in=80) + result = calibrate(RAW, spec) + assert result.spec is spec + assert len(result.values) == len(RAW) + assert result.diagnostics.n == len(RAW) + + def test_the_result_exports_as_a_named_frame(self) -> None: + spec = direct_spec("innovation", full_out=20, crossover=50, full_in=80) + frame = calibrate(RAW, spec).to_frame() + assert list(frame.columns) == ["innovation"] + + def test_the_report_names_the_condition(self) -> None: + spec = direct_spec("innovation", full_out=20, crossover=50, full_in=80) + assert "Calibration of innovation" in str(calibrate(RAW, spec)) + + def test_diagnostics_travel_with_a_bad_calibration(self) -> None: + """A calibration that ruins the analysis still returns, and says why.""" + spec = crisp_spec("x", thresholds=(50.0,)) + result = calibrate([10.0, 20.0, 30.0], spec) + assert result.diagnostics.warnings + assert any("never present" in warning for warning in result.diagnostics.warnings) From d0be0e900899549344bef3644ed98698b70fd2df Mon Sep 17 00:00:00 2001 From: Diogo Ribeiro Date: Tue, 11 Aug 2026 09:45:41 +0100 Subject: [PATCH 4/4] docs: document specs, diagnostics and the quantile caveat --- docs/guide/calibration.md | 100 ++++++++++++++++++++++++++++++++++++++ docs/guide/robustness.md | 6 +-- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/docs/guide/calibration.md b/docs/guide/calibration.md index 51ac772..3fd29f1 100644 --- a/docs/guide/calibration.md +++ b/docs/guide/calibration.md @@ -107,6 +107,106 @@ build_truth_table(data, outcome="Y", conditions=["A"], allow_crossover_cases=Tru With the override, scores of exactly 0.5 are assigned to the *present* corner, because corner assignment uses `x >= 0.5`. +## Specifications + +A calibration is a decision worth recording. `CalibrationSpec` makes it a value +you can store, compare, ship in a replication package, and replay: + +```python +from setqca import calibrate, direct_spec + +spec = direct_spec( + "innovation", + full_out=20, + crossover=50, + full_in=80, + note="OECD reporting threshold; see section 3.2", +) +result = calibrate(raw["innovation"], spec) + +result.values # the calibrated memberships +result.spec # what produced them +result.diagnostics # and what is worrying about them +``` + +The `note` carries the *reason* through serialisation, because the reason is +part of the specification: + +```python +spec.to_json() +CalibrationSpec.from_json(text) # round-trips exactly +``` + +A specification is validated when it is written, not when it eventually meets +data — badly ordered anchors raise immediately. + +### Indirect calibration + +When theory dictates a shape the three-anchor transformation cannot express — a +plateau, a step, an asymmetric ramp — give the mapping explicitly: + +```python +from setqca import indirect_spec + +spec = indirect_spec( + "capacity", + mapping=((0, 0.0), (30, 0.5), (70, 0.5), (100, 1.0)), + note="no meaningful variation between 30 and 70", +) +``` + +Points are interpolated linearly and held flat beyond the ends. The mapping must +be non-decreasing, since a calibration that reverses direction is a different +concept, not a calibration. + +## Diagnostics + +A calibration can be arithmetically valid and analytically useless. + +```python +from setqca import diagnose_calibration, diagnose_frame + +print(diagnose_calibration(calibrated)) +diagnose_frame(data) # one row per condition +``` + +Five failures are reported: + +| Warning | Why it matters | +| --- | --- | +| Cases exactly at 0.5 | The truth-table corner is undefined. This is the only one that makes the vector *unusable*. | +| Pile-up near the crossover | Small anchor changes will move cases between corners, so the result is fragile. | +| Compression to the extremes | The calibration is effectively crisp; the fuzzy detail has been squeezed out. | +| Low variance | The condition barely varies and carries little information. | +| Never present / never absent | Every case falls on one side, so the condition cannot discriminate. | + +None is fatal by itself — they are reported so you can decide, not enforced. + +## Quantile helpers, and why they are not a calibration + +```python +from setqca import suggest_anchors + +print(suggest_anchors(raw["innovation"])) +``` + +```text +Suggested from quantiles (0.05, 0.5, 0.95): full_out=12, crossover=48, full_in=91 + Quantiles describe the sample, not the concept. Anchors must be justified + substantively; these are a starting point for that argument, not a substitute + for it. +``` + +!!! danger "Data-driven anchors are not calibration" + A set defined by its own distribution cannot support a claim about set + membership. If the crossover is the sample median, then "more in than out" + means "above average for these cases" — which changes when you add a case, + and says nothing about the concept. + + The helper exists to show you where your cases actually lie so you can + argue for anchors. It returns the caveat attached to the result, and + nothing in this package will apply quantile anchors for you. + ## Reusing a calibration `DirectCalibration` is a frozen dataclass, so a calibration is a value you can diff --git a/docs/guide/robustness.md b/docs/guide/robustness.md index 80bab58..a60d565 100644 --- a/docs/guide/robustness.md +++ b/docs/guide/robustness.md @@ -93,10 +93,10 @@ Four scales are available: from setqca.analysis.robustness import solution_similarity similarity = solution_similarity(left_terms, right_terms, data) -similarity.identical # exact set equality -similarity.term_overlap # Jaccard over terms +similarity.identical # exact set equality +similarity.term_overlap # Jaccard over terms similarity.configurational # Jaccard over the literals used -similarity.membership # fuzzy Jaccard over case membership +similarity.membership # fuzzy Jaccard over case membership ``` The last is the one that catches agreement the text hides: two solutions can be