feat: selection-preserving terminal labels (17, 19)

README.md

@@ -1,5 +1,7 @@
 # quant-data-kit
 
+研究可信度升级：接口、使用示例、验收及限制见 [11–20 使用说明](docs/SELECTION_LABELS.md)。
+
 Hong Kong daily research uses the explicit `quant_data_kit.hong_kong` module
 (`pip install -e '.[hong-kong]'`). It preserves five-digit HK codes, HKD/share
 units, raw source responses and XHKG session times. Providers are selected

docs/SELECTION_LABELS.md

@@ -0,0 +1,26 @@
+# Selection-preserving labels (17, 19)
+
+`quant_data_kit.financial.labels.forward_labels(levels, selections, sessions, horizon=...,
+as_of=..., terminals=None, lower_return=None, upper_return=None)` keeps every selected sample.
+
+* `levels`: date, instrument_id, positive economic return-index value, aware available_at.
+* `selections`: unique sample_id, date, instrument_id; optional missing_reason is
+  suspended, delisted_unknown_terminal or missing_price.
+* `sessions`: explicit ordered unique exchange-session dates (not per-security row numbers).
+* `terminals`: unique selected sample_id, effective date, holding-period realized_return,
+  aware available_at and nonempty source. Return −1 requires evidence; it is not imputation.
+
+Labels are observed, terminal, immature, suspended, delisted_unknown_terminal or missing_price.
+Ordinary horizon labels require all intermediate economic levels, avoiding a shortened horizon
+across a gap. Only evidence available at the cutoff is usable. Terminal proceeds remain cash
+at zero interest to the label endpoint; the adapter must convert distributions/splits/share
+basis consistently before supplying terminal holding-period returns. Unknown terminal values
+are never defaulted to zero or last price. Effective dates and selected identities are strict.
+
+Explicit lower/upper returns are **scenario assumptions**, not statistical confidence bounds.
+`label_coverage` keeps the full denominator and refuses a whole-selection scenario mean while
+any row lacks a bound (including immature outcomes). Report counts beside observed-only IC.
+
+Tests include evidenced zero, late terminal evidence, missing interior prices, unsupported
+bounds and independent hand/share and amount-unit conversions. Input data economics are still
+the caller's responsibility; this API does not manufacture exchange calendars or terminal cash.

src/quant_data_kit/financial/labels.py

@@ -0,0 +1,173 @@
+"""Selection-preserving labels: missing outcomes are evidence, never vanished rows.
+
+Levels must be explicitly declared economic-return levels. Terminal observations
+are per-selection holding-period returns with supplied lineage, NOT assumed zeros.
+Market adapters must convert terminal cash with the same share/action basis.
+"""
+
+import numpy as np
+import pandas as pd
+
+from .common import utc
+
+
+def forward_labels(
+    levels,
+    selections,
+    sessions,
+    *,
+    horizon,
+    as_of,
+    terminals=None,
+    lower_return=None,
+    upper_return=None,
+):
+    required = {"date", "instrument_id", "value", "available_at"}
+    if not required <= set(levels) or not {"sample_id", "date", "instrument_id"} <= set(selections):
+        raise ValueError("dated economic levels and explicit selected sample IDs required")
+    grid = pd.DatetimeIndex(sessions)
+    if (
+        type(horizon) is not int
+        or horizon < 1
+        or grid.hasnans
+        or not grid.is_unique
+        or not grid.is_monotonic_increasing
+        or grid.tz is not None
+        or not grid.equals(grid.normalize())
+    ):
+        raise ValueError(
+            "positive horizon and explicit ordered plain-date session calendar required"
+        )
+    at = utc(as_of)
+    prices, selected = levels.copy(), selections.copy()
+    for frame in (prices, selected):
+        frame["date"] = pd.to_datetime(frame.date)
+        if not frame.date.isin(grid).all():
+            raise ValueError("observations outside explicit session calendar")
+    if prices.duplicated(["instrument_id", "date"]).any() or selected.sample_id.duplicated().any():
+        raise ValueError("duplicate prices or selected sample IDs")
+    if (
+        any(not isinstance(v, str) or not v.strip() for v in selected.sample_id)
+        or any(not isinstance(v, str) or not v.strip() for v in selected.instrument_id)
+        or any(not isinstance(v, str) or not v.strip() for v in prices.instrument_id)
+    ):
+        raise ValueError("selected identities cannot be missing")
+    prices["available_at"] = prices.available_at.map(utc)
+    prices["value"] = pd.to_numeric(prices.value, errors="raise")
+    if not np.isfinite(prices.value).all() or (prices.value <= 0).any():
+        raise ValueError("observed levels must be positive; evidenced terminal zero is separate")
+    visible = prices.loc[prices.available_at <= at].set_index(["instrument_id", "date"])
+    terminal = {} if terminals is None else terminals.copy()
+    if terminals is not None:
+        if not {"sample_id", "date", "realized_return", "available_at", "source"} <= set(terminal):
+            raise ValueError(
+                "terminal labels require per-sample economic return and source evidence"
+            )
+        if (
+            terminal.sample_id.duplicated().any()
+            or not terminal.sample_id.isin(selected.sample_id).all()
+        ):
+            raise ValueError("ambiguous or unselected terminal sample")
+        terminal["date"] = pd.to_datetime(terminal.date)
+        if (
+            terminal.date.isna().any()
+            or terminal.date.dt.tz is not None
+            or not terminal.date.equals(terminal.date.dt.normalize())
+        ):
+            raise ValueError("terminal effective dates must be finite plain dates")
+        terminal["available_at"] = terminal.available_at.map(utc)
+        if (
+            not np.isfinite(terminal.realized_return).all()
+            or (terminal.realized_return < -1).any()
+            or terminal.source.isna().any()
+            or terminal.source.astype(str).str.strip().eq("").any()
+        ):
+            raise ValueError("terminal return must be evidenced and no less than -1")
+        terminal = {
+            r.sample_id: r
+            for r in terminal.itertuples()
+            if r.available_at <= at and r.date <= at.tz_localize(None).normalize()
+        }
+    bounds = None
+    if lower_return is not None or upper_return is not None:
+        if (
+            lower_return is None
+            or upper_return is None
+            or not np.isfinite([lower_return, upper_return]).all()
+            or not -1 <= lower_return <= upper_return
+        ):
+            raise ValueError("explicit finite ordered lower/upper return scenarios required")
+        bounds = (lower_return, upper_return)
+    records = []
+    for row in selected.itertuples():
+        location = grid.get_loc(row.date)
+        endpoint = grid[location + horizon] if location + horizon < len(grid) else None
+        value, status, source = None, "immature", None
+        event = terminal.get(row.sample_id)
+        if event is not None and (
+            event.date <= row.date or endpoint is not None and event.date > endpoint
+        ):
+            raise ValueError("terminal observation is outside the selected label interval")
+        if event is not None:
+            # Cash is retained through the label endpoint; no implicit reinvestment/interest.
+            value, status, source = float(event.realized_return), "terminal", event.source
+        elif endpoint is not None and endpoint <= at.tz_localize(None).normalize():
+            keys = [(row.instrument_id, day) for day in grid[location : location + horizon + 1]]
+            if all(key in visible.index for key in keys):
+                value = float(visible.loc[keys[-1], "value"] / visible.loc[keys[0], "value"] - 1)
+                status = "observed"
+            else:
+                status = getattr(row, "missing_reason", "missing_price")
+                if status not in {"suspended", "delisted_unknown_terminal", "missing_price"}:
+                    raise ValueError("explicit supported missing-label reason required")
+        records.append(
+            {
+                "sample_id": row.sample_id,
+                "instrument_id": row.instrument_id,
+                "date": row.date,
+                "label_end": endpoint,
+                "return": value,
+                "status": status,
+                "source": source,
+                "lower_return": value
+                if value is not None
+                else bounds[0]
+                if bounds and status != "immature"
+                else None,
+                "upper_return": value
+                if value is not None
+                else bounds[1]
+                if bounds and status != "immature"
+                else None,
+            }
+        )
+    return pd.DataFrame(
+        records,
+        columns=[
+            "sample_id",
+            "instrument_id",
+            "date",
+            "label_end",
+            "return",
+            "status",
+            "source",
+            "lower_return",
+            "upper_return",
+        ],
+    )
+
+
+def label_coverage(labels):
+    counts = labels.status.value_counts().to_dict()
+    return {
+        "selected_samples": len(labels),
+        "observed_or_terminal": int(labels["return"].notna().sum()),
+        "status_counts": {str(k): int(v) for k, v in counts.items()},
+        "lower_mean_scenario": float(labels.lower_return.mean())
+        if labels.lower_return.notna().all() and len(labels)
+        else None,
+        "upper_mean_scenario": float(labels.upper_return.mean())
+        if labels.upper_return.notna().all() and len(labels)
+        else None,
+        "policy": "unobserved outcomes retained; scenario bounds are assumptions, not point estimates",
+    }

tests/test_selection_labels.py

@@ -0,0 +1,111 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from quant_data_kit.financial.labels import forward_labels, label_coverage
+from quant_data_kit.financial.units import normalize_trading_units
+
+
+def sample():
+    days = pd.date_range("2024-01-01", periods=6)
+    levels = pd.DataFrame(
+        {
+            "date": days,
+            "instrument_id": "A",
+            "value": [100, 101, 102, 103, 104, 105],
+            "available_at": days.tz_localize("UTC") + pd.Timedelta(hours=20),
+        }
+    )
+    selections = pd.DataFrame(
+        {
+            "sample_id": ["zero", "missing", "young"],
+            "date": [days[0], days[1], days[-1]],
+            "instrument_id": ["A", "B", "A"],
+            "missing_reason": ["missing_price", "delisted_unknown_terminal", "missing_price"],
+        }
+    )
+    return days, levels, selections
+
+
+def test_zero_terminal_survives_and_unknown_terminal_has_only_explicit_scenarios():
+    days, levels, selected = sample()
+    terminal = pd.DataFrame(
+        [
+            {
+                "sample_id": "zero",
+                "date": days[2],
+                "realized_return": -1.0,
+                "available_at": "2024-01-03T20:00:00Z",
+                "source": "evidenced-zero-cash",
+            }
+        ]
+    )
+    labels = forward_labels(
+        levels,
+        selected,
+        days,
+        horizon=3,
+        as_of="2024-01-06T23:00:00Z",
+        terminals=terminal,
+        lower_return=-1,
+        upper_return=0.5,
+    )
+    assert labels.status.tolist() == ["terminal", "delisted_unknown_terminal", "immature"]
+    assert labels["return"].iloc[0] == -1
+    assert np.isnan(labels["return"].iloc[1])
+    assert labels.lower_return.iloc[1] == -1 and labels.upper_return.iloc[1] == 0.5
+    coverage = label_coverage(labels)
+    assert coverage["selected_samples"] == 3 and coverage["observed_or_terminal"] == 1
+    assert coverage["lower_mean_scenario"] is None  # immature sample cannot be silently dropped
+
+
+def test_future_terminal_evidence_cannot_backfill_current_labels():
+    days, levels, selected = sample()
+    terminal = pd.DataFrame(
+        [
+            {
+                "sample_id": "zero",
+                "date": days[2],
+                "realized_return": -1.0,
+                "available_at": "2024-02-01T00:00:00Z",
+                "source": "late-evidence",
+            }
+        ]
+    )
+    labels = forward_labels(
+        levels, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z", terminals=terminal
+    )
+    assert labels.status.iloc[0] == "observed"
+    assert labels["return"].iloc[0] == pytest.approx(0.03)
+    assert labels.lower_return.iloc[1:].isna().all()
+
+
+def test_interior_missing_price_is_not_a_shortened_horizon():
+    days, levels, selected = sample()
+    labels = forward_labels(
+        levels.drop(index=1), selected, days, horizon=3, as_of="2024-01-06T23:00:00Z"
+    )
+    assert labels.status.iloc[0] == "missing_price" and labels["return"].isna().all()
+
+
+def test_label_contract_guards_and_trading_unit_hand_differential():
+    days, levels, selected = sample()
+    for frame in (pd.concat([levels, levels.iloc[:1]]), levels.assign(value=0)):
+        with pytest.raises(ValueError):
+            forward_labels(frame, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z")
+    with pytest.raises(ValueError):
+        forward_labels(
+            levels, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z", lower_return=-1
+        )
+    for lots in (1, 3, 10, 12345):
+        actual = normalize_trading_units(
+            pd.DataFrame({"volume": [lots], "amount": [2]}),
+            volume_unit="lots",
+            amount_unit="ten_thousand_currency",
+            currency="CNY",
+            share_basis="raw",
+            lot_size=100,
+        )
+        assert actual.volume.iloc[0] == lots * 100
+        assert actual.amount.iloc[0] == 20000
+        assert actual.volume.iloc[0] != lots  # injected hand/share confusion would fail