From ba136c2fa2eea121bfb2ad7887b536c3952586f7 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:16:48 +0800 Subject: [PATCH] feat: preserve selected samples with evidenced terminal labels --- README.md | 2 + docs/SELECTION_LABELS.md | 26 ++++ src/quant_data_kit/financial/labels.py | 173 +++++++++++++++++++++++++ tests/test_selection_labels.py | 111 ++++++++++++++++ 4 files changed, 312 insertions(+) create mode 100644 docs/SELECTION_LABELS.md create mode 100644 src/quant_data_kit/financial/labels.py create mode 100644 tests/test_selection_labels.py diff --git a/README.md b/README.md index ef193ca..68842e2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # quant-data-kit +研究可信度升级：接口、使用示例、验收及限制见 [11–20 使用说明](docs/SELECTION_LABELS.md)。 + Hong Kong daily research uses the explicit `quant_data_kit.hong_kong` module (`pip install -e '.[hong-kong]'`). It preserves five-digit HK codes, HKD/share units, raw source responses and XHKG session times. Providers are selected diff --git a/docs/SELECTION_LABELS.md b/docs/SELECTION_LABELS.md new file mode 100644 index 0000000..06a8138 --- /dev/null +++ b/docs/SELECTION_LABELS.md @@ -0,0 +1,26 @@ +# Selection-preserving labels (17, 19) + +`quant_data_kit.financial.labels.forward_labels(levels, selections, sessions, horizon=..., +as_of=..., terminals=None, lower_return=None, upper_return=None)` keeps every selected sample. + +* `levels`: date, instrument_id, positive economic return-index value, aware available_at. +* `selections`: unique sample_id, date, instrument_id; optional missing_reason is + suspended, delisted_unknown_terminal or missing_price. +* `sessions`: explicit ordered unique exchange-session dates (not per-security row numbers). +* `terminals`: unique selected sample_id, effective date, holding-period realized_return, + aware available_at and nonempty source. Return −1 requires evidence; it is not imputation. + +Labels are observed, terminal, immature, suspended, delisted_unknown_terminal or missing_price. +Ordinary horizon labels require all intermediate economic levels, avoiding a shortened horizon +across a gap. Only evidence available at the cutoff is usable. Terminal proceeds remain cash +at zero interest to the label endpoint; the adapter must convert distributions/splits/share +basis consistently before supplying terminal holding-period returns. Unknown terminal values +are never defaulted to zero or last price. Effective dates and selected identities are strict. + +Explicit lower/upper returns are **scenario assumptions**, not statistical confidence bounds. +`label_coverage` keeps the full denominator and refuses a whole-selection scenario mean while +any row lacks a bound (including immature outcomes). Report counts beside observed-only IC. + +Tests include evidenced zero, late terminal evidence, missing interior prices, unsupported +bounds and independent hand/share and amount-unit conversions. Input data economics are still +the caller's responsibility; this API does not manufacture exchange calendars or terminal cash. diff --git a/src/quant_data_kit/financial/labels.py b/src/quant_data_kit/financial/labels.py new file mode 100644 index 0000000..17c9c03 --- /dev/null +++ b/src/quant_data_kit/financial/labels.py @@ -0,0 +1,173 @@ +"""Selection-preserving labels: missing outcomes are evidence, never vanished rows. + +Levels must be explicitly declared economic-return levels. Terminal observations +are per-selection holding-period returns with supplied lineage, NOT assumed zeros. +Market adapters must convert terminal cash with the same share/action basis. +""" + +import numpy as np +import pandas as pd + +from .common import utc + + +def forward_labels( + levels, + selections, + sessions, + *, + horizon, + as_of, + terminals=None, + lower_return=None, + upper_return=None, +): + required = {"date", "instrument_id", "value", "available_at"} + if not required <= set(levels) or not {"sample_id", "date", "instrument_id"} <= set(selections): + raise ValueError("dated economic levels and explicit selected sample IDs required") + grid = pd.DatetimeIndex(sessions) + if ( + type(horizon) is not int + or horizon < 1 + or grid.hasnans + or not grid.is_unique + or not grid.is_monotonic_increasing + or grid.tz is not None + or not grid.equals(grid.normalize()) + ): + raise ValueError( + "positive horizon and explicit ordered plain-date session calendar required" + ) + at = utc(as_of) + prices, selected = levels.copy(), selections.copy() + for frame in (prices, selected): + frame["date"] = pd.to_datetime(frame.date) + if not frame.date.isin(grid).all(): + raise ValueError("observations outside explicit session calendar") + if prices.duplicated(["instrument_id", "date"]).any() or selected.sample_id.duplicated().any(): + raise ValueError("duplicate prices or selected sample IDs") + if ( + any(not isinstance(v, str) or not v.strip() for v in selected.sample_id) + or any(not isinstance(v, str) or not v.strip() for v in selected.instrument_id) + or any(not isinstance(v, str) or not v.strip() for v in prices.instrument_id) + ): + raise ValueError("selected identities cannot be missing") + prices["available_at"] = prices.available_at.map(utc) + prices["value"] = pd.to_numeric(prices.value, errors="raise") + if not np.isfinite(prices.value).all() or (prices.value <= 0).any(): + raise ValueError("observed levels must be positive; evidenced terminal zero is separate") + visible = prices.loc[prices.available_at <= at].set_index(["instrument_id", "date"]) + terminal = {} if terminals is None else terminals.copy() + if terminals is not None: + if not {"sample_id", "date", "realized_return", "available_at", "source"} <= set(terminal): + raise ValueError( + "terminal labels require per-sample economic return and source evidence" + ) + if ( + terminal.sample_id.duplicated().any() + or not terminal.sample_id.isin(selected.sample_id).all() + ): + raise ValueError("ambiguous or unselected terminal sample") + terminal["date"] = pd.to_datetime(terminal.date) + if ( + terminal.date.isna().any() + or terminal.date.dt.tz is not None + or not terminal.date.equals(terminal.date.dt.normalize()) + ): + raise ValueError("terminal effective dates must be finite plain dates") + terminal["available_at"] = terminal.available_at.map(utc) + if ( + not np.isfinite(terminal.realized_return).all() + or (terminal.realized_return < -1).any() + or terminal.source.isna().any() + or terminal.source.astype(str).str.strip().eq("").any() + ): + raise ValueError("terminal return must be evidenced and no less than -1") + terminal = { + r.sample_id: r + for r in terminal.itertuples() + if r.available_at <= at and r.date <= at.tz_localize(None).normalize() + } + bounds = None + if lower_return is not None or upper_return is not None: + if ( + lower_return is None + or upper_return is None + or not np.isfinite([lower_return, upper_return]).all() + or not -1 <= lower_return <= upper_return + ): + raise ValueError("explicit finite ordered lower/upper return scenarios required") + bounds = (lower_return, upper_return) + records = [] + for row in selected.itertuples(): + location = grid.get_loc(row.date) + endpoint = grid[location + horizon] if location + horizon < len(grid) else None + value, status, source = None, "immature", None + event = terminal.get(row.sample_id) + if event is not None and ( + event.date <= row.date or endpoint is not None and event.date > endpoint + ): + raise ValueError("terminal observation is outside the selected label interval") + if event is not None: + # Cash is retained through the label endpoint; no implicit reinvestment/interest. + value, status, source = float(event.realized_return), "terminal", event.source + elif endpoint is not None and endpoint <= at.tz_localize(None).normalize(): + keys = [(row.instrument_id, day) for day in grid[location : location + horizon + 1]] + if all(key in visible.index for key in keys): + value = float(visible.loc[keys[-1], "value"] / visible.loc[keys[0], "value"] - 1) + status = "observed" + else: + status = getattr(row, "missing_reason", "missing_price") + if status not in {"suspended", "delisted_unknown_terminal", "missing_price"}: + raise ValueError("explicit supported missing-label reason required") + records.append( + { + "sample_id": row.sample_id, + "instrument_id": row.instrument_id, + "date": row.date, + "label_end": endpoint, + "return": value, + "status": status, + "source": source, + "lower_return": value + if value is not None + else bounds[0] + if bounds and status != "immature" + else None, + "upper_return": value + if value is not None + else bounds[1] + if bounds and status != "immature" + else None, + } + ) + return pd.DataFrame( + records, + columns=[ + "sample_id", + "instrument_id", + "date", + "label_end", + "return", + "status", + "source", + "lower_return", + "upper_return", + ], + ) + + +def label_coverage(labels): + counts = labels.status.value_counts().to_dict() + return { + "selected_samples": len(labels), + "observed_or_terminal": int(labels["return"].notna().sum()), + "status_counts": {str(k): int(v) for k, v in counts.items()}, + "lower_mean_scenario": float(labels.lower_return.mean()) + if labels.lower_return.notna().all() and len(labels) + else None, + "upper_mean_scenario": float(labels.upper_return.mean()) + if labels.upper_return.notna().all() and len(labels) + else None, + "policy": "unobserved outcomes retained; scenario bounds are assumptions, not point estimates", + } diff --git a/tests/test_selection_labels.py b/tests/test_selection_labels.py new file mode 100644 index 0000000..56d950f --- /dev/null +++ b/tests/test_selection_labels.py @@ -0,0 +1,111 @@ +import numpy as np +import pandas as pd +import pytest + +from quant_data_kit.financial.labels import forward_labels, label_coverage +from quant_data_kit.financial.units import normalize_trading_units + + +def sample(): + days = pd.date_range("2024-01-01", periods=6) + levels = pd.DataFrame( + { + "date": days, + "instrument_id": "A", + "value": [100, 101, 102, 103, 104, 105], + "available_at": days.tz_localize("UTC") + pd.Timedelta(hours=20), + } + ) + selections = pd.DataFrame( + { + "sample_id": ["zero", "missing", "young"], + "date": [days[0], days[1], days[-1]], + "instrument_id": ["A", "B", "A"], + "missing_reason": ["missing_price", "delisted_unknown_terminal", "missing_price"], + } + ) + return days, levels, selections + + +def test_zero_terminal_survives_and_unknown_terminal_has_only_explicit_scenarios(): + days, levels, selected = sample() + terminal = pd.DataFrame( + [ + { + "sample_id": "zero", + "date": days[2], + "realized_return": -1.0, + "available_at": "2024-01-03T20:00:00Z", + "source": "evidenced-zero-cash", + } + ] + ) + labels = forward_labels( + levels, + selected, + days, + horizon=3, + as_of="2024-01-06T23:00:00Z", + terminals=terminal, + lower_return=-1, + upper_return=0.5, + ) + assert labels.status.tolist() == ["terminal", "delisted_unknown_terminal", "immature"] + assert labels["return"].iloc[0] == -1 + assert np.isnan(labels["return"].iloc[1]) + assert labels.lower_return.iloc[1] == -1 and labels.upper_return.iloc[1] == 0.5 + coverage = label_coverage(labels) + assert coverage["selected_samples"] == 3 and coverage["observed_or_terminal"] == 1 + assert coverage["lower_mean_scenario"] is None # immature sample cannot be silently dropped + + +def test_future_terminal_evidence_cannot_backfill_current_labels(): + days, levels, selected = sample() + terminal = pd.DataFrame( + [ + { + "sample_id": "zero", + "date": days[2], + "realized_return": -1.0, + "available_at": "2024-02-01T00:00:00Z", + "source": "late-evidence", + } + ] + ) + labels = forward_labels( + levels, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z", terminals=terminal + ) + assert labels.status.iloc[0] == "observed" + assert labels["return"].iloc[0] == pytest.approx(0.03) + assert labels.lower_return.iloc[1:].isna().all() + + +def test_interior_missing_price_is_not_a_shortened_horizon(): + days, levels, selected = sample() + labels = forward_labels( + levels.drop(index=1), selected, days, horizon=3, as_of="2024-01-06T23:00:00Z" + ) + assert labels.status.iloc[0] == "missing_price" and labels["return"].isna().all() + + +def test_label_contract_guards_and_trading_unit_hand_differential(): + days, levels, selected = sample() + for frame in (pd.concat([levels, levels.iloc[:1]]), levels.assign(value=0)): + with pytest.raises(ValueError): + forward_labels(frame, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z") + with pytest.raises(ValueError): + forward_labels( + levels, selected, days, horizon=3, as_of="2024-01-06T23:00:00Z", lower_return=-1 + ) + for lots in (1, 3, 10, 12345): + actual = normalize_trading_units( + pd.DataFrame({"volume": [lots], "amount": [2]}), + volume_unit="lots", + amount_unit="ten_thousand_currency", + currency="CNY", + share_basis="raw", + lot_size=100, + ) + assert actual.volume.iloc[0] == lots * 100 + assert actual.amount.iloc[0] == 20000 + assert actual.volume.iloc[0] != lots # injected hand/share confusion would fail