feat: factor family diagnostics, nested selection and complete labels

README.md

@@ -1,5 +1,7 @@
 # quant-factors
 
+研究可信度升级：接口、使用示例、验收及限制见 [11–20 使用说明](docs/RESEARCH_INTEGRITY_11_20.md)。
+
 Shared factor computation library for PureSaber quant research.
 
 ## Install
@@ -12,7 +14,7 @@ python -m pip check
 
 ## Usage
 
-本开发分支将QDK固定到`271a65ee383158b3dd7570e19a3f2b4123f09f79`，
+本开发分支将QDK固定到`ba136c2fa2eea121bfb2ad7887b536c3952586f7`，
 并保留日线 PIT 风险分支的因子证据改进。跨仓开发请使用配套本地工作区；
 最终依赖提交与锁文件须在验收后同步更新。
 旧发布标签保持不变。

docs/RESEARCH_INTEGRITY_11_20.md

@@ -0,0 +1,31 @@
+# Research family and label APIs (11, 12, 14, 17–19)
+
+Install `.[research]` for the optional shared quant-lab statistical/nested APIs.
+
+* `combinatorial.family_statistics`: shared DSR and original CSCV/PBO; pass complete excess
+  return matrices, explicit periods_per_year and block count. It is not chronological OOS.
+* `purged_combinatorial_splits(starts, ends, blocks=6, test_blocks=2, embargo=...)`: tests all
+  combinations, purges exact label-interval overlaps and adds post-label-end embargo. It keeps
+  train/test/purged/embargoed indices and identifies a different estimand from original CSCV.
+  Disjoint test blocks do not incorrectly purge every row between them.
+* `dependent_mean_evidence`: common-index stationary/circular bootstrap sensitivity plus HAC.
+  Do not treat correlated stock-day observations as independent samples.
+* `nested_factor_selection`: fit learned direction/window/neutralization/model settings inside
+  the inner training callback and refit the selected recipe on outer training. Callbacks receive
+  bounded copies, but must not use global future data. Returned scores are not funded accounts.
+
+`research.factor_report(..., terminal_labels=None, sessions=None)` now returns `label_samples`
+and `label_coverage` for each horizon. Missing/delisted/immature selected rows remain visible.
+Terminal evidence is per-sample holding-period return with source, availability and optionally
+horizon; sample IDs are `symbol:YYYY-MM-DD`. Use the same economic return-index/share basis.
+If available_at is absent, the report explicitly uses a descriptive end-of-day availability
+assumption; for strict PIT research supply actual availability and an exchange calendar.
+The default calendar is the union of supplied dates, not proof of complete exchange coverage.
+
+Existing observed-label IC remains descriptive. The new denominator and status breakdown
+expose which securities were excluded from its numeric calculation. Unknown terminal cash
+cannot be silently replaced with zero. See QDK `docs/SELECTION_LABELS.md` for the shared contract.
+
+Regression tests preserve future-prefix invariance of v2 turnover/Amihud factors and show
+that an evidenced terminal −100% sample survives factor reporting. No statistical result
+automatically promotes a factor or authorizes a trade.

pyproject.toml

@@ -10,15 +10,16 @@ readme = "README.md"
 license = { text = "MIT" }
 requires-python = ">=3.10"
 dependencies = [
-    "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@271a65ee383158b3dd7570e19a3f2b4123f09f79",
+    "quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@ba136c2fa2eea121bfb2ad7887b536c3952586f7",
     "pandas>=2.0",
     "numpy>=1.24",
     "pyarrow>=14.0",
     "pyyaml>=6.0",
 ]
 
 [project.optional-dependencies]
-dev = ["pytest>=7.4", "pytest-cov>=5.0", "ruff>=0.9.0", "exceptiongroup>=1", "tomli>=1"]
+research = ["quant-lab[statistics] @ git+https://github.com/PureSaber/quant-lab.git@a923a791315312a4d9caecf3b9776f08d53bb736"]
+dev = ["pytest>=7.4", "pytest-cov>=5.0", "ruff>=0.9.0", "exceptiongroup>=1", "tomli>=1", "setuptools>=68", "wheel>=0.46", "quant-lab[statistics] @ git+https://github.com/PureSaber/quant-lab.git@a923a791315312a4d9caecf3b9776f08d53bb736"]
 
 [tool.quant-workspace]
 layer = "strategy"

requirements.lock

@@ -1,9 +1,7 @@
-#
-# This file is autogenerated by pip-compile with Python 3.12
-# by the following command:
-#
-#    pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --no-emit-index-url --no-index --output-file=requirements.lock --strip-extras pyproject.toml
-#
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --extra dev --extra research --python-version 3.10 --constraint=requirements-constraints.txt --output-file requirements.lock
+arch==8.0.0
+    # via quant-lab
 attrs==26.1.0
     # via
     #   jsonschema
@@ -15,7 +13,9 @@ coverage==7.16.0
 duckdb==1.5.5
     # via quant-data-kit
 exceptiongroup==1.3.1
-    # via quant-factors (pyproject.toml)
+    # via
+    #   quant-factors (pyproject.toml)
+    #   pytest
 iniconfig==2.3.0
     # via pytest
 jsonschema==4.26.0
@@ -26,31 +26,47 @@ jsonschema-specifications==2025.9.1
     # via jsonschema
 numpy==2.2.6
     # via
+    #   quant-factors (pyproject.toml)
+    #   arch
     #   pandas
+    #   patsy
     #   quant-data-kit
-    #   quant-factors (pyproject.toml)
+    #   quant-lab
+    #   scipy
+    #   statsmodels
 orjson==3.12.0
     # via quant-data-kit
 packaging==26.3
-    # via pytest
+    # via
+    #   arch
+    #   patsy
+    #   pytest
+    #   statsmodels
+    #   wheel
 pandas==2.3.3
     # via
-    #   quant-data-kit
     #   quant-factors (pyproject.toml)
+    #   arch
+    #   quant-data-kit
+    #   quant-lab
+    #   statsmodels
+patsy==1.0.3
+    # via statsmodels
 pluggy==1.6.0
     # via
     #   pytest
     #   pytest-cov
 pyarrow==25.0.1
     # via
-    #   quant-data-kit
     #   quant-factors (pyproject.toml)
+    #   quant-data-kit
+    #   quant-lab
 pygments==2.21.0
     # via pytest
 pytest==9.1.1
     # via
-    #   pytest-cov
     #   quant-factors (pyproject.toml)
+    #   pytest-cov
 pytest-cov==7.1.0
     # via quant-factors (pyproject.toml)
 python-dateutil==2.9.0.post0
@@ -59,9 +75,12 @@ pytz==2026.3.post1
     # via pandas
 pyyaml==6.0.3
     # via
-    #   quant-data-kit
     #   quant-factors (pyproject.toml)
-quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@271a65ee383158b3dd7570e19a3f2b4123f09f79
+    #   quant-data-kit
+    #   quant-lab
+quant-data-kit @ git+https://github.com/PureSaber/quant-data-kit.git@ba136c2fa2eea121bfb2ad7887b536c3952586f7
+    # via quant-factors (pyproject.toml)
+quant-lab @ git+https://github.com/PureSaber/quant-lab.git@a923a791315312a4d9caecf3b9776f08d53bb736
     # via quant-factors (pyproject.toml)
 referencing==0.37.0
     # via
@@ -74,10 +93,21 @@ rpds-py==0.30.0
     #   referencing
 ruff==0.16.5
     # via quant-factors (pyproject.toml)
+scipy==1.15.3
+    # via
+    #   arch
+    #   statsmodels
+setuptools==84.0.0
+    # via quant-factors (pyproject.toml)
 six==1.17.0
     # via python-dateutil
+statsmodels==0.14.6
+    # via arch
 tomli==2.4.1
-    # via quant-factors (pyproject.toml)
+    # via
+    #   quant-factors (pyproject.toml)
+    #   coverage
+    #   pytest
 typing-extensions==4.16.0
     # via
     #   exceptiongroup
@@ -87,7 +117,5 @@ tzdata==2026.3
     # via pandas
 websockets==15.0.1
     # via quant-data-kit
-
-# The following packages are considered to be unsafe in a requirements file:
-setuptools==84.0.0
-    # via quant-factors (pyproject.toml::build-system.requires)
+wheel==0.48.0
+    # via quant-factors (pyproject.toml)

src/quant_factors/combinatorial.py

@@ -0,0 +1,94 @@
+"""Purged combinatorial splits: a distinct estimand from original CSCV/PBO."""
+
+from itertools import combinations
+from math import comb
+
+import numpy as np
+import pandas as pd
+
+
+def purged_combinatorial_splits(
+    starts, ends, *, blocks=6, test_blocks=2, embargo=0, max_splits=20000
+):
+    starts, ends = pd.DatetimeIndex(starts), pd.DatetimeIndex(ends)
+    if (
+        len(starts) != len(ends)
+        or starts.hasnans
+        or ends.hasnans
+        or not starts.is_monotonic_increasing
+        or (ends < starts).any()
+    ):
+        raise ValueError("ordered starts and valid observed label end times required")
+    if (
+        type(blocks) is not int
+        or type(test_blocks) is not int
+        or not 2 <= blocks <= len(starts)
+        or not 1 <= test_blocks < blocks
+        or type(embargo) is not int
+        or embargo < 0
+        or comb(blocks, test_blocks) > max_splits
+    ):
+        raise ValueError("invalid combinatorial split/embargo limits")
+    partitions = np.array_split(np.arange(len(starts)), blocks)
+    records = []
+    for selected in combinations(range(blocks), test_blocks):
+        test = np.concatenate([partitions[i] for i in selected])
+        blocked = np.zeros(len(starts), dtype=bool)
+        # Exact interval overlap, not one min/max interval spanning disjoint test blocks.
+        for i in test:
+            blocked |= (starts <= ends[i]) & (ends >= starts[i])
+        purged = np.flatnonzero(blocked & ~np.isin(np.arange(len(starts)), test))
+        embargoed = np.zeros(len(starts), dtype=bool)
+        for block in selected:
+            last = partitions[block][-1]
+            # Embargo starts after test labels end, not just their observation start.
+            stop_time = ends[partitions[block]].max()
+            first = max(last + 1, starts.searchsorted(stop_time, side="right"))
+            embargoed[first : first + embargo] = True
+        train = np.flatnonzero(~(blocked | embargoed))
+        if not len(train):
+            raise ValueError("no training data after purge/embargo")
+        records.append(
+            {
+                "train_indices": train,
+                "test_indices": test,
+                "test_blocks": selected,
+                "purged_indices": purged,
+                "embargoed_indices": np.flatnonzero(embargoed),
+                "estimand": "purged-combinatorial-validation-not-original-CSCV",
+            }
+        )
+    return records
+
+
+def family_statistics(returns, *, periods_per_year, blocks=8):
+    """Optional shared research layer; pip install quant-factors[research]."""
+    from quant_lab.selection import cscv_pbo, deflated_sharpe
+
+    return {
+        "dsr": deflated_sharpe(returns, periods_per_year=periods_per_year),
+        "cscv": cscv_pbo(returns, blocks=blocks),
+    }
+
+
+def dependent_mean_evidence(values, *, block_lengths, lags, seed=17, repetitions=1000):
+    from quant_lab.selection import bootstrap_means, hac_mean
+
+    return {
+        "bootstrap": bootstrap_means(
+            values, block_lengths=block_lengths, repetitions=repetitions, seed=seed
+        ),
+        "hac": {name: hac_mean(values[name], lags=lags) for name in values},
+    }
+
+
+def nested_factor_selection(data, candidates, **settings):
+    """Fit direction, windows and neutralization inside callbacks, never globally.
+
+    One row per decision date; panel-aware callbacks may use a bounded per-date
+    cross section. Return values are descriptive scores, not a funded OOS account.
+    Requires the optional ``research`` dependency group.
+    """
+    from quant_lab.nested import nested_selection
+
+    return nested_selection(data, candidates, **settings)

src/quant_factors/research.py

@@ -49,6 +49,8 @@ def factor_report(
     baseline_names: tuple[str, ...] = (),
     neutralize_by: tuple[str, ...] = (),
     signal_delay: int = 0,
+    terminal_labels: pd.DataFrame | None = None,
+    sessions=None,
 ) -> dict:
     """Coverage, IC decay, redundancy, yearly and optional industry/regime evidence.
 
@@ -82,11 +84,63 @@ def factor_report(
             if transformed is not None:
                 transformed[names] = transformed.groupby("date")[names].rank(pct=True)
                 transformed[names] = transformed.groupby("symbol")[names].shift(signal_delay)
+    from quant_data_kit.financial.labels import forward_labels
+
+    grid = (
+        pd.DatetimeIndex(sorted(panel.date.unique()))
+        if sessions is None
+        else pd.DatetimeIndex(sessions)
+    )
+    selected = panel[["date", "symbol"]].rename(columns={"symbol": "instrument_id"})
+    selected["sample_id"] = (
+        selected.instrument_id.astype(str) + ":" + selected.date.dt.strftime("%Y-%m-%d")
+    )
+    levels = selected[["date", "instrument_id"]].copy()
+    levels["value"] = panel.get("return_close", panel.close)
+    levels["available_at"] = (
+        pd.to_datetime(panel.available_at, utc=True)
+        if "available_at" in panel
+        else panel.date.dt.tz_localize("UTC") + pd.Timedelta(hours=23, minutes=59)
+    )
+    levels = levels.loc[np.isfinite(levels.value) & levels.value.gt(0)]
+    label_records = []
     for horizon in horizons:
-        panel[f"return_{horizon}"] = panel.groupby("symbol").close.transform(
-            lambda x, h=horizon: x.shift(-h) / x - 1
+        terminal = None
+        if terminal_labels is not None:
+            if "horizon" not in terminal_labels:
+                raise ValueError("terminal label evidence must declare its horizon")
+            terminal = terminal_labels.loc[terminal_labels.horizon.eq(horizon)].drop(
+                columns="horizon"
+            )
+            terminal = terminal.loc[terminal.sample_id.isin(selected.sample_id)]
+        labels = forward_labels(
+            levels,
+            selected,
+            grid,
+            horizon=horizon,
+            as_of=cutoff_date.tz_localize("UTC"),
+            terminals=terminal,
         )
-        panel[f"end_{horizon}"] = panel.groupby("symbol").date.shift(-horizon)
+        panel[f"return_{horizon}"] = labels["return"].to_numpy(dtype=float)
+        panel[f"end_{horizon}"] = pd.to_datetime(labels.label_end).to_numpy()
+        for row in labels.to_dict("records"):
+            if (
+                start
+                and row["date"] < pd.Timestamp(start)
+                or end
+                and row["date"] > pd.Timestamp(end)
+            ):
+                continue
+            label_records.append(
+                {
+                    "sample_id": row["sample_id"],
+                    "date": str(row["date"].date()),
+                    "symbol": row["instrument_id"],
+                    "horizon": horizon,
+                    "status": row["status"],
+                    "return": None if pd.isna(row["return"]) else row["return"],
+                }
+            )
     if start:
         panel = panel[panel.date >= pd.Timestamp(start)]
     if end:
@@ -237,6 +291,19 @@ def factor_report(
         "cutoff": str(cutoff_date.date()),
         "requirements": requirements,
         "coverage": coverage,
+        "label_samples": label_records,
+        "label_coverage": [
+            {
+                "horizon": h,
+                "selected_samples": sum(r["horizon"] == h for r in label_records),
+                "status_counts": pd.Series(
+                    [r["status"] for r in label_records if r["horizon"] == h]
+                )
+                .value_counts()
+                .to_dict(),
+            }
+            for h in horizons
+        ],
         "ic_decay": evidence,
         "segments": segments,
         "correlations": redundancy,