新增美国市场研究数据契约与SEC时点财报

docs/us-research.md

@@ -0,0 +1,15 @@
+# US research data v1
+
+`quant_data_kit.us_research` is an explicit USD research namespace. It does not change the legacy six-digit domestic symbol adapters.
+
+- `calendar.schedule`: XNYS sessions with UTC opens/closes, DST and half-days.
+- `calendar.settlement_session`: standard T+2/T+1 date regimes from 2018.
+- `prices.validate_prices/write_bundle/load_bundle`: OHLC/action validation and immutable hashed CSV snapshots.
+- `prices.fetch_yahoo`: opt-in Yahoo retrospective split-normalized prices, not raw as-traded prices or historical constituent certification. Dividend payment dates remain unknown.
+- `universe.members_asof`: source-backed membership events selected by effective session and knowledge time.
+- `sec.download_sec/read_sec_snapshot`: explicit SEC contact identity, rate-limited snapshots, full submissions pagination and hash checks.
+- `sec.filing_table/fact_table/facts_asof/annual_quality`: accession acceptance plus modeled processing lag, retained restatements, period-matched annual quality metrics. Facts with unknown acceptance are excluded and audited.
+
+Install `.[us-research]` for optional calendar/Yahoo/SEC clients. Unit tests never require live data or contact credentials. `historical_pit` is source evidence supplied by the importer, not automatic certification by the package.
+
+Settlement uses regular exchange holidays plus Columbus/Veterans bank holidays, independently of exchange-only ad-hoc closures. Saturday Veterans Day is not observed on Friday by Federal Reserve Banks. The daily research account models settled funds as available on the settlement date, without intraday clearing or broker-specific holds. Sources: [Federal Reserve bank holidays](https://www.federalreserve.gov/releases/k8/default.htm), [DTCC Veterans Day 2024](https://www.dtcc.com/-/media/Files/pdf/2024/10/11/a9503.pdf), [SIFMA January 2025 closure matrix](https://www.sifma.org/resources/guides-playbooks/unscheduled-close-market-matrix). Future or exceptional clearing changes require calendar review.

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+us-research = ["exchange-calendars>=4.5,<5", "yfinance>=0.2.65,<1", "requests>=2.32,<3"]
 hong-kong = ["akshare>=1.12", "exchange-calendars>=4.10,<5"]
 akshare = ["akshare>=1.12"]
 baostock = ["baostock>=0.8.9"]
@@ -36,7 +37,7 @@ all-equity-providers = [
     "yfinance>=0.2.65",
     "requests>=2.32,<3",
 ]
-dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0"]
+dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0", "exchange-calendars>=4.5,<5"]
 
 [project.scripts]
 qdk-validate = "quant_data_kit.cli:main_validate"

requirements.lock

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --output-file=requirements.lock --strip-extras pyproject.toml
+#    pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --no-index --output-file=requirements.lock --strip-extras pyproject.toml
 #
 attrs==26.1.0
     # via
@@ -14,6 +14,8 @@ coverage==7.16.0
     # via quant-data-kit (pyproject.toml)
 duckdb==1.5.5
     # via quant-data-kit (pyproject.toml)
+exchange-calendars==4.13.2
+    # via quant-data-kit (pyproject.toml)
 iniconfig==2.3.0
     # via pytest
 jsonschema==4.26.0
@@ -22,16 +24,21 @@ jsonschema-rs==0.52.0
     # via quant-data-kit (pyproject.toml)
 jsonschema-specifications==2025.9.1
     # via jsonschema
+korean-lunar-calendar==0.4.0
+    # via exchange-calendars
 numpy==2.2.6
     # via
+    #   exchange-calendars
     #   pandas
     #   quant-data-kit (pyproject.toml)
 orjson==3.12.0
     # via quant-data-kit (pyproject.toml)
 packaging==26.3
     # via pytest
 pandas==2.3.3
-    # via quant-data-kit (pyproject.toml)
+    # via
+    #   exchange-calendars
+    #   quant-data-kit (pyproject.toml)
 pluggy==1.6.0
     # via pytest
 psutil==7.2.2
@@ -40,6 +47,8 @@ pyarrow==25.0.1
     # via quant-data-kit (pyproject.toml)
 pygments==2.21.0
     # via pytest
+pyluach==2.3.0
+    # via exchange-calendars
 pytest==9.1.1
     # via quant-data-kit (pyproject.toml)
 python-dateutil==2.9.0.post0
@@ -61,12 +70,16 @@ ruff==0.16.5
     # via quant-data-kit (pyproject.toml)
 six==1.17.0
     # via python-dateutil
+toolz==1.1.0
+    # via exchange-calendars
 typing-extensions==4.16.0
     # via
     #   quant-data-kit (pyproject.toml)
     #   referencing
 tzdata==2026.3
-    # via pandas
+    # via
+    #   exchange-calendars
+    #   pandas
 websockets==15.0.1
     # via quant-data-kit (pyproject.toml)
 

src/quant_data_kit/us_research/__init__.py

@@ -0,0 +1,6 @@
+"""Explicit US research contracts, separate from legacy domestic adapters."""
+
+from .calendar import schedule, settlement_session
+from .prices import load_bundle, validate_prices, write_bundle
+
+__all__ = ["load_bundle", "schedule", "settlement_session", "validate_prices", "write_bundle"]

src/quant_data_kit/us_research/calendar.py

@@ -0,0 +1,54 @@
+"""Regular US equity sessions; UTC instants retain DST and early closes."""
+
+from functools import lru_cache
+from typing import ClassVar
+
+import pandas as pd
+from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, USColumbusDay, sunday_to_monday
+
+
+class _AdditionalBankHolidays(AbstractHolidayCalendar):
+    # Federal Reserve Banks do not move a Saturday Veterans Day to Friday.
+    rules: ClassVar = [
+        USColumbusDay,
+        Holiday("Veterans Day", month=11, day=11, observance=sunday_to_monday),
+    ]
+
+
+@lru_cache(maxsize=32)
+def _calendar(start: str, end: str):
+    import exchange_calendars as xcals
+
+    return xcals.get_calendar("XNYS", start=start, end=end)
+
+
+def schedule(start: str, end: str) -> pd.DataFrame:
+    start_ts, end_ts = pd.Timestamp(start).normalize(), pd.Timestamp(end).normalize()
+    if start_ts.tz is not None or end_ts.tz is not None or start_ts > end_ts:
+        raise ValueError("session bounds must be ordered, timezone-naive dates")
+    cal = _calendar(str(start_ts.date()), str((end_ts + pd.Timedelta(days=15)).date()))
+    result = cal.schedule.loc[str(start_ts.date()) : str(end_ts.date()), ["open", "close"]].copy()
+    result.index = pd.DatetimeIndex(result.index).tz_localize(None)
+    result.index.name = "session"
+    return result
+
+
+@lru_cache(maxsize=4096)
+def settlement_session(session: str) -> str:
+    """Standard US equity cash settlement since 2018, not same-day resale rules."""
+    day = pd.Timestamp(session).normalize()
+    if day < pd.Timestamp("2018-01-01"):
+        raise ValueError("US settlement model is supported from 2018 onward")
+    end = day + pd.Timedelta(days=15)
+    trading_days = schedule(str(day.date()), str(end.date())).index
+    if not len(trading_days) or trading_days[0] != day:
+        raise ValueError("trade date is not a regular US equity session")
+    # Settlement has its own calendar: bank holidays prevent cash settlement,
+    # while exchange-only ad-hoc closures (e.g. national mourning) need not.
+    cal = _calendar(str(day.date()), str(end.date()))
+    holidays = cal.regular_holidays.holidays(day, end).union(
+        _AdditionalBankHolidays().holidays(day, end)
+    )
+    days = pd.bdate_range(day + pd.Timedelta(days=1), end).difference(holidays)
+    lag = 1 if day >= pd.Timestamp("2024-05-28") else 2
+    return str(days[lag - 1].date())

src/quant_data_kit/us_research/prices.py

@@ -0,0 +1,230 @@
+"""Immutable USD daily research panels with explicit adjustment semantics."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from .calendar import schedule
+
+REQUIRED = {
+    "instrument_id",
+    "symbol",
+    "session",
+    "open",
+    "high",
+    "low",
+    "close",
+    "volume",
+    "dividend",
+    "split_ratio",
+    "currency",
+    "price_basis",
+    "available_at",
+    "source",
+}
+
+
+def utc(value, field: str = "timestamp") -> pd.Timestamp:
+    stamp = pd.Timestamp(value)
+    if pd.isna(stamp) or stamp.tz is None:
+        raise ValueError(f"{field} requires a timezone-aware timestamp")
+    return stamp.tz_convert("UTC")
+
+
+def us_symbol(value: str) -> str:
+    symbol = value.strip().upper()
+    if not re.fullmatch(r"[A-Z][A-Z0-9]*(?:[.-][A-Z0-9]+)*", symbol):
+        raise ValueError(f"invalid explicit US equity symbol: {value!r}")
+    return symbol
+
+
+def validate_prices(frame: pd.DataFrame) -> pd.DataFrame:
+    missing = REQUIRED - set(frame.columns)
+    if missing or frame.empty:
+        raise ValueError(f"nonempty US price panel required; missing {sorted(missing)}")
+    out = frame.copy()
+    for name in ("instrument_id", "symbol", "source"):
+        if out[name].isna().any() or out[name].astype(str).str.strip().eq("").any():
+            raise ValueError(f"empty {name}")
+    out["symbol"] = out.symbol.map(us_symbol)
+    out["session"] = pd.to_datetime(out.session, errors="raise")
+    if out.session.dt.tz is not None or not out.session.eq(out.session.dt.normalize()).all():
+        raise ValueError("session must contain plain dates")
+    if out.duplicated(["instrument_id", "session"]).any():
+        raise ValueError("duplicate instrument/session")
+    if not out.currency.eq("USD").all():
+        raise ValueError("US research requires explicit USD prices")
+    if not out.price_basis.isin(["raw", "split_normalized"]).all():
+        raise ValueError("prices must be raw or split_normalized, never total-return prices")
+    if out.groupby("instrument_id").price_basis.nunique().gt(1).any():
+        raise ValueError("price basis changed within an instrument")
+    numeric = ["open", "high", "low", "close", "volume", "dividend", "split_ratio"]
+    for name in numeric:
+        out[name] = pd.to_numeric(out[name], errors="raise")
+    if not np.isfinite(out[numeric].to_numpy(dtype=float)).all():
+        raise ValueError("nonfinite price or action")
+    if (out[["open", "high", "low", "close", "split_ratio"]] <= 0).any().any():
+        raise ValueError("prices and split ratios must be positive")
+    if (out[["volume", "dividend"]] < 0).any().any():
+        raise ValueError("volume and dividends must be nonnegative")
+    if (
+        (out.high < out[["open", "close", "low"]].max(axis=1))
+        | (out.low > out[["open", "close", "high"]].min(axis=1))
+    ).any():
+        raise ValueError("inconsistent OHLC")
+    sessions = schedule(str(out.session.min().date()), str(out.session.max().date()))
+    if not out.session.isin(sessions.index).all():
+        raise ValueError("price row outside regular US equity sessions")
+    for _, group in out.groupby("instrument_id"):
+        expected = sessions.loc[group.session.min() : group.session.max()].index
+        if len(group) != len(expected):
+            raise ValueError("missing interior daily bars; explicit status handling is required")
+    out["available_at"] = out.available_at.map(lambda value: utc(value, "available_at"))
+    closes = out.session.map(sessions.close)
+    if (out.available_at < closes).any():
+        raise ValueError("complete daily bars cannot be available before session close")
+    if "pay_date" not in out:
+        out["pay_date"] = None
+    for row in out.loc[out.pay_date.notna() & out.pay_date.ne("")].itertuples():
+        pay = pd.Timestamp(row.pay_date)
+        if pay.tz is not None or pay < row.session or pay != pay.normalize():
+            raise ValueError("invalid dividend payment date")
+    return out.sort_values(["session", "instrument_id"]).reset_index(drop=True)
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def write_bundle(
+    path: str | Path,
+    prices: pd.DataFrame,
+    provenance: dict,
+    *,
+    tables: dict[str, pd.DataFrame] | None = None,
+) -> dict:
+    prices = validate_prices(prices)
+    required = {"source", "observed_at", "evidence_kind", "universe_kind"}
+    if not required.issubset(provenance):
+        raise ValueError(f"provenance needs {sorted(required)}")
+    utc(provenance["observed_at"], "observed_at")
+    if provenance["evidence_kind"] not in {"synthetic", "retrospective", "historical_pit"}:
+        raise ValueError("unknown evidence kind")
+    path = Path(path)
+    path.mkdir(parents=True, exist_ok=False)
+    payload = {"prices": prices, **(tables or {})}
+    if payload["prices"] is not prices:
+        raise ValueError("cannot replace validated prices")
+    files = {}
+    for name, table in payload.items():
+        if not re.fullmatch(r"[a-z][a-z0-9_]*", name):
+            raise ValueError("invalid table name")
+        target = path / f"{name}.csv"
+        table.to_csv(target, index=False, lineterminator="\n")
+        files[target.name] = {"sha256": sha256(target), "rows": len(table)}
+    manifest = {"schema": "puresaber.us-research/1", "provenance": provenance, "files": files}
+    (path / "manifest.json").write_text(
+        json.dumps(manifest, ensure_ascii=False, indent=2, allow_nan=False), encoding="utf-8"
+    )
+    return manifest
+
+
+def load_bundle(path: str | Path) -> tuple[dict, dict[str, pd.DataFrame]]:
+    path = Path(path)
+    manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8"))
+    if manifest.get("schema") != "puresaber.us-research/1":
+        raise ValueError("unsupported bundle schema")
+    tables = {}
+    for name, entry in manifest["files"].items():
+        if not re.fullmatch(r"[a-z][a-z0-9_]*\.csv", name):
+            raise ValueError("invalid manifest file name")
+        target = path / name
+        if sha256(target) != entry["sha256"]:
+            raise ValueError(f"bundle hash mismatch: {name}")
+        table = pd.read_csv(target, dtype={"instrument_id": str, "symbol": str})
+        if len(table) != entry["rows"]:
+            raise ValueError(f"bundle row count mismatch: {name}")
+        tables[name[:-4]] = table
+    tables["prices"] = validate_prices(tables["prices"])
+    return manifest, tables
+
+
+def fetch_yahoo(symbols: list[str], start: str, end: str) -> tuple[pd.DataFrame, dict]:
+    """Retrospective split-normalized quotes. No historical-universe/PIT claim.
+
+    Yahoo OHLC with auto_adjust=False is split-normalized, not as-traded raw.
+    Reported splits remain metadata; consumers must not multiply positions again.
+    Payment dates are not supplied, so dividends stay receivable in cash replay.
+    """
+    import yfinance as yf
+
+    if not symbols or len(set(symbols)) != len(symbols):
+        raise ValueError("unique symbols required")
+    sessions = schedule(start, end)
+    observed = pd.Timestamp.now(tz="UTC")
+    if sessions.empty or sessions.close.max() >= observed:
+        raise ValueError("request completed historical sessions only")
+    pieces = []
+    for symbol in symbols:
+        symbol = us_symbol(symbol)
+        ticker = yf.Ticker(symbol)
+        history = ticker.history(
+            start=start,
+            end=str((pd.Timestamp(end) + pd.Timedelta(days=1)).date()),
+            auto_adjust=False,
+            back_adjust=False,
+            actions=True,
+            repair=False,
+            raise_errors=True,
+        )
+        metadata = ticker.get_history_metadata()
+        if metadata.get("currency") != "USD":
+            raise ValueError(f"{symbol}: source did not confirm USD")
+        if history.empty or not {"Dividends", "Stock Splits"}.issubset(history):
+            raise ValueError(f"{symbol}: missing prices/actions")
+        local_dates = history.index.tz_convert("America/New_York").tz_localize(None).normalize()
+        for day, (_, row) in zip(local_dates, history.iterrows(), strict=True):
+            if day not in sessions.index:
+                raise ValueError(f"{symbol}: unexpected session {day}")
+            pieces.append(
+                {
+                    "instrument_id": f"US:YAHOO:{symbol}",
+                    "symbol": symbol,
+                    "session": day,
+                    "open": row.Open,
+                    "high": row.High,
+                    "low": row.Low,
+                    "close": row.Close,
+                    "volume": row.Volume,
+                    "dividend": row.Dividends,
+                    "split_ratio": row["Stock Splits"] or 1.0,
+                    "currency": "USD",
+                    "price_basis": "split_normalized",
+                    "available_at": sessions.loc[day, "close"],
+                    "source": f"yfinance:{yf.__version__}",
+                    "pay_date": None,
+                }
+            )
+    provenance = {
+        "source": "Yahoo Finance via yfinance",
+        "observed_at": observed.isoformat(),
+        "evidence_kind": "retrospective",
+        "universe_kind": "fixed_cohort",
+        "symbols": symbols,
+        "start": start,
+        "end": end,
+        "limitations": [
+            "Current symbol identities; no historical constituent certification",
+            "Close timestamps are modeled availability, not archived PIT evidence",
+            "Split-normalized research share units, not as-traded shares",
+            "Dividend payment dates absent; cash replay retains receivables",
+            "Yahoo revisions and licensing apply; snapshot is not a redistribution grant",
+        ],
+    }
+    return validate_prices(pd.DataFrame(pieces)), provenance