From f4a3fe532b59ed4e5112ad9d188245c180773855 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:45:29 +0800 Subject: [PATCH 1/2] Add explicit US research snapshots and SEC point-in-time facts --- docs/us-research.md | 13 ++ pyproject.toml | 3 +- requirements.lock | 19 +- src/quant_data_kit/us_research/__init__.py | 6 + src/quant_data_kit/us_research/calendar.py | 36 +++ src/quant_data_kit/us_research/prices.py | 230 ++++++++++++++++++ src/quant_data_kit/us_research/sec.py | 239 +++++++++++++++++++ src/quant_data_kit/us_research/universe.py | 37 +++ tests/test_us_research.py | 258 +++++++++++++++++++++ 9 files changed, 837 insertions(+), 4 deletions(-) create mode 100644 docs/us-research.md create mode 100644 src/quant_data_kit/us_research/__init__.py create mode 100644 src/quant_data_kit/us_research/calendar.py create mode 100644 src/quant_data_kit/us_research/prices.py create mode 100644 src/quant_data_kit/us_research/sec.py create mode 100644 src/quant_data_kit/us_research/universe.py create mode 100644 tests/test_us_research.py diff --git a/docs/us-research.md b/docs/us-research.md new file mode 100644 index 0000000..2131d57 --- /dev/null +++ b/docs/us-research.md @@ -0,0 +1,13 @@ +# US research data v1 + +`quant_data_kit.us_research` is an explicit USD research namespace. It does not change the legacy six-digit domestic symbol adapters. + +- `calendar.schedule`: XNYS sessions with UTC opens/closes, DST and half-days. +- `calendar.settlement_session`: standard T+2/T+1 date regimes from 2018. +- `prices.validate_prices/write_bundle/load_bundle`: OHLC/action validation and immutable hashed CSV snapshots. +- `prices.fetch_yahoo`: opt-in Yahoo retrospective split-normalized prices, not raw as-traded prices or historical constituent certification. Dividend payment dates remain unknown. +- `universe.members_asof`: source-backed membership events selected by effective session and knowledge time. +- `sec.download_sec/read_sec_snapshot`: explicit SEC contact identity, rate-limited snapshots, full submissions pagination and hash checks. +- `sec.filing_table/fact_table/facts_asof/annual_quality`: accession acceptance plus modeled processing lag, retained restatements, period-matched annual quality metrics. Facts with unknown acceptance are excluded and audited. + +Install `.[us-research]` for optional calendar/Yahoo/SEC clients. Unit tests never require live data or contact credentials. `historical_pit` is source evidence supplied by the importer, not automatic certification by the package. diff --git a/pyproject.toml b/pyproject.toml index 3098d23..f821ad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ ] [project.optional-dependencies] +us-research = ["exchange-calendars>=4.5,<5", "yfinance>=0.2.65,<1", "requests>=2.32,<3"] akshare = ["akshare>=1.12"] baostock = ["baostock>=0.8.9"] tushare = ["tushare>=1.4.24"] @@ -35,7 +36,7 @@ all-equity-providers = [ "yfinance>=0.2.65", "requests>=2.32,<3", ] -dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0"] +dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0", "exchange-calendars>=4.5,<5"] [project.scripts] qdk-validate = "quant_data_kit.cli:main_validate" diff --git a/requirements.lock b/requirements.lock index 4908d9a..7e2192b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --output-file=requirements.lock --strip-extras pyproject.toml +# pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --no-index --output-file=requirements.lock --strip-extras pyproject.toml # attrs==26.1.0 # via @@ -14,6 +14,8 @@ coverage==7.16.0 # via quant-data-kit (pyproject.toml) duckdb==1.5.5 # via quant-data-kit (pyproject.toml) +exchange-calendars==4.13.2 + # via quant-data-kit (pyproject.toml) iniconfig==2.3.0 # via pytest jsonschema==4.26.0 @@ -22,8 +24,11 @@ jsonschema-rs==0.52.0 # via quant-data-kit (pyproject.toml) jsonschema-specifications==2025.9.1 # via jsonschema +korean-lunar-calendar==0.4.0 + # via exchange-calendars numpy==2.2.6 # via + # exchange-calendars # pandas # quant-data-kit (pyproject.toml) orjson==3.12.0 @@ -31,7 +36,9 @@ orjson==3.12.0 packaging==26.3 # via pytest pandas==2.3.3 - # via quant-data-kit (pyproject.toml) + # via + # exchange-calendars + # quant-data-kit (pyproject.toml) pluggy==1.6.0 # via pytest psutil==7.2.2 @@ -40,6 +47,8 @@ pyarrow==25.0.1 # via quant-data-kit (pyproject.toml) pygments==2.21.0 # via pytest +pyluach==2.3.0 + # via exchange-calendars pytest==9.1.1 # via quant-data-kit (pyproject.toml) python-dateutil==2.9.0.post0 @@ -61,12 +70,16 @@ ruff==0.16.5 # via quant-data-kit (pyproject.toml) six==1.17.0 # via python-dateutil +toolz==1.1.0 + # via exchange-calendars typing-extensions==4.16.0 # via # quant-data-kit (pyproject.toml) # referencing tzdata==2026.3 - # via pandas + # via + # exchange-calendars + # pandas websockets==15.0.1 # via quant-data-kit (pyproject.toml) diff --git a/src/quant_data_kit/us_research/__init__.py b/src/quant_data_kit/us_research/__init__.py new file mode 100644 index 0000000..e06caaa --- /dev/null +++ b/src/quant_data_kit/us_research/__init__.py @@ -0,0 +1,6 @@ +"""Explicit US research contracts, separate from legacy domestic adapters.""" + +from .calendar import schedule, settlement_session +from .prices import load_bundle, validate_prices, write_bundle + +__all__ = ["load_bundle", "schedule", "settlement_session", "validate_prices", "write_bundle"] diff --git a/src/quant_data_kit/us_research/calendar.py b/src/quant_data_kit/us_research/calendar.py new file mode 100644 index 0000000..b51076f --- /dev/null +++ b/src/quant_data_kit/us_research/calendar.py @@ -0,0 +1,36 @@ +"""Regular US equity sessions; UTC instants retain DST and early closes.""" + +from functools import lru_cache + +import pandas as pd + + +@lru_cache(maxsize=32) +def _calendar(start: str, end: str): + import exchange_calendars as xcals + + return xcals.get_calendar("XNYS", start=start, end=end) + + +def schedule(start: str, end: str) -> pd.DataFrame: + start_ts, end_ts = pd.Timestamp(start).normalize(), pd.Timestamp(end).normalize() + if start_ts.tz is not None or end_ts.tz is not None or start_ts > end_ts: + raise ValueError("session bounds must be ordered, timezone-naive dates") + cal = _calendar(str(start_ts.date()), str((end_ts + pd.Timedelta(days=15)).date())) + result = cal.schedule.loc[str(start_ts.date()) : str(end_ts.date()), ["open", "close"]].copy() + result.index = pd.DatetimeIndex(result.index).tz_localize(None) + result.index.name = "session" + return result + + +@lru_cache(maxsize=4096) +def settlement_session(session: str) -> str: + """Standard US equity cash settlement since 2018, not same-day resale rules.""" + day = pd.Timestamp(session).normalize() + if day < pd.Timestamp("2018-01-01"): + raise ValueError("US settlement model is supported from 2018 onward") + days = schedule(str(day.date()), str((day + pd.Timedelta(days=15)).date())).index + if not len(days) or days[0] != day: + raise ValueError("trade date is not a regular US equity session") + lag = 1 if day >= pd.Timestamp("2024-05-28") else 2 + return str(days[lag].date()) diff --git a/src/quant_data_kit/us_research/prices.py b/src/quant_data_kit/us_research/prices.py new file mode 100644 index 0000000..a3955b3 --- /dev/null +++ b/src/quant_data_kit/us_research/prices.py @@ -0,0 +1,230 @@ +"""Immutable USD daily research panels with explicit adjustment semantics.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import numpy as np +import pandas as pd + +from .calendar import schedule + +REQUIRED = { + "instrument_id", + "symbol", + "session", + "open", + "high", + "low", + "close", + "volume", + "dividend", + "split_ratio", + "currency", + "price_basis", + "available_at", + "source", +} + + +def utc(value, field: str = "timestamp") -> pd.Timestamp: + stamp = pd.Timestamp(value) + if pd.isna(stamp) or stamp.tz is None: + raise ValueError(f"{field} requires a timezone-aware timestamp") + return stamp.tz_convert("UTC") + + +def us_symbol(value: str) -> str: + symbol = value.strip().upper() + if not re.fullmatch(r"[A-Z][A-Z0-9]*(?:[.-][A-Z0-9]+)*", symbol): + raise ValueError(f"invalid explicit US equity symbol: {value!r}") + return symbol + + +def validate_prices(frame: pd.DataFrame) -> pd.DataFrame: + missing = REQUIRED - set(frame.columns) + if missing or frame.empty: + raise ValueError(f"nonempty US price panel required; missing {sorted(missing)}") + out = frame.copy() + for name in ("instrument_id", "symbol", "source"): + if out[name].isna().any() or out[name].astype(str).str.strip().eq("").any(): + raise ValueError(f"empty {name}") + out["symbol"] = out.symbol.map(us_symbol) + out["session"] = pd.to_datetime(out.session, errors="raise") + if out.session.dt.tz is not None or not out.session.eq(out.session.dt.normalize()).all(): + raise ValueError("session must contain plain dates") + if out.duplicated(["instrument_id", "session"]).any(): + raise ValueError("duplicate instrument/session") + if not out.currency.eq("USD").all(): + raise ValueError("US research requires explicit USD prices") + if not out.price_basis.isin(["raw", "split_normalized"]).all(): + raise ValueError("prices must be raw or split_normalized, never total-return prices") + if out.groupby("instrument_id").price_basis.nunique().gt(1).any(): + raise ValueError("price basis changed within an instrument") + numeric = ["open", "high", "low", "close", "volume", "dividend", "split_ratio"] + for name in numeric: + out[name] = pd.to_numeric(out[name], errors="raise") + if not np.isfinite(out[numeric].to_numpy(dtype=float)).all(): + raise ValueError("nonfinite price or action") + if (out[["open", "high", "low", "close", "split_ratio"]] <= 0).any().any(): + raise ValueError("prices and split ratios must be positive") + if (out[["volume", "dividend"]] < 0).any().any(): + raise ValueError("volume and dividends must be nonnegative") + if ( + (out.high < out[["open", "close", "low"]].max(axis=1)) + | (out.low > out[["open", "close", "high"]].min(axis=1)) + ).any(): + raise ValueError("inconsistent OHLC") + sessions = schedule(str(out.session.min().date()), str(out.session.max().date())) + if not out.session.isin(sessions.index).all(): + raise ValueError("price row outside regular US equity sessions") + for _, group in out.groupby("instrument_id"): + expected = sessions.loc[group.session.min() : group.session.max()].index + if len(group) != len(expected): + raise ValueError("missing interior daily bars; explicit status handling is required") + out["available_at"] = out.available_at.map(lambda value: utc(value, "available_at")) + closes = out.session.map(sessions.close) + if (out.available_at < closes).any(): + raise ValueError("complete daily bars cannot be available before session close") + if "pay_date" not in out: + out["pay_date"] = None + for row in out.loc[out.pay_date.notna() & out.pay_date.ne("")].itertuples(): + pay = pd.Timestamp(row.pay_date) + if pay.tz is not None or pay < row.session or pay != pay.normalize(): + raise ValueError("invalid dividend payment date") + return out.sort_values(["session", "instrument_id"]).reset_index(drop=True) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_bundle( + path: str | Path, + prices: pd.DataFrame, + provenance: dict, + *, + tables: dict[str, pd.DataFrame] | None = None, +) -> dict: + prices = validate_prices(prices) + required = {"source", "observed_at", "evidence_kind", "universe_kind"} + if not required.issubset(provenance): + raise ValueError(f"provenance needs {sorted(required)}") + utc(provenance["observed_at"], "observed_at") + if provenance["evidence_kind"] not in {"synthetic", "retrospective", "historical_pit"}: + raise ValueError("unknown evidence kind") + path = Path(path) + path.mkdir(parents=True, exist_ok=False) + payload = {"prices": prices, **(tables or {})} + if payload["prices"] is not prices: + raise ValueError("cannot replace validated prices") + files = {} + for name, table in payload.items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + raise ValueError("invalid table name") + target = path / f"{name}.csv" + table.to_csv(target, index=False, lineterminator="\n") + files[target.name] = {"sha256": sha256(target), "rows": len(table)} + manifest = {"schema": "puresaber.us-research/1", "provenance": provenance, "files": files} + (path / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, allow_nan=False), encoding="utf-8" + ) + return manifest + + +def load_bundle(path: str | Path) -> tuple[dict, dict[str, pd.DataFrame]]: + path = Path(path) + manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema") != "puresaber.us-research/1": + raise ValueError("unsupported bundle schema") + tables = {} + for name, entry in manifest["files"].items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*\.csv", name): + raise ValueError("invalid manifest file name") + target = path / name + if sha256(target) != entry["sha256"]: + raise ValueError(f"bundle hash mismatch: {name}") + table = pd.read_csv(target, dtype={"instrument_id": str, "symbol": str}) + if len(table) != entry["rows"]: + raise ValueError(f"bundle row count mismatch: {name}") + tables[name[:-4]] = table + tables["prices"] = validate_prices(tables["prices"]) + return manifest, tables + + +def fetch_yahoo(symbols: list[str], start: str, end: str) -> tuple[pd.DataFrame, dict]: + """Retrospective split-normalized quotes. No historical-universe/PIT claim. + + Yahoo OHLC with auto_adjust=False is split-normalized, not as-traded raw. + Reported splits remain metadata; consumers must not multiply positions again. + Payment dates are not supplied, so dividends stay receivable in cash replay. + """ + import yfinance as yf + + if not symbols or len(set(symbols)) != len(symbols): + raise ValueError("unique symbols required") + sessions = schedule(start, end) + observed = pd.Timestamp.now(tz="UTC") + if sessions.empty or sessions.close.max() >= observed: + raise ValueError("request completed historical sessions only") + pieces = [] + for symbol in symbols: + symbol = us_symbol(symbol) + ticker = yf.Ticker(symbol) + history = ticker.history( + start=start, + end=str((pd.Timestamp(end) + pd.Timedelta(days=1)).date()), + auto_adjust=False, + back_adjust=False, + actions=True, + repair=False, + raise_errors=True, + ) + metadata = ticker.get_history_metadata() + if metadata.get("currency") != "USD": + raise ValueError(f"{symbol}: source did not confirm USD") + if history.empty or not {"Dividends", "Stock Splits"}.issubset(history): + raise ValueError(f"{symbol}: missing prices/actions") + local_dates = history.index.tz_convert("America/New_York").tz_localize(None).normalize() + for day, (_, row) in zip(local_dates, history.iterrows(), strict=True): + if day not in sessions.index: + raise ValueError(f"{symbol}: unexpected session {day}") + pieces.append( + { + "instrument_id": f"US:YAHOO:{symbol}", + "symbol": symbol, + "session": day, + "open": row.Open, + "high": row.High, + "low": row.Low, + "close": row.Close, + "volume": row.Volume, + "dividend": row.Dividends, + "split_ratio": row["Stock Splits"] or 1.0, + "currency": "USD", + "price_basis": "split_normalized", + "available_at": sessions.loc[day, "close"], + "source": f"yfinance:{yf.__version__}", + "pay_date": None, + } + ) + provenance = { + "source": "Yahoo Finance via yfinance", + "observed_at": observed.isoformat(), + "evidence_kind": "retrospective", + "universe_kind": "fixed_cohort", + "symbols": symbols, + "start": start, + "end": end, + "limitations": [ + "Current symbol identities; no historical constituent certification", + "Close timestamps are modeled availability, not archived PIT evidence", + "Split-normalized research share units, not as-traded shares", + "Dividend payment dates absent; cash replay retains receivables", + "Yahoo revisions and licensing apply; snapshot is not a redistribution grant", + ], + } + return validate_prices(pd.DataFrame(pieces)), provenance diff --git a/src/quant_data_kit/us_research/sec.py b/src/quant_data_kit/us_research/sec.py new file mode 100644 index 0000000..8fde43b --- /dev/null +++ b/src/quant_data_kit/us_research/sec.py @@ -0,0 +1,239 @@ +"""SEC snapshots and accession-level facts without backdating restatements.""" + +from __future__ import annotations + +import json +import re +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from .prices import sha256, utc + + +def download_sec(cik: str, destination: str | Path, user_agent: str) -> Path: + """Fetch companyfacts and all submissions pages with a real contact identity.""" + import requests + + if not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", user_agent): + raise ValueError("SEC_USER_AGENT must contain a real contact email") + if not re.fullmatch(r"\d{1,10}", str(cik)): + raise ValueError("CIK must contain 1 to 10 digits") + cik = str(cik).zfill(10) + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + manifest = { + "schema": "puresaber.sec-snapshot/1", + "cik": cik, + "observed_at": pd.Timestamp.now(tz="UTC").isoformat(), + "files": {}, + } + session = requests.Session() + session.headers["User-Agent"] = user_agent + + def fetch(relative: str, name: str): + time.sleep(0.25) + response = session.get("https://data.sec.gov/" + relative, timeout=60) + response.raise_for_status() + payload = response.json() + target = destination / name + target.write_bytes(response.content) + manifest["files"][name] = {"url": response.url, "sha256": sha256(target)} + return payload + + try: + submissions = fetch(f"submissions/CIK{cik}.json", "submissions.json") + for item in submissions["filings"].get("files", []): + name = item["name"] + if not re.fullmatch(r"CIK\d{10}-submissions-\d+\.json", name): + raise ValueError("unexpected SEC archive name") + fetch("submissions/" + name, name) + fetch(f"api/xbrl/companyfacts/CIK{cik}.json", "companyfacts.json") + (destination / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + finally: + session.close() + return destination + + +def read_sec_snapshot(path: str | Path) -> tuple[dict, list[dict], dict]: + path = Path(path) + manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema") != "puresaber.sec-snapshot/1": + raise ValueError("unsupported SEC snapshot") + records = {} + for name, entry in manifest["files"].items(): + if Path(name).name != name or "/" in name or "\\" in name: + raise ValueError("invalid SEC snapshot path") + if sha256(path / name) != entry["sha256"]: + raise ValueError(f"SEC hash mismatch: {name}") + records[name] = json.loads((path / name).read_text(encoding="utf-8")) + submissions = [records["submissions.json"]["filings"]["recent"]] + submissions += [ + value for name, value in records.items() if name.startswith("CIK") and "submissions" in name + ] + return records["companyfacts.json"], submissions, manifest + + +def filing_table( + submissions: list[dict], instrument_id: str, *, lag_minutes: int = 5 +) -> pd.DataFrame: + if isinstance(lag_minutes, bool) or not isinstance(lag_minutes, int) or lag_minutes < 0: + raise ValueError("lag_minutes must be a nonnegative integer") + rows = [] + for page in submissions: + columns = ("accessionNumber", "acceptanceDateTime", "form", "filingDate") + if not all(name in page for name in columns): + raise ValueError("SEC submissions missing acceptance timestamps") + if len({len(page[name]) for name in columns}) != 1: + raise ValueError("SEC submissions columns have different lengths") + for accession, accepted, form, filed in zip(*(page[name] for name in columns), strict=True): + at = utc(accepted, "SEC acceptanceDateTime") + rows.append( + { + "instrument_id": instrument_id, + "accession": accession, + "accepted_at": at, + "available_at": at + pd.Timedelta(minutes=lag_minutes), + "form": form, + "filed": filed, + "source": "SEC EDGAR", + } + ) + result = pd.DataFrame(rows) + if result.empty: + raise ValueError("no SEC filings") + result = result.drop_duplicates() + if result.accession.duplicated().any(): + raise ValueError("conflicting SEC accession metadata") + return result.sort_values(["available_at", "accession"]).reset_index(drop=True) + + +def fact_table( + companyfacts: dict, + filings: pd.DataFrame, + *, + concepts: tuple[str, ...] = ( + "Assets", + "NetIncomeLoss", + "NetCashProvidedByUsedInOperatingActivities", + ), + unit: str = "USD", +) -> tuple[pd.DataFrame, dict]: + """Preserve each filing vintage and exact period; exclude unknown acceptance.""" + index = filings.set_index("accession") + if index.index.duplicated().any() or filings.instrument_id.nunique() != 1: + raise ValueError("one entity and unique accessions required") + rows, missing = [], set() + for concept in concepts: + values = companyfacts.get("facts", {}).get("us-gaap", {}).get(concept, {}) + for fact in values.get("units", {}).get(unit, []): + accession = fact["accn"] + if accession not in index.index: + missing.add(accession) + continue + filing = index.loc[accession] + value = float(fact["val"]) + if not np.isfinite(value): + raise ValueError("nonfinite SEC fact") + end = pd.Timestamp(fact["end"]) + start = pd.Timestamp(fact["start"]) if "start" in fact else pd.NaT + if end.date() > filing.accepted_at.date() or (pd.notna(start) and start > end): + raise ValueError("SEC fact period is inconsistent with filing time") + rows.append( + { + "instrument_id": filing.instrument_id, + "concept": concept, + "unit": unit, + "period_start": start, + "period_end": end, + "value": value, + "accession": accession, + "form": filing.form, + "accepted_at": filing.accepted_at, + "available_at": filing.available_at, + } + ) + columns = [ + "instrument_id", + "concept", + "unit", + "period_start", + "period_end", + "value", + "accession", + "form", + "accepted_at", + "available_at", + ] + result = pd.DataFrame(rows, columns=columns).drop_duplicates() + keys = ["instrument_id", "concept", "unit", "period_start", "period_end", "accession"] + if result.duplicated(keys).any(): + raise ValueError("ambiguous same-accession financial facts") + return result, { + "excluded_unknown_accessions": sorted(missing), + "rows": len(result), + "availability": "SEC acceptance plus configured processing lag", + } + + +def facts_asof(facts: pd.DataFrame, at) -> pd.DataFrame: + facts = facts.copy() + facts["available_at"] = facts.available_at.map(lambda value: utc(value, "fact available_at")) + visible = facts.loc[facts.available_at <= utc(at)].copy() + keys = ["instrument_id", "concept", "unit", "period_start", "period_end"] + return visible.sort_values(["available_at", "accession"]).drop_duplicates(keys, keep="last") + + +def annual_quality(facts: pd.DataFrame, at, *, max_age_days: int = 550) -> pd.DataFrame: + """Annual net-income/assets and cash accruals; no mislabeled quarterly/TTM mixing.""" + visible = facts_asof(facts, at) + visible["period_start"] = pd.to_datetime(visible.period_start) + visible["period_end"] = pd.to_datetime(visible.period_end) + results = [] + for instrument, data in visible.groupby("instrument_id"): + income = data.loc[data.concept.eq("NetIncomeLoss") & data.unit.eq("USD")].copy() + duration = (income.period_end - income.period_start).dt.days + income = income.loc[duration.between(330, 380)].sort_values("period_end") + if income.empty: + continue + latest = income.iloc[-1] + if (utc(at).tz_localize(None) - latest.period_end).days > max_age_days: + continue + assets = data.loc[ + data.concept.eq("Assets") + & data.unit.eq("USD") + & data.period_start.isna() + & data.period_end.eq(latest.period_end) + ] + cash = data.loc[ + data.concept.eq("NetCashProvidedByUsedInOperatingActivities") + & data.unit.eq("USD") + & data.period_start.eq(latest.period_start) + & data.period_end.eq(latest.period_end) + ] + if len(assets) != 1 or len(cash) != 1 or assets.iloc[0].value <= 0: + continue + a, c = assets.iloc[0], cash.iloc[0] + results.append( + { + "instrument_id": instrument, + "period_end": latest.period_end, + "roa_annual": latest.value / a.value, + "accruals_annual": (latest.value - c.value) / a.value, + "available_at": max(latest.available_at, a.available_at, c.available_at), + "accessions": "|".join(sorted({latest.accession, a.accession, c.accession})), + } + ) + return pd.DataFrame( + results, + columns=[ + "instrument_id", + "period_end", + "roa_annual", + "accruals_annual", + "available_at", + "accessions", + ], + ) diff --git a/src/quant_data_kit/us_research/universe.py b/src/quant_data_kit/us_research/universe.py new file mode 100644 index 0000000..aeb7baa --- /dev/null +++ b/src/quant_data_kit/us_research/universe.py @@ -0,0 +1,37 @@ +"""Membership events selected by knowledge time and effective session.""" + +import pandas as pd + +from .prices import utc + + +def validate_membership(frame: pd.DataFrame) -> pd.DataFrame: + required = {"instrument_id", "effective_session", "known_at", "member", "source", "event_id"} + if frame.empty or not required.issubset(frame): + raise ValueError("nonempty historical membership events required") + result = frame.copy() + for field in ("instrument_id", "source", "event_id"): + if result[field].isna().any() or result[field].astype(str).str.strip().eq("").any(): + raise ValueError(f"membership has empty {field}") + if result.event_id.duplicated().any(): + raise ValueError("duplicate membership event") + result["effective_session"] = pd.to_datetime(result.effective_session) + result["known_at"] = result.known_at.map(lambda value: utc(value, "known_at")) + if not result.member.isin([True, False, 0, 1]).all(): + raise ValueError("membership must be boolean") + result["member"] = result.member.astype(bool) + if result.duplicated(["instrument_id", "effective_session", "known_at"]).any(): + raise ValueError("ambiguous membership events") + return result + + +def members_asof(events: pd.DataFrame, session: str, at) -> set[str]: + visible = events.loc[ + (events.known_at <= utc(at)) & (events.effective_session <= pd.Timestamp(session)) + ] + latest = ( + visible.sort_values(["effective_session", "known_at", "event_id"]) + .groupby("instrument_id", sort=False) + .tail(1) + ) + return set(latest.loc[latest.member, "instrument_id"]) diff --git a/tests/test_us_research.py b/tests/test_us_research.py new file mode 100644 index 0000000..93ef350 --- /dev/null +++ b/tests/test_us_research.py @@ -0,0 +1,258 @@ +import json +import sys +from types import SimpleNamespace + +import pandas as pd +import pytest + +from quant_data_kit.us_research.calendar import schedule, settlement_session +from quant_data_kit.us_research.prices import ( + fetch_yahoo, + load_bundle, + us_symbol, + validate_prices, + write_bundle, +) +from quant_data_kit.us_research.sec import ( + annual_quality, + download_sec, + fact_table, + facts_asof, + filing_table, +) +from quant_data_kit.us_research.universe import members_asof, validate_membership + + +@pytest.fixture +def panel(): + cal = schedule("2024-03-08", "2024-03-12") + return pd.DataFrame( + [ + { + "instrument_id": "US:AAPL", + "symbol": "AAPL", + "session": day, + "open": 100, + "high": 102, + "low": 99, + "close": 101, + "volume": 1000, + "dividend": 0, + "split_ratio": 1, + "currency": "USD", + "price_basis": "raw", + "available_at": row.close, + "source": "fixture", + } + for day, row in cal.iterrows() + ] + ) + + +def test_calendar_dst_halfday_settlement(): + cal = schedule("2024-03-08", "2024-03-11") + assert cal.iloc[0].open.hour == 14 + assert cal.iloc[1].open.hour == 13 + assert schedule("2024-11-29", "2024-11-29").iloc[0].close.hour == 18 + assert schedule("2024-07-04", "2024-07-04").empty + assert settlement_session("2024-05-24") == "2024-05-29" + assert settlement_session("2024-05-28") == "2024-05-29" + with pytest.raises(ValueError): + settlement_session("2024-05-27") + with pytest.raises(ValueError): + settlement_session("2017-01-03") + with pytest.raises(ValueError): + schedule("2024-05-30", "2024-05-01") + + +@pytest.mark.parametrize("value", ["AAPL", "BRK.B", "BRK-B", "SPY"]) +def test_us_symbols_are_not_domestic(value): + assert us_symbol(value) == value + + +@pytest.mark.parametrize("value", ["000001", "AAPL.SZ/", "../SPY", ""]) +def test_invalid_symbols(value): + with pytest.raises(ValueError): + us_symbol(value) + + +@pytest.mark.parametrize( + "column,value", + [ + ("close", float("nan")), + ("open", -1), + ("high", 1), + ("volume", -1), + ("dividend", -1), + ("split_ratio", 0), + ("currency", "CNY"), + ("price_basis", "adjusted"), + ("source", ""), + ("available_at", "2024-03-08T09:00:00Z"), + ("available_at", "2024-03-08"), + ("pay_date", "2020-01-01"), + ], +) +def test_bad_panel_rejected(panel, column, value): + panel[column] = panel[column].astype(object) if column in panel else None + panel.loc[0, column] = value + with pytest.raises((ValueError, TypeError)): + validate_prices(panel) + + +def test_bundle_immutable_and_hash_checked(panel, tmp_path): + path = tmp_path / "bundle" + provenance = { + "source": "fixture", + "observed_at": "2024-03-13T00:00:00Z", + "evidence_kind": "synthetic", + "universe_kind": "fixed_cohort", + } + write_bundle(path, panel, provenance) + manifest, tables = load_bundle(path) + assert tables["prices"].currency.eq("USD").all() + assert manifest["provenance"] == provenance + with pytest.raises(FileExistsError): + write_bundle(path, panel, provenance) + with (path / "prices.csv").open("a") as handle: + handle.write("corruption") + with pytest.raises(ValueError, match="hash"): + load_bundle(path) + + +def test_gaps_duplicates_and_basis_are_rejected(panel): + for bad in (panel.drop(index=1), pd.concat([panel, panel.iloc[[0]]])): + with pytest.raises(ValueError): + validate_prices(bad) + panel.loc[0, "price_basis"] = "split_normalized" + with pytest.raises(ValueError): + validate_prices(panel) + + +def test_yahoo_contract_has_no_double_split_or_pit_claim(monkeypatch): + hist = pd.DataFrame( + { + "Open": [50], + "High": [51], + "Low": [49], + "Close": [50], + "Volume": [2000], + "Dividends": [1], + "Stock Splits": [2], + }, + index=pd.DatetimeIndex(["2024-03-08"], tz="America/New_York"), + ) + calls = [] + + class Ticker: + def __init__(self, symbol): + calls.append(symbol) + + def history(self, **kwargs): + assert not kwargs["auto_adjust"] and kwargs["actions"] + return hist + + def get_history_metadata(self): + return {"currency": "USD"} + + monkeypatch.setitem(sys.modules, "yfinance", SimpleNamespace(Ticker=Ticker, __version__="test")) + prices, provenance = fetch_yahoo(["AAPL"], "2024-03-08", "2024-03-08") + assert calls == ["AAPL"] + assert prices.iloc[0].close == 50 + assert prices.iloc[0].price_basis == "split_normalized" + assert provenance["evidence_kind"] == "retrospective" + assert prices.pay_date.isna().all() + + +def test_membership_knowledge_time_not_current_constituents(): + events = validate_membership( + pd.DataFrame( + [ + { + "instrument_id": "OLD", + "effective_session": "2024-01-02", + "known_at": "2024-01-01T00:00Z", + "member": True, + "source": "archive", + "event_id": "1", + }, + { + "instrument_id": "OLD", + "effective_session": "2024-02-01", + "known_at": "2024-02-03T00:00Z", + "member": False, + "source": "revision", + "event_id": "2", + }, + ] + ) + ) + assert members_asof(events, "2024-02-02", "2024-02-02T12:00Z") == {"OLD"} + assert members_asof(events, "2024-02-04", "2024-02-04T12:00Z") == set() + with pytest.raises(ValueError): + validate_membership(pd.concat([events, events])) + + +def test_sec_restatement_vintages_and_periods(): + filings = filing_table( + [ + { + "accessionNumber": ["a", "b"], + "acceptanceDateTime": ["2023-02-01T22:00Z", "2024-02-01T22:00Z"], + "form": ["10-K", "10-K/A"], + "filingDate": ["2023-02-01", "2024-02-01"], + } + ], + "US:A", + ) + concepts = {} + for name, value in [ + ("Assets", 1000), + ("NetIncomeLoss", 100), + ("NetCashProvidedByUsedInOperatingActivities", 130), + ]: + values = [ + {"val": value, "end": "2022-12-31", "accn": "a"}, + {"val": value * 2, "end": "2022-12-31", "accn": "b"}, + ] + if name != "Assets": + for row in values: + row["start"] = "2022-01-01" + concepts[name] = {"units": {"USD": values}} + facts, audit = fact_table({"facts": {"us-gaap": concepts}}, filings) + assert audit["excluded_unknown_accessions"] == [] + assert facts_asof(facts, "2023-02-01T22:04Z").empty + assert set(facts_asof(facts, "2023-02-02T00:00Z").accession) == {"a"} + assert set(facts_asof(facts, "2024-02-02T00:00Z").accession) == {"b"} + quality = annual_quality(facts, "2023-02-02T00:00Z") + assert quality.iloc[0].roa_annual == pytest.approx(0.1) + assert quality.iloc[0].accruals_annual == pytest.approx(-0.03) + assert annual_quality(facts, "2026-01-01T00:00Z").empty + + +def test_sec_missing_acceptance_never_backdated(): + with pytest.raises(ValueError): + filing_table([{"accessionNumber": ["a"]}], "US:A") + with pytest.raises(ValueError, match="contact"): + download_sec("123", "unused", "anonymous") + with pytest.raises(ValueError, match="CIK"): + download_sec("../x", "unused", "test@example.com") + + +def test_manifest_traversal_rejected(panel, tmp_path): + path = tmp_path / "b" + write_bundle( + path, + panel, + { + "source": "x", + "observed_at": "2024-01-01T00:00Z", + "evidence_kind": "synthetic", + "universe_kind": "fixed_cohort", + }, + ) + manifest = json.loads((path / "manifest.json").read_text()) + manifest["files"]["../prices.csv"] = manifest["files"].pop("prices.csv") + (path / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + load_bundle(path) From 6c40fedfd01fb6cff795c3faeacb04ff7f094c83 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:58:48 +0800 Subject: [PATCH 2/2] Separate US settlement calendar and validate SEC identity before optional imports --- docs/us-research.md | 2 ++ src/quant_data_kit/us_research/calendar.py | 24 +++++++++++++++++++--- src/quant_data_kit/us_research/sec.py | 4 ++-- tests/test_us_research.py | 15 ++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/us-research.md b/docs/us-research.md index 2131d57..fe3653f 100644 --- a/docs/us-research.md +++ b/docs/us-research.md @@ -11,3 +11,5 @@ - `sec.filing_table/fact_table/facts_asof/annual_quality`: accession acceptance plus modeled processing lag, retained restatements, period-matched annual quality metrics. Facts with unknown acceptance are excluded and audited. Install `.[us-research]` for optional calendar/Yahoo/SEC clients. Unit tests never require live data or contact credentials. `historical_pit` is source evidence supplied by the importer, not automatic certification by the package. + +Settlement uses regular exchange holidays plus Columbus/Veterans bank holidays, independently of exchange-only ad-hoc closures. Saturday Veterans Day is not observed on Friday by Federal Reserve Banks. The daily research account models settled funds as available on the settlement date, without intraday clearing or broker-specific holds. Sources: [Federal Reserve bank holidays](https://www.federalreserve.gov/releases/k8/default.htm), [DTCC Veterans Day 2024](https://www.dtcc.com/-/media/Files/pdf/2024/10/11/a9503.pdf), [SIFMA January 2025 closure matrix](https://www.sifma.org/resources/guides-playbooks/unscheduled-close-market-matrix). Future or exceptional clearing changes require calendar review. diff --git a/src/quant_data_kit/us_research/calendar.py b/src/quant_data_kit/us_research/calendar.py index b51076f..2135d19 100644 --- a/src/quant_data_kit/us_research/calendar.py +++ b/src/quant_data_kit/us_research/calendar.py @@ -1,8 +1,18 @@ """Regular US equity sessions; UTC instants retain DST and early closes.""" from functools import lru_cache +from typing import ClassVar import pandas as pd +from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, USColumbusDay, sunday_to_monday + + +class _AdditionalBankHolidays(AbstractHolidayCalendar): + # Federal Reserve Banks do not move a Saturday Veterans Day to Friday. + rules: ClassVar = [ + USColumbusDay, + Holiday("Veterans Day", month=11, day=11, observance=sunday_to_monday), + ] @lru_cache(maxsize=32) @@ -29,8 +39,16 @@ def settlement_session(session: str) -> str: day = pd.Timestamp(session).normalize() if day < pd.Timestamp("2018-01-01"): raise ValueError("US settlement model is supported from 2018 onward") - days = schedule(str(day.date()), str((day + pd.Timedelta(days=15)).date())).index - if not len(days) or days[0] != day: + end = day + pd.Timedelta(days=15) + trading_days = schedule(str(day.date()), str(end.date())).index + if not len(trading_days) or trading_days[0] != day: raise ValueError("trade date is not a regular US equity session") + # Settlement has its own calendar: bank holidays prevent cash settlement, + # while exchange-only ad-hoc closures (e.g. national mourning) need not. + cal = _calendar(str(day.date()), str(end.date())) + holidays = cal.regular_holidays.holidays(day, end).union( + _AdditionalBankHolidays().holidays(day, end) + ) + days = pd.bdate_range(day + pd.Timedelta(days=1), end).difference(holidays) lag = 1 if day >= pd.Timestamp("2024-05-28") else 2 - return str(days[lag].date()) + return str(days[lag - 1].date()) diff --git a/src/quant_data_kit/us_research/sec.py b/src/quant_data_kit/us_research/sec.py index 8fde43b..d10ae37 100644 --- a/src/quant_data_kit/us_research/sec.py +++ b/src/quant_data_kit/us_research/sec.py @@ -15,12 +15,12 @@ def download_sec(cik: str, destination: str | Path, user_agent: str) -> Path: """Fetch companyfacts and all submissions pages with a real contact identity.""" - import requests - if not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", user_agent): raise ValueError("SEC_USER_AGENT must contain a real contact email") if not re.fullmatch(r"\d{1,10}", str(cik)): raise ValueError("CIK must contain 1 to 10 digits") + import requests + cik = str(cik).zfill(10) destination = Path(destination) destination.mkdir(parents=True, exist_ok=False) diff --git a/tests/test_us_research.py b/tests/test_us_research.py index 93ef350..e1876a1 100644 --- a/tests/test_us_research.py +++ b/tests/test_us_research.py @@ -65,6 +65,21 @@ def test_calendar_dst_halfday_settlement(): schedule("2024-05-30", "2024-05-01") +@pytest.mark.parametrize( + "trade,settles", + [ + ("2024-10-11", "2024-10-15"), + ("2024-11-08", "2024-11-12"), + ("2023-11-09", "2023-11-13"), + ("2024-03-28", "2024-04-02"), + ("2025-01-08", "2025-01-09"), + ("2018-12-04", "2018-12-06"), + ], +) +def test_settlement_uses_bank_holidays_and_keeps_exchange_only_closures(trade, settles): + assert settlement_session(trade) == settles + + @pytest.mark.parametrize("value", ["AAPL", "BRK.B", "BRK-B", "SPY"]) def test_us_symbols_are_not_domestic(value): assert us_symbol(value) == value