From a459b328f79fbc55663d4f0c041d3805ccde21ce Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 26 Sep 2026 16:24:51 +0800 Subject: [PATCH 1/6] Add evidence-bound historical instrument masters --- docs/RESEARCH_DATASETS.md | 43 ++- src/quant_data_kit/instrument_master.py | 393 ++++++++++++++++++++++++ src/quant_data_kit/research_dataset.py | 140 ++++++++- tests/test_instrument_master.py | 165 ++++++++++ tests/test_research_dataset.py | 54 +++- 5 files changed, 779 insertions(+), 16 deletions(-) create mode 100644 src/quant_data_kit/instrument_master.py create mode 100644 tests/test_instrument_master.py diff --git a/docs/RESEARCH_DATASETS.md b/docs/RESEARCH_DATASETS.md index 06cdef4..e3c3d8e 100644 --- a/docs/RESEARCH_DATASETS.md +++ b/docs/RESEARCH_DATASETS.md @@ -14,6 +14,7 @@ The Python API is available directly from the module: ```python from quant_data_kit.research_dataset import ( + bind_instrument_master, build_dataset, inspect_dataset, load_research_snapshot, @@ -56,11 +57,43 @@ hash-bound `source.json` and Parquet history, so `quant_data_kit.research_coverage.load_history(snapshot / "history")` works without adapting a loose file. -The catalog's `available_at` is the real snapshot capture time. QDK does not -backdate instrument-master availability to the research start. Until a separate -historical exchange/security-master source proves earlier publication, a strict -historical replay must remain blocked; a structurally loadable catalog is not -evidence that the rules were known at each historical decision time. +The default catalog's `available_at` is the real snapshot capture time. QDK +does not backdate instrument-master availability to the research start. A +structurally loadable catalog is not evidence that the rules were known at each +historical decision time. + +## Bind a historical instrument master + +`quant_data_kit.instrument_master` imports an explicit source declaration, +catalog and official documents into one immutable bundle. Import rejects a +changed source hash, an unknown evidence ID, a rule published after its claimed +effective time, a coverage gap and catalog availability before its initial +evidence. Every source needs an HTTPS identity, publication time, effective +interval and exact SHA-256. + +```powershell +python -m quant_data_kit.instrument_master import ` + --source ..\artifacts\pit-sources\source ` + --output ..\artifacts\pit-sources\bundle + +python -m quant_data_kit.research_dataset bind-master ` + --root ..\artifacts\etf-research ` + --instrument-master ..\artifacts\pit-sources\bundle +``` + +Binding publishes a child snapshot. It copies already verified market data and +actions without a provider request, replaces the current-capture catalog with +the evidence-bound catalog, and retains the complete master bundle below +`instrument_master/`. A later incremental update automatically carries that +bundle forward and rechecks its symbol set, validity interval and availability. + +The bundle proves only its declared fields and intervals. A retrospective +listing document becomes usable at the document publication time; it is not +backdated to the listing day. An empty present-day suspension query is not +expanded into daily `tradable=true`, and volume does not prove all-day +tradability. If the source declares a fixed retrospective pool, missing daily +status or missing dynamic-universe history, consumers must preserve those +limits rather than label the result a complete dynamic PIT backtest. Every live manifest records the AKShare package version, QDK version, endpoint names, normalized request and capture time. Every Parquet file has an exact diff --git a/src/quant_data_kit/instrument_master.py b/src/quant_data_kit/instrument_master.py new file mode 100644 index 0000000..fca207b --- /dev/null +++ b/src/quant_data_kit/instrument_master.py @@ -0,0 +1,393 @@ +"""Evidence-bound historical instrument-master imports. + +The bundle proves only the fields and intervals declared in its catalog. It +does not turn a current security list, an empty suspension query, or observed +volume into historical tradability. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sys +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pandas as pd + +SOURCE_SCHEMA = "qdk.historical-instrument-master-source/v1" +BUNDLE_SCHEMA = "qdk.historical-instrument-master/v1" +CATALOG_SCHEMA = "qdk.etf-instrument-catalog/v2" + +BASE_COLUMNS = { + "catalog_schema_version", + "catalog_source_version", + "symbol", + "asset_class", + "product_type", + "venue", + "price_scale", + "price_tick", + "quantity_step", + "lot_size", + "commission_rate", + "stamp_duty_rate", + "effective_from", + "effective_to", + "available_at", + "listing_date", + "listing_date_basis", + "listing_evidence_id", + "trading_rule_evidence_ids", + "price_limit_evidence_ids", +} +DOCUMENT_KINDS = { + "calendar", + "listing", + "price_limit", + "status_observation", + "trading_rule", +} + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + + +def _digest_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _digest(path: Path) -> str: + return _digest_bytes(path.read_bytes()) + + +def _timestamp(value: object, field: str) -> pd.Timestamp: + stamp = pd.Timestamp(value) + if pd.isna(stamp) or stamp.tzinfo is None: + raise ValueError(f"{field} requires an explicit time zone") + return stamp.tz_convert("UTC") + + +def _nonempty(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a nonempty string") + return value.strip() + + +def _relative_file(root: Path, value: object, field: str) -> Path: + relative = Path(_nonempty(value, field)) + if relative.is_absolute(): + raise ValueError(f"{field} must be relative") + resolved = (root / relative).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError as exc: + raise ValueError(f"{field} escapes the evidence root") from exc + if not resolved.is_file(): + raise FileNotFoundError(resolved) + return resolved + + +def _evidence_ids(value: object, field: str) -> list[str]: + ids = [item.strip() for item in _nonempty(value, field).split(";") if item.strip()] + if not ids or len(ids) != len(set(ids)): + raise ValueError(f"{field} must contain unique semicolon-separated IDs") + return ids + + +def _document_index(declaration: dict[str, Any], root: Path) -> dict[str, dict[str, Any]]: + if declaration.get("schema_version") != SOURCE_SCHEMA: + raise ValueError("Unsupported historical instrument-master source schema") + captured_at = _timestamp(declaration.get("captured_at"), "captured_at") + documents = declaration.get("documents") + if not isinstance(documents, list) or not documents: + raise ValueError("Historical instrument master requires evidence documents") + indexed: dict[str, dict[str, Any]] = {} + for raw in documents: + if not isinstance(raw, dict): + raise TypeError("Evidence document entries must be objects") + document_id = _nonempty(raw.get("document_id"), "document_id") + if document_id in indexed: + raise ValueError(f"Duplicate evidence document ID: {document_id}") + kind = _nonempty(raw.get("kind"), f"{document_id}.kind") + if kind not in DOCUMENT_KINDS: + raise ValueError(f"Unsupported evidence document kind: {kind}") + publisher = _nonempty(raw.get("publisher"), f"{document_id}.publisher") + source_uri = _nonempty(raw.get("source_uri"), f"{document_id}.source_uri") + if not source_uri.startswith("https://"): + raise ValueError(f"{document_id}.source_uri must use HTTPS") + published_at = _timestamp(raw.get("published_at"), f"{document_id}.published_at") + effective_from = _timestamp(raw.get("effective_from"), f"{document_id}.effective_from") + effective_to = _timestamp(raw.get("effective_to"), f"{document_id}.effective_to") + if published_at > captured_at: + raise ValueError(f"Evidence {document_id} was published after capture") + if kind in {"trading_rule", "price_limit"} and published_at > effective_from: + raise ValueError(f"Evidence {document_id} was not public before it became effective") + if effective_from >= effective_to: + raise ValueError(f"Evidence {document_id} has an invalid effective interval") + source_file = _relative_file(root, raw.get("file"), f"{document_id}.file") + expected_hash = _nonempty(raw.get("sha256"), f"{document_id}.sha256").lower() + if len(expected_hash) != 64 or any(c not in "0123456789abcdef" for c in expected_hash): + raise ValueError(f"Evidence {document_id} has an invalid SHA-256") + if _digest(source_file) != expected_hash: + raise ValueError(f"Evidence {document_id} hash mismatch") + indexed[document_id] = { + **raw, + "document_id": document_id, + "kind": kind, + "publisher": publisher, + "source_uri": source_uri, + "published_at": published_at, + "effective_from": effective_from, + "effective_to": effective_to, + "source_file": source_file, + "sha256": expected_hash, + } + return indexed + + +def _validate_interval_coverage( + row: pd.Series, + *, + ids: list[str], + documents: dict[str, dict[str, Any]], + kinds: set[str], + field: str, +) -> None: + start = _timestamp(row["effective_from"], "effective_from") + end = _timestamp(row["effective_to"], "effective_to") + intervals = [] + for document_id in ids: + document = documents.get(document_id) + if document is None: + raise ValueError(f"Unknown evidence ID {document_id} in {field}") + if document["kind"] not in kinds: + raise ValueError(f"Evidence {document_id} has the wrong kind for {field}") + intervals.append((document["effective_from"], document["effective_to"])) + cursor = start + for interval_start, interval_end in sorted(intervals): + if interval_end <= cursor: + continue + if interval_start > cursor: + raise ValueError(f"{field} has an evidence gap at {cursor.isoformat()}") + cursor = interval_end + if cursor >= end: + return + raise ValueError(f"{field} does not cover effective_to") + + +def validate_instrument_catalog( + catalog: pd.DataFrame, documents: dict[str, dict[str, Any]] +) -> pd.DataFrame: + """Validate a flat consumer catalog against versioned source evidence.""" + missing = BASE_COLUMNS - set(catalog.columns) + if missing or catalog.empty: + raise ValueError(f"Instrument catalog is empty or missing fields: {sorted(missing)}") + out = catalog.copy() + for column in BASE_COLUMNS: + if out[column].isna().any(): + raise ValueError(f"Instrument catalog contains null {column}") + if not out["catalog_schema_version"].eq(CATALOG_SCHEMA).all(): + raise ValueError("Unsupported instrument catalog schema") + if out["symbol"].duplicated().any(): + raise ValueError("Instrument catalog must have one flattened row per symbol") + for index, row in out.iterrows(): + symbol = _nonempty(row["symbol"], "symbol") + for field in ("catalog_source_version", "asset_class", "product_type", "venue"): + _nonempty(row[field], f"{symbol}.{field}") + if row["product_type"] == "etf" and row["asset_class"] != "etf": + raise ValueError(f"{symbol} ETF product_type requires ETF asset_class") + effective_from = _timestamp(row["effective_from"], f"{symbol}.effective_from") + effective_to = _timestamp(row["effective_to"], f"{symbol}.effective_to") + available_at = _timestamp(row["available_at"], f"{symbol}.available_at") + if not (available_at <= effective_from < effective_to): + raise ValueError(f"{symbol} has noncausal availability or invalid validity") + listing_date = pd.Timestamp(row["listing_date"]) + if pd.isna(listing_date) or listing_date.date() > effective_from.date(): + raise ValueError(f"{symbol} has an invalid listing date") + if row["listing_date_basis"] != "official-published-document": + raise ValueError(f"{symbol} listing date lacks official-document basis") + try: + price_scale = int(row["price_scale"]) + price_tick = Decimal(str(row["price_tick"])) + quantity_step = Decimal(str(row["quantity_step"])) + lot_size = int(row["lot_size"]) + except (InvalidOperation, TypeError, ValueError) as exc: + raise ValueError(f"{symbol} has invalid tick/lot fields") from exc + if ( + price_scale < 0 + or price_tick <= 0 + or quantity_step <= 0 + or lot_size <= 0 + or price_tick * (Decimal(10) ** price_scale) % 1 + ): + raise ValueError(f"{symbol} has invalid tick/lot fields") + listing_id = _nonempty(row["listing_evidence_id"], f"{symbol}.listing_evidence_id") + listing = documents.get(listing_id) + if listing is None or listing["kind"] != "listing": + raise ValueError(f"{symbol} listing evidence is missing or has the wrong kind") + if listing["published_at"] > available_at: + raise ValueError(f"{symbol} was marked available before its listing evidence") + rule_ids = _evidence_ids( + row["trading_rule_evidence_ids"], f"{symbol}.trading_rule_evidence_ids" + ) + limit_ids = _evidence_ids( + row["price_limit_evidence_ids"], f"{symbol}.price_limit_evidence_ids" + ) + first_rules = [ + documents[item] + for item in rule_ids + if documents[item]["effective_from"] <= effective_from < documents[item]["effective_to"] + ] + first_limits = [ + documents[item] + for item in limit_ids + if documents[item]["effective_from"] <= effective_from < documents[item]["effective_to"] + ] + if not first_rules or not first_limits: + raise ValueError(f"{symbol} lacks evidence effective at catalog start") + if any(document["published_at"] > available_at for document in first_rules + first_limits): + raise ValueError(f"{symbol} was marked available before its initial rules") + _validate_interval_coverage( + row, + ids=rule_ids, + documents=documents, + kinds={"trading_rule"}, + field=f"{symbol}.trading_rule_evidence_ids", + ) + _validate_interval_coverage( + row, + ids=limit_ids, + documents=documents, + kinds={"trading_rule", "price_limit"}, + field=f"{symbol}.price_limit_evidence_ids", + ) + out.loc[index, "symbol"] = symbol + return out.sort_values("symbol").reset_index(drop=True) + + +def import_instrument_master(source_root: Path, output: Path) -> dict[str, Any]: + """Publish an immutable, hash-bound source bundle from declared official documents.""" + source_root = source_root.resolve() + output = output.resolve() + if output.exists(): + raise FileExistsError(output) + declaration_path = source_root / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + documents = _document_index(declaration, source_root) + catalog_path = _relative_file(source_root, declaration.get("catalog_file"), "catalog_file") + expected_catalog_hash = _nonempty(declaration.get("catalog_sha256"), "catalog_sha256").lower() + if _digest(catalog_path) != expected_catalog_hash: + raise ValueError("Instrument catalog source hash mismatch") + catalog = validate_instrument_catalog(pd.read_csv(catalog_path, dtype=str), documents) + staging = output.with_name(f".{output.name}.tmp-{uuid4().hex}") + staging.mkdir(parents=True) + try: + normalized_catalog = staging / "catalog.csv" + catalog.to_csv(normalized_catalog, index=False) + copied_documents = [] + for document_id, document in sorted(documents.items()): + suffix = document["source_file"].suffix.lower() + destination = staging / "documents" / f"{document_id}{suffix}" + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(document["source_file"], destination) + copied_documents.append( + { + key: value + for key, value in document.items() + if key + not in { + "source_file", + "published_at", + "effective_from", + "effective_to", + "file", + } + } + | { + "file": destination.relative_to(staging).as_posix(), + "published_at": document["published_at"].isoformat(), + "effective_from": document["effective_from"].isoformat(), + "effective_to": document["effective_to"].isoformat(), + } + ) + manifest = { + "schema_version": BUNDLE_SCHEMA, + "captured_at": _timestamp(declaration["captured_at"], "captured_at").isoformat(), + "rights_note": _nonempty(declaration.get("rights_note"), "rights_note"), + "coverage": declaration.get("coverage", {}), + "source_declaration_sha256": _digest(declaration_path), + "source_catalog_sha256": expected_catalog_hash, + "catalog": { + "file": "catalog.csv", + "sha256": _digest(normalized_catalog), + "rows": len(catalog), + "symbols": sorted(catalog["symbol"].tolist()), + }, + "documents": copied_documents, + } + manifest["bundle_sha256"] = _digest_bytes(_canonical_bytes(manifest)) + (staging / "manifest.json").write_bytes(_canonical_bytes(manifest)) + os.replace(staging, output) + return manifest + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def load_instrument_master(root: Path) -> tuple[dict[str, Any], pd.DataFrame]: + """Load a published bundle after verifying every file and semantic time constraint.""" + root = root.resolve() + manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema_version") != BUNDLE_SCHEMA: + raise ValueError("Unsupported historical instrument-master bundle schema") + identity = {key: value for key, value in manifest.items() if key != "bundle_sha256"} + if manifest.get("bundle_sha256") != _digest_bytes(_canonical_bytes(identity)): + raise ValueError("Historical instrument-master manifest identity mismatch") + catalog_path = _relative_file(root, manifest["catalog"]["file"], "catalog.file") + if _digest(catalog_path) != manifest["catalog"]["sha256"]: + raise ValueError("Historical instrument-master catalog hash mismatch") + documents = {} + declaration = { + "schema_version": SOURCE_SCHEMA, + "captured_at": manifest["captured_at"], + "documents": manifest["documents"], + } + documents = _document_index(declaration, root) + catalog = validate_instrument_catalog(pd.read_csv(catalog_path, dtype=str), documents) + if ( + len(catalog) != manifest["catalog"]["rows"] + or sorted(catalog["symbol"].tolist()) != manifest["catalog"]["symbols"] + ): + raise ValueError("Historical instrument-master manifest differs from catalog") + return manifest, catalog + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m quant_data_kit.instrument_master", + description="Import or inspect evidence-bound historical instrument masters", + ) + commands = parser.add_subparsers(dest="command", required=True) + publish = commands.add_parser("import") + publish.add_argument("--source", type=Path, required=True) + publish.add_argument("--output", type=Path, required=True) + inspect = commands.add_parser("inspect") + inspect.add_argument("--root", type=Path, required=True) + args = parser.parse_args(argv) + if args.command == "import": + result = import_instrument_master(args.source, args.output) + else: + result, _ = load_instrument_master(args.root) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/src/quant_data_kit/research_dataset.py b/src/quant_data_kit/research_dataset.py index 13ba2ed..e1607d9 100644 --- a/src/quant_data_kit/research_dataset.py +++ b/src/quant_data_kit/research_dataset.py @@ -26,6 +26,7 @@ from quant_data_kit._version import __version__ from quant_data_kit.calendar import load_sse_trade_dates +from quant_data_kit.instrument_master import load_instrument_master from quant_data_kit.providers._symbols import normalize_symbol from quant_data_kit.providers.benchmark import fetch_hs300_benchmark from quant_data_kit.providers.corporate_actions import fetch_etf_corporate_actions @@ -542,7 +543,7 @@ def _instrument_catalog( "catalog_schema_version": "qdk.etf-instrument-catalog/v1", "catalog_source_version": source_version, "symbol": symbol, - "asset_class": "equity", + "asset_class": "etf", "product_type": "etf", "venue": _venue(symbol), "price_scale": 3, @@ -711,6 +712,7 @@ def _publish( validation: dict[str, Any], parent_snapshot_id: str | None, update_evidence: dict[str, Any], + instrument_master_root: Path | None = None, ) -> dict[str, Any]: root = root.resolve() root.mkdir(parents=True, exist_ok=True) @@ -718,14 +720,43 @@ def _publish( staging.mkdir() try: history = _history(frames["actions"], symbols, start, captured_at) - catalog_frame = _instrument_catalog( - symbols, - start=start, - end=end, - captured_at=captured_at, - raw=frames["raw"], - source=source, - ) + master_manifest = None + if instrument_master_root is None: + catalog_frame = _instrument_catalog( + symbols, + start=start, + end=end, + captured_at=captured_at, + raw=frames["raw"], + source=source, + ) + else: + master_manifest, catalog_frame = load_instrument_master(instrument_master_root) + if sorted(catalog_frame["symbol"].tolist()) != sorted(symbols): + raise ValueError("Historical instrument master must exactly match dataset symbols") + first_open = ( + start.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=9, minutes=25) + ).tz_convert("UTC") + last_close = (end.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=15)).tz_convert( + "UTC" + ) + effective_from = pd.to_datetime(catalog_frame["effective_from"], utc=True) + effective_to = pd.to_datetime(catalog_frame["effective_to"], utc=True) + available_at = pd.to_datetime(catalog_frame["available_at"], utc=True) + if not ( + (effective_from <= first_open).all() + and (effective_to > last_close).all() + and (available_at <= first_open).all() + ): + raise ValueError("Historical instrument master does not causally cover the dataset") + validation = { + **validation, + "instrument_master": { + "passed": True, + "bundle_sha256": master_manifest["bundle_sha256"], + "coverage": master_manifest.get("coverage", {}), + }, + } frames = {**frames, "catalog": catalog_frame, "history": history} files = {} providers = { @@ -738,7 +769,11 @@ def _publish( if source["mode"] == "live_public_api" else source["mode"] ), - "catalog": "qdk.etf-instrument-catalog/v1", + "catalog": ( + "qdk.etf-instrument-catalog/v1" + if master_manifest is None + else "qdk.historical-instrument-master/v1" + ), "history": "derived_from_actions_captured_view", } for name, filename in FILE_NAMES.items(): @@ -792,6 +827,18 @@ def _publish( "sha256": _sha256(history_source), }, } + if instrument_master_root is not None: + master_destination = staging / "instrument_master" + shutil.copytree(instrument_master_root.resolve(), master_destination) + for path in sorted(master_destination.rglob("*")): + if path.is_file(): + relative = path.relative_to(staging).as_posix() + evidence_files[ + f"instrument_master:{path.relative_to(master_destination).as_posix()}" + ] = { + "file": relative, + "sha256": _sha256(path), + } query = { "symbols": symbols, "start": start.date().isoformat(), @@ -815,6 +862,11 @@ def _publish( "consumer_contract": { "asm_decision_workflow_load_inputs": True, "history_availability": "captured-current; no inferred intraday announcement time", + "instrument_master_availability": ( + "captured-current; strict historical replay remains blocked" + if master_manifest is None + else "official-document evidence with versioned rule intervals" + ), }, } digest = hashlib.sha256(_canonical_bytes(manifest)).hexdigest() @@ -984,6 +1036,9 @@ def update_dataset( ) merged = _normalize_frames(merged, symbols=symbols, start=start_day, end=end_day) validation = validate_dataset_frames(merged, symbols=symbols, start=start_day, end=end_day) + master_root = None + if previous["files"]["catalog"].get("provider") == "qdk.historical-instrument-master/v1": + master_root = root.resolve() / "snapshots" / previous["snapshot_id"] / "instrument_master" return _publish( root, merged, @@ -1001,6 +1056,7 @@ def update_dataset( "previous_end": old_end.date().isoformat(), "revised_rows": revised, }, + instrument_master_root=master_root, ) @@ -1026,6 +1082,59 @@ def inspect_dataset(root: Path, snapshot_id: str | None = None) -> dict[str, Any } +def bind_instrument_master( + root: Path, + instrument_master_root: Path, + *, + captured_at: str | None = None, +) -> dict[str, Any]: + """Publish a child snapshot bound to verified historical master evidence. + + Market data and actions are copied from the selected immutable parent. The + operation never rewrites that parent and never manufactures daily status + rows from prices or an empty retrospective suspension query. + """ + root = root.resolve() + previous, frames = load_research_snapshot(root) + capture_time = _captured_at(captured_at) + previous_capture = _captured_at(previous["captured_at"]) + if capture_time < previous_capture: + raise ValueError("Instrument-master binding cannot predate the parent snapshot") + source = previous["source"] + source_frames = { + name: frames[name] for name in ("raw", "adjusted", "benchmark", "calendar", "actions") + } + validation = validate_dataset_frames( + source_frames, + symbols=list(previous["symbols"]), + start=_day(previous["requested_start"], "start"), + end=_day(previous["requested_end"], "end"), + ) + master_manifest, _ = load_instrument_master(instrument_master_root) + validation["instrument_master"] = { + "passed": True, + "bundle_sha256": master_manifest["bundle_sha256"], + "coverage": master_manifest.get("coverage", {}), + } + return _publish( + root, + source_frames, + symbols=list(previous["symbols"]), + start=_day(previous["requested_start"], "start"), + end=_day(previous["requested_end"], "end"), + captured_at=capture_time, + source=source, + validation=validation, + parent_snapshot_id=previous["snapshot_id"], + update_evidence={ + "operation": "bind_historical_instrument_master", + "market_data_refetched": False, + "instrument_master_bundle_sha256": master_manifest["bundle_sha256"], + }, + instrument_master_root=instrument_master_root, + ) + + def _source_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument("--source-dir", type=Path) parser.add_argument("--source-uri", default="") @@ -1058,6 +1167,10 @@ def _parser() -> argparse.ArgumentParser: inspect = commands.add_parser("inspect") inspect.add_argument("--root", type=Path, required=True) inspect.add_argument("--snapshot-id") + bind = commands.add_parser("bind-master") + bind.add_argument("--root", type=Path, required=True) + bind.add_argument("--instrument-master", type=Path, required=True) + bind.add_argument("--captured-at") return parser @@ -1089,6 +1202,13 @@ def main(argv: list[str] | None = None) -> int: license_note=args.license_note, ) result = inspect_dataset(args.root, manifest["snapshot_id"]) + elif args.command == "bind-master": + manifest = bind_instrument_master( + args.root, + args.instrument_master, + captured_at=args.captured_at, + ) + result = inspect_dataset(args.root, manifest["snapshot_id"]) else: result = inspect_dataset(args.root, args.snapshot_id) print(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False)) diff --git a/tests/test_instrument_master.py b/tests/test_instrument_master.py new file mode 100644 index 0000000..b8aa8aa --- /dev/null +++ b/tests/test_instrument_master.py @@ -0,0 +1,165 @@ +import hashlib +import json +from pathlib import Path + +import pandas as pd +import pytest + +from quant_data_kit.instrument_master import ( + CATALOG_SCHEMA, + import_instrument_master, + load_instrument_master, +) + + +def _hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) -> Path: + root.mkdir() + documents = root / "documents" + documents.mkdir() + payloads = { + "listing": b"official historical listing notice", + "rule-2023": b"official 2023 trading rule", + "rule-2026": b"official 2026 trading rule", + } + for name, payload in payloads.items(): + (documents / f"{name}.txt").write_bytes(payload) + catalog = pd.DataFrame( + [ + { + "catalog_schema_version": CATALOG_SCHEMA, + "catalog_source_version": "official-test-v1", + "symbol": "510300", + "asset_class": "etf", + "product_type": "etf", + "venue": "SSE", + "price_scale": "3", + "price_tick": "0.001", + "quantity_step": "1", + "lot_size": "100", + "commission_rate": "0", + "stamp_duty_rate": "0", + "effective_from": "2025-01-02T00:00:00Z", + "effective_to": "2026-09-25T00:00:00Z", + "available_at": available_at, + "listing_date": "2012-05-28", + "listing_date_basis": "official-published-document", + "listing_evidence_id": "listing", + "trading_rule_evidence_ids": "rule-2023;rule-2026", + "price_limit_evidence_ids": "rule-2023;rule-2026", + "price_limit_rate": "0.10", + } + ] + ) + catalog_path = root / "catalog.csv" + catalog.to_csv(catalog_path, index=False) + declaration = { + "schema_version": "qdk.historical-instrument-master-source/v1", + "captured_at": "2026-09-26T00:00:00Z", + "rights_note": "test fixture", + "coverage": { + "instrument_master": "field-evidence-complete", + "daily_trading_status": "unavailable-no-positive-state-inference", + }, + "catalog_file": "catalog.csv", + "catalog_sha256": _hash(catalog_path), + "documents": [ + { + "document_id": "listing", + "kind": "listing", + "publisher": "official exchange", + "title": "listing notice", + "source_uri": "https://exchange.example/listing", + "published_at": "2012-05-23T08:00:00Z", + "effective_from": "2012-05-28T00:00:00Z", + "effective_to": "2100-01-01T00:00:00Z", + "file": "documents/listing.txt", + "sha256": _hash(documents / "listing.txt"), + }, + { + "document_id": "rule-2023", + "kind": "trading_rule", + "publisher": "official exchange", + "title": "2023 rule", + "source_uri": "https://exchange.example/rule-2023", + "published_at": "2023-02-17T08:00:00Z", + "effective_from": "2023-04-10T00:00:00Z", + "effective_to": "2026-07-06T00:00:00Z", + "file": "documents/rule-2023.txt", + "sha256": _hash(documents / "rule-2023.txt"), + }, + { + "document_id": "rule-2026", + "kind": "trading_rule", + "publisher": "official exchange", + "title": "2026 rule", + "source_uri": "https://exchange.example/rule-2026", + "published_at": ("2026-07-07T08:00:00Z" if late_rule else "2026-04-24T08:00:00Z"), + "effective_from": "2026-07-06T00:00:00Z", + "effective_to": "2100-01-01T00:00:00Z", + "file": "documents/rule-2026.txt", + "sha256": _hash(documents / "rule-2026.txt"), + }, + ], + } + (root / "declaration.json").write_text( + json.dumps(declaration, ensure_ascii=False), encoding="utf-8" + ) + return root + + +def test_import_verifies_source_identity_hashes_and_rule_intervals(tmp_path): + source = _source(tmp_path / "source") + bundle = tmp_path / "bundle" + manifest = import_instrument_master(source, bundle) + loaded, catalog = load_instrument_master(bundle) + assert loaded == manifest + assert catalog.loc[0, "symbol"] == "510300" + assert loaded["coverage"]["daily_trading_status"].startswith("unavailable") + + document = bundle / loaded["documents"][0]["file"] + document.write_bytes(b"tampered") + with pytest.raises(ValueError, match="hash mismatch"): + load_instrument_master(bundle) + + +def test_manifest_source_identity_change_is_rejected(tmp_path): + bundle = tmp_path / "bundle" + import_instrument_master(_source(tmp_path / "source"), bundle) + manifest_path = bundle / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["documents"][0]["source_uri"] = "https://different.example/listing" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="manifest identity"): + load_instrument_master(bundle) + + +def test_catalog_rejects_future_availability_and_late_rule_publication(tmp_path): + with pytest.raises(ValueError, match="before its initial rules"): + import_instrument_master( + _source(tmp_path / "early", available_at="2023-02-16T08:00:00Z"), + tmp_path / "early-bundle", + ) + with pytest.raises(ValueError, match="not public before it became effective"): + import_instrument_master( + _source(tmp_path / "late", late_rule=True), + tmp_path / "late-bundle", + ) + + +def test_etf_catalog_rejects_equity_asset_class(tmp_path): + source = _source(tmp_path / "wrong-class") + catalog_path = source / "catalog.csv" + catalog = pd.read_csv(catalog_path, dtype=str) + catalog.loc[0, "asset_class"] = "equity" + catalog.to_csv(catalog_path, index=False) + declaration_path = source / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + declaration["catalog_sha256"] = _hash(catalog_path) + declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + + with pytest.raises(ValueError, match="ETF product_type requires ETF asset_class"): + import_instrument_master(source, tmp_path / "wrong-class-bundle") diff --git a/tests/test_research_dataset.py b/tests/test_research_dataset.py index a6d0b14..0d68bff 100644 --- a/tests/test_research_dataset.py +++ b/tests/test_research_dataset.py @@ -4,14 +4,17 @@ import pandas as pd import pytest +from quant_data_kit.instrument_master import import_instrument_master from quant_data_kit.providers.corporate_actions import normalize_etf_actions from quant_data_kit.research_coverage import load_history from quant_data_kit.research_dataset import ( + bind_instrument_master, build_dataset, inspect_dataset, load_research_snapshot, update_dataset, ) +from tests.test_instrument_master import _source as _master_source def _prices(dates, closes, adjustment): @@ -75,7 +78,15 @@ def _source(root: Path, dates, raw_closes, adjusted_closes, *, actions=True): pd.DataFrame( { "date": pd.DatetimeIndex( - ["2026-01-15", "2026-01-16", "2026-01-19", "2026-01-20", "2026-01-21", "2026-01-22"] + [ + "2026-01-15", + "2026-01-16", + "2026-01-19", + "2026-01-20", + "2026-01-21", + "2026-01-22", + "2026-01-23", + ] ) } ).to_parquet(root / "calendar.parquet", index=False) @@ -120,6 +131,7 @@ def test_local_build_is_immutable_asm_compatible_and_records_deferred_cash(tmp_p assert len(frames["history"]) == 4 assert frames["history"]["available_at"].nunique() == 1 assert frames["catalog"].loc[0, "product_type"] == "etf" + assert frames["catalog"].loc[0, "asset_class"] == "etf" assert frames["catalog"].loc[0, "price_tick"] == pytest.approx(0.001) history_manifest, compatible_history = load_history(snapshot / "history") assert history_manifest["schema_version"] == "qdk.research-history/v1" @@ -127,6 +139,46 @@ def test_local_build_is_immutable_asm_compatible_and_records_deferred_cash(tmp_p assert inspect_dataset(root)["validation"]["passed"] +def test_bind_historical_master_publishes_child_without_refetch(tmp_path): + source = tmp_path / "source" + dates = pd.DatetimeIndex(["2026-01-15", "2026-01-16", "2026-01-19", "2026-01-20", "2026-01-21"]) + _source(source, dates, [10, 10, 9, 9.1, 9.2], [9, 9, 9, 9.1, 9.2]) + root = tmp_path / "dataset" + parent = _build(root, source) + master = tmp_path / "master" + import_instrument_master(_master_source(tmp_path / "master-source"), master) + + child = bind_instrument_master( + root, + master, + captured_at="2026-09-26T00:00:00Z", + ) + assert child["parent_snapshot_id"] == parent["snapshot_id"] + assert child["update_evidence"]["market_data_refetched"] is False + assert child["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v1" + assert child["consumer_contract"]["instrument_master_availability"].startswith("official") + _, frames = load_research_snapshot(root, child["snapshot_id"]) + assert frames["catalog"].loc[0, "available_at"] == "2023-02-17T08:00:00Z" + + refresh = tmp_path / "refresh" + refresh_dates = pd.DatetimeIndex( + ["2026-01-15", "2026-01-16", "2026-01-19", "2026-01-20", "2026-01-21", "2026-01-22"] + ) + _source(refresh, refresh_dates, [10, 10, 9, 9.1, 9.2, 9.3], [9, 9, 9, 9.1, 9.2, 9.3]) + updated = update_dataset( + root, + end="2026-01-22", + captured_at="2026-09-27T00:00:00Z", + source_dir=refresh, + source_uri="file://declared-refresh", + source_version="vendor-export-8", + license_note="test fixture declaration", + ) + assert updated["parent_snapshot_id"] == child["snapshot_id"] + assert updated["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v1" + assert updated["validation"]["instrument_master"]["passed"] is True + + def test_incremental_update_keeps_parent_and_records_overlap_revision(tmp_path): initial = tmp_path / "initial" initial_dates = pd.DatetimeIndex(["2026-01-15", "2026-01-16", "2026-01-19"]) From db5e37d2aae43dc896a5d4b8ed43fad7840d770f Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 26 Sep 2026 17:02:14 +0800 Subject: [PATCH 2/6] Enforce PIT instrument master evidence contracts --- src/quant_data_kit/instrument_master.py | 732 +++++++++++++++++++----- src/quant_data_kit/research_dataset.py | 101 ++-- tests/test_instrument_master.py | 297 ++++++++-- tests/test_research_dataset.py | 16 +- 4 files changed, 903 insertions(+), 243 deletions(-) diff --git a/src/quant_data_kit/instrument_master.py b/src/quant_data_kit/instrument_master.py index fca207b..2b30e57 100644 --- a/src/quant_data_kit/instrument_master.py +++ b/src/quant_data_kit/instrument_master.py @@ -1,8 +1,11 @@ -"""Evidence-bound historical instrument-master imports. - -The bundle proves only the fields and intervals declared in its catalog. It -does not turn a current security list, an empty suspension query, or observed -volume into historical tradability. +"""Evidence-bound, point-in-time historical instrument-master imports. + +Catalog rows are rule versions rather than flattened symbol records. Every +consumer field is bound to a human-extracted typed assertion with a locator and +short excerpt on a hash-verified official attachment, or is explicitly +declared unavailable. Validation proves the attachment and declared assertion +have not changed; it does not claim to parse or semantically certify the source +text. Current observations end at the capture time. """ from __future__ import annotations @@ -20,11 +23,12 @@ import pandas as pd -SOURCE_SCHEMA = "qdk.historical-instrument-master-source/v1" -BUNDLE_SCHEMA = "qdk.historical-instrument-master/v1" -CATALOG_SCHEMA = "qdk.etf-instrument-catalog/v2" +SOURCE_SCHEMA = "qdk.historical-instrument-master-source/v2" +BUNDLE_SCHEMA = "qdk.historical-instrument-master/v2" +CATALOG_SCHEMA = "qdk.etf-instrument-catalog/v3" +CONSUMER_CATALOG_SCHEMA = "qdk.etf-instrument-catalog-view/v1" -BASE_COLUMNS = { +CATALOG_COLUMNS = ( "catalog_schema_version", "catalog_source_version", "symbol", @@ -37,22 +41,88 @@ "lot_size", "commission_rate", "stamp_duty_rate", + "fee_fields_scope", + "fee_evidence_id", "effective_from", "effective_to", "available_at", "listing_date", "listing_date_basis", "listing_evidence_id", - "trading_rule_evidence_ids", - "price_limit_evidence_ids", -} + "trading_rule_evidence_id", + "price_limit_evidence_id", + "price_limit_rate", + "master_scope", +) +BASE_COLUMNS = set(CATALOG_COLUMNS) DOCUMENT_KINDS = { "calendar", + "fee_schedule", "listing", "price_limit", "status_observation", "trading_rule", } +DOCUMENT_COLUMNS = { + "document_id", + "kind", + "publisher", + "title", + "source_uri", + "published_at", + "available_at", + "effective_from", + "effective_to", + "validity_basis", + "availability_basis", + "file", + "sha256", + "assertions", +} +ASSERTION_FIELDS = { + "asset_class", + "product_type", + "venue", + "listing_date", + "price_scale", + "price_tick", + "quantity_step", + "lot_size", + "commission_rate", + "stamp_duty_rate", + "price_limit_rate", +} +LISTING_FIELDS = ("asset_class", "product_type", "venue", "listing_date") +TRADING_RULE_FIELDS = ("venue", "price_scale", "price_tick", "quantity_step", "lot_size") +PRICE_LIMIT_FIELDS = ("price_limit_rate",) +FEE_FIELDS = ("commission_rate", "stamp_duty_rate") +UNAVAILABLE_FEE_SCOPE = "unavailable-requires-consumer-strategy-cost-model" +CERTIFIED_FEE_SCOPE = "official-document-rates" +MASTER_SCOPE = "retrospective-fixed-pool" +CURRENT_VALIDITY = "observed-through-capture-refresh-required" +HISTORICAL_VALIDITIES = {"superseded-by-next-version", "bounded-event"} +ASSERTION_METHODS = {"direct", "derived"} +SOURCE_COLUMNS = { + "schema_version", + "captured_at", + "rights_note", + "coverage", + "catalog_file", + "catalog_sha256", + "documents", +} +BUNDLE_COLUMNS = { + "schema_version", + "captured_at", + "refresh_required_after", + "rights_note", + "coverage", + "semantic_validation", + "source", + "catalog", + "documents", + "bundle_sha256", +} def _canonical_bytes(value: object) -> bytes: @@ -94,11 +164,53 @@ def _relative_file(root: Path, value: object, field: str) -> Path: return resolved -def _evidence_ids(value: object, field: str) -> list[str]: - ids = [item.strip() for item in _nonempty(value, field).split(";") if item.strip()] - if not ids or len(ids) != len(set(ids)): - raise ValueError(f"{field} must contain unique semicolon-separated IDs") - return ids +def _validate_source_schema(declaration: dict[str, Any]) -> None: + if set(declaration) != SOURCE_COLUMNS: + missing = sorted(SOURCE_COLUMNS - set(declaration)) + extra = sorted(set(declaration) - SOURCE_COLUMNS) + raise ValueError(f"Source declaration schema mismatch: missing={missing}, extra={extra}") + coverage = declaration["coverage"] + if not isinstance(coverage, dict) or not coverage: + raise ValueError("Source declaration coverage must be a nonempty object") + for field, value in coverage.items(): + _nonempty(field, "coverage field") + _nonempty(value, f"coverage.{field}") + + +def _assertion_index(raw: object, document_id: str) -> dict[str, dict[str, dict[str, str]]]: + if not isinstance(raw, list): + raise TypeError(f"{document_id}.assertions must be a list") + indexed: dict[str, dict[str, dict[str, str]]] = {} + for position, assertion in enumerate(raw): + if not isinstance(assertion, dict) or set(assertion) != {"symbol", "fields"}: + raise ValueError(f"{document_id}.assertions[{position}] has an invalid schema") + symbol = _nonempty(assertion["symbol"], f"{document_id}.assertions[{position}].symbol") + fields = assertion["fields"] + if not isinstance(fields, dict) or not fields or not set(fields) <= ASSERTION_FIELDS: + raise ValueError(f"{document_id}.assertions[{position}].fields is invalid") + normalized = {} + for field, claim in fields.items(): + if not isinstance(claim, dict) or set(claim) != { + "value", + "locator", + "excerpt", + "method", + }: + raise ValueError(f"{document_id}.{symbol}.{field} claim schema is invalid") + method = _nonempty(claim["method"], f"{document_id}.{symbol}.{field}.method") + if method not in ASSERTION_METHODS: + raise ValueError(f"{document_id}.{symbol}.{field}.method is unsupported") + normalized[field] = { + "value": _nonempty(claim["value"], f"{document_id}.{symbol}.{field}.value"), + "locator": _nonempty(claim["locator"], f"{document_id}.{symbol}.{field}.locator"), + "excerpt": _nonempty(claim["excerpt"], f"{document_id}.{symbol}.{field}.excerpt"), + "method": method, + } + duplicate = set(indexed.get(symbol, {})) & set(normalized) + if duplicate: + raise ValueError(f"{document_id} repeats assertions for {symbol}: {sorted(duplicate)}") + indexed.setdefault(symbol, {}).update(normalized) + return indexed def _document_index(declaration: dict[str, Any], root: Path) -> dict[str, dict[str, Any]]: @@ -110,193 +222,470 @@ def _document_index(declaration: dict[str, Any], root: Path) -> dict[str, dict[s raise ValueError("Historical instrument master requires evidence documents") indexed: dict[str, dict[str, Any]] = {} for raw in documents: - if not isinstance(raw, dict): - raise TypeError("Evidence document entries must be objects") - document_id = _nonempty(raw.get("document_id"), "document_id") + if not isinstance(raw, dict) or set(raw) != DOCUMENT_COLUMNS: + extra = sorted(set(raw) - DOCUMENT_COLUMNS) if isinstance(raw, dict) else [] + missing = sorted(DOCUMENT_COLUMNS - set(raw)) if isinstance(raw, dict) else [] + raise ValueError(f"Evidence document schema mismatch: missing={missing}, extra={extra}") + document_id = _nonempty(raw["document_id"], "document_id") if document_id in indexed: raise ValueError(f"Duplicate evidence document ID: {document_id}") - kind = _nonempty(raw.get("kind"), f"{document_id}.kind") + kind = _nonempty(raw["kind"], f"{document_id}.kind") if kind not in DOCUMENT_KINDS: raise ValueError(f"Unsupported evidence document kind: {kind}") - publisher = _nonempty(raw.get("publisher"), f"{document_id}.publisher") - source_uri = _nonempty(raw.get("source_uri"), f"{document_id}.source_uri") + publisher = _nonempty(raw["publisher"], f"{document_id}.publisher") + title = _nonempty(raw["title"], f"{document_id}.title") + source_uri = _nonempty(raw["source_uri"], f"{document_id}.source_uri") if not source_uri.startswith("https://"): raise ValueError(f"{document_id}.source_uri must use HTTPS") - published_at = _timestamp(raw.get("published_at"), f"{document_id}.published_at") - effective_from = _timestamp(raw.get("effective_from"), f"{document_id}.effective_from") - effective_to = _timestamp(raw.get("effective_to"), f"{document_id}.effective_to") - if published_at > captured_at: - raise ValueError(f"Evidence {document_id} was published after capture") - if kind in {"trading_rule", "price_limit"} and published_at > effective_from: - raise ValueError(f"Evidence {document_id} was not public before it became effective") - if effective_from >= effective_to: - raise ValueError(f"Evidence {document_id} has an invalid effective interval") - source_file = _relative_file(root, raw.get("file"), f"{document_id}.file") - expected_hash = _nonempty(raw.get("sha256"), f"{document_id}.sha256").lower() + published_at = _timestamp(raw["published_at"], f"{document_id}.published_at") + available_at = _timestamp(raw["available_at"], f"{document_id}.available_at") + effective_from = _timestamp(raw["effective_from"], f"{document_id}.effective_from") + effective_to = _timestamp(raw["effective_to"], f"{document_id}.effective_to") + validity_basis = _nonempty(raw["validity_basis"], f"{document_id}.validity_basis") + availability_basis = _nonempty( + raw["availability_basis"], f"{document_id}.availability_basis" + ) + if not published_at <= available_at <= captured_at: + raise ValueError(f"Evidence {document_id} has noncausal publication availability") + if not effective_from < effective_to <= captured_at: + raise ValueError(f"Evidence {document_id} extends beyond its observed capture horizon") + if kind in {"trading_rule", "price_limit", "fee_schedule"} and ( + available_at > effective_from + ): + raise ValueError(f"Evidence {document_id} was not available before it became effective") + if effective_to == captured_at: + if validity_basis != CURRENT_VALIDITY: + raise ValueError(f"Evidence {document_id} must declare its refresh boundary") + elif validity_basis not in HISTORICAL_VALIDITIES: + raise ValueError(f"Evidence {document_id} has an unsupported historical validity basis") + source_file = _relative_file(root, raw["file"], f"{document_id}.file") + expected_hash = _nonempty(raw["sha256"], f"{document_id}.sha256").lower() if len(expected_hash) != 64 or any(c not in "0123456789abcdef" for c in expected_hash): raise ValueError(f"Evidence {document_id} has an invalid SHA-256") if _digest(source_file) != expected_hash: raise ValueError(f"Evidence {document_id} hash mismatch") + assertions = _assertion_index(raw["assertions"], document_id) + if kind in {"listing", "trading_rule", "price_limit", "fee_schedule"} and not assertions: + raise ValueError(f"Evidence {document_id} requires typed field assertions") indexed[document_id] = { **raw, "document_id": document_id, "kind": kind, "publisher": publisher, + "title": title, "source_uri": source_uri, "published_at": published_at, + "available_at": available_at, "effective_from": effective_from, "effective_to": effective_to, + "validity_basis": validity_basis, + "availability_basis": availability_basis, "source_file": source_file, "sha256": expected_hash, + "assertion_index": assertions, } return indexed -def _validate_interval_coverage( - row: pd.Series, - *, - ids: list[str], +def _decimal(value: object, field: str) -> Decimal: + try: + result = Decimal(str(value)) + except (InvalidOperation, TypeError, ValueError) as exc: + raise ValueError(f"{field} must be a finite decimal") from exc + if not result.is_finite(): + raise ValueError(f"{field} must be a finite decimal") + return result + + +def _equal_field(field: str, left: object, right: object) -> bool: + if field in { + "price_scale", + "price_tick", + "quantity_step", + "lot_size", + "commission_rate", + "stamp_duty_rate", + "price_limit_rate", + }: + return _decimal(left, field) == _decimal(right, field) + if field == "listing_date": + return pd.Timestamp(left).date() == pd.Timestamp(right).date() + return str(left) == str(right) + + +def _document( documents: dict[str, dict[str, Any]], - kinds: set[str], + document_id: object, + *, + symbol: str, field: str, + kinds: set[str], +) -> dict[str, Any]: + evidence_id = _nonempty(document_id, f"{symbol}.{field}") + document = documents.get(evidence_id) + if document is None or document["kind"] not in kinds: + raise ValueError(f"{symbol}.{field} references missing or wrong-kind evidence") + return document + + +def _assert_fields( + row: pd.Series, + document: dict[str, Any], + fields: tuple[str, ...], ) -> None: - start = _timestamp(row["effective_from"], "effective_from") - end = _timestamp(row["effective_to"], "effective_to") - intervals = [] - for document_id in ids: - document = documents.get(document_id) - if document is None: - raise ValueError(f"Unknown evidence ID {document_id} in {field}") - if document["kind"] not in kinds: - raise ValueError(f"Evidence {document_id} has the wrong kind for {field}") - intervals.append((document["effective_from"], document["effective_to"])) - cursor = start - for interval_start, interval_end in sorted(intervals): - if interval_end <= cursor: - continue - if interval_start > cursor: - raise ValueError(f"{field} has an evidence gap at {cursor.isoformat()}") - cursor = interval_end - if cursor >= end: - return - raise ValueError(f"{field} does not cover effective_to") + symbol = str(row["symbol"]) + assertions = document["assertion_index"].get(symbol, {}) + missing = set(fields) - set(assertions) + if missing: + raise ValueError( + f"Evidence {document['document_id']} lacks {symbol} assertions: {sorted(missing)}" + ) + for field in fields: + if not _equal_field(field, row[field], assertions[field]["value"]): + raise ValueError( + f"{symbol}.{field} differs from evidence {document['document_id']} assertion" + ) + + +def _validate_row( + row: pd.Series, + documents: dict[str, dict[str, Any]], + captured_at: pd.Timestamp, +) -> None: + symbol = _nonempty(row["symbol"], "symbol") + if row["asset_class"] != "etf" or row["product_type"] != "etf": + raise ValueError(f"{symbol} must be explicitly evidenced as an ETF") + if row["venue"] not in {"SSE", "SZSE"}: + raise ValueError(f"{symbol} has an unsupported ETF venue") + if row["master_scope"] != MASTER_SCOPE: + raise ValueError(f"{symbol} has an unsupported master_scope") + effective_from = _timestamp(row["effective_from"], f"{symbol}.effective_from") + effective_to = _timestamp(row["effective_to"], f"{symbol}.effective_to") + available_at = _timestamp(row["available_at"], f"{symbol}.available_at") + if not available_at <= effective_from < effective_to <= captured_at: + raise ValueError(f"{symbol} has noncausal availability or unobserved future validity") + listing_date = pd.Timestamp(row["listing_date"]) + if pd.isna(listing_date) or listing_date.tzinfo is not None: + raise ValueError(f"{symbol}.listing_date must be a calendar date") + if listing_date.date() > effective_from.date(): + raise ValueError(f"{symbol} has an invalid listing date") + if row["listing_date_basis"] != "official-published-document": + raise ValueError(f"{symbol} listing date lacks official-document basis") + + price_scale = int(_decimal(row["price_scale"], f"{symbol}.price_scale")) + price_tick = _decimal(row["price_tick"], f"{symbol}.price_tick") + quantity_step = _decimal(row["quantity_step"], f"{symbol}.quantity_step") + lot_size = int(_decimal(row["lot_size"], f"{symbol}.lot_size")) + price_limit_rate = _decimal(row["price_limit_rate"], f"{symbol}.price_limit_rate") + if ( + price_scale < 0 + or price_tick <= 0 + or quantity_step <= 0 + or lot_size <= 0 + or not 0 < price_limit_rate < 1 + or price_tick * (Decimal(10) ** price_scale) % 1 + ): + raise ValueError(f"{symbol} has invalid tick, lot or price-limit fields") + + listing = _document( + documents, + row["listing_evidence_id"], + symbol=symbol, + field="listing_evidence_id", + kinds={"listing"}, + ) + trading = _document( + documents, + row["trading_rule_evidence_id"], + symbol=symbol, + field="trading_rule_evidence_id", + kinds={"trading_rule"}, + ) + price_limit = _document( + documents, + row["price_limit_evidence_id"], + symbol=symbol, + field="price_limit_evidence_id", + kinds={"price_limit", "trading_rule"}, + ) + for field, document in ( + ("listing_evidence_id", listing), + ("trading_rule_evidence_id", trading), + ("price_limit_evidence_id", price_limit), + ): + if ( + not document["effective_from"] + <= effective_from + < effective_to + <= document["effective_to"] + ): + raise ValueError(f"{symbol}.{field} does not cover its rule version") + _assert_fields(row, listing, LISTING_FIELDS) + _assert_fields(row, trading, TRADING_RULE_FIELDS) + _assert_fields(row, price_limit, PRICE_LIMIT_FIELDS) + + evidence_available = max( + listing["available_at"], trading["available_at"], price_limit["available_at"] + ) + scope = row["fee_fields_scope"] + if scope == UNAVAILABLE_FEE_SCOPE: + if row["commission_rate"] or row["stamp_duty_rate"] or row["fee_evidence_id"]: + raise ValueError(f"{symbol} unavailable fee fields must remain empty") + elif scope == CERTIFIED_FEE_SCOPE: + fee = _document( + documents, + row["fee_evidence_id"], + symbol=symbol, + field="fee_evidence_id", + kinds={"fee_schedule"}, + ) + if not fee["effective_from"] <= effective_from < effective_to <= fee["effective_to"]: + raise ValueError(f"{symbol}.fee_evidence_id does not cover its rule version") + _assert_fields(row, fee, FEE_FIELDS) + for field in FEE_FIELDS: + if not 0 <= _decimal(row[field], f"{symbol}.{field}") < 1: + raise ValueError(f"{symbol}.{field} is outside [0, 1)") + evidence_available = max(evidence_available, fee["available_at"]) + else: + raise ValueError(f"{symbol} has an unsupported fee_fields_scope") + if available_at != evidence_available: + raise ValueError(f"{symbol}.available_at must equal its latest referenced evidence") def validate_instrument_catalog( - catalog: pd.DataFrame, documents: dict[str, dict[str, Any]] + catalog: pd.DataFrame, + documents: dict[str, dict[str, Any]], + *, + captured_at: pd.Timestamp, ) -> pd.DataFrame: - """Validate a flat consumer catalog against versioned source evidence.""" - missing = BASE_COLUMNS - set(catalog.columns) - if missing or catalog.empty: - raise ValueError(f"Instrument catalog is empty or missing fields: {sorted(missing)}") - out = catalog.copy() - for column in BASE_COLUMNS: - if out[column].isna().any(): - raise ValueError(f"Instrument catalog contains null {column}") + """Validate a closed, versioned catalog against typed document assertions.""" + if catalog.empty or set(catalog.columns) != BASE_COLUMNS: + missing = sorted(BASE_COLUMNS - set(catalog.columns)) + extra = sorted(set(catalog.columns) - BASE_COLUMNS) + raise ValueError(f"Instrument catalog schema mismatch: missing={missing}, extra={extra}") + out = catalog.loc[:, CATALOG_COLUMNS].copy().fillna("") + for column in BASE_COLUMNS - {"commission_rate", "stamp_duty_rate", "fee_evidence_id"}: + if out[column].astype(str).str.strip().eq("").any(): + raise ValueError(f"Instrument catalog contains empty {column}") if not out["catalog_schema_version"].eq(CATALOG_SCHEMA).all(): raise ValueError("Unsupported instrument catalog schema") - if out["symbol"].duplicated().any(): - raise ValueError("Instrument catalog must have one flattened row per symbol") for index, row in out.iterrows(): - symbol = _nonempty(row["symbol"], "symbol") - for field in ("catalog_source_version", "asset_class", "product_type", "venue"): - _nonempty(row[field], f"{symbol}.{field}") - if row["product_type"] == "etf" and row["asset_class"] != "etf": - raise ValueError(f"{symbol} ETF product_type requires ETF asset_class") - effective_from = _timestamp(row["effective_from"], f"{symbol}.effective_from") - effective_to = _timestamp(row["effective_to"], f"{symbol}.effective_to") - available_at = _timestamp(row["available_at"], f"{symbol}.available_at") - if not (available_at <= effective_from < effective_to): - raise ValueError(f"{symbol} has noncausal availability or invalid validity") - listing_date = pd.Timestamp(row["listing_date"]) - if pd.isna(listing_date) or listing_date.date() > effective_from.date(): - raise ValueError(f"{symbol} has an invalid listing date") - if row["listing_date_basis"] != "official-published-document": - raise ValueError(f"{symbol} listing date lacks official-document basis") - try: - price_scale = int(row["price_scale"]) - price_tick = Decimal(str(row["price_tick"])) - quantity_step = Decimal(str(row["quantity_step"])) - lot_size = int(row["lot_size"]) - except (InvalidOperation, TypeError, ValueError) as exc: - raise ValueError(f"{symbol} has invalid tick/lot fields") from exc + _validate_row(row, documents, captured_at) + out.loc[index, "symbol"] = str(row["symbol"]).strip() + effective = pd.to_datetime(out["effective_from"], format="mixed", utc=True) + if pd.DataFrame({"symbol": out["symbol"], "effective_from": effective}).duplicated().any(): + raise ValueError("Instrument catalog repeats a symbol rule version") + out = out.assign(_effective_from=effective).sort_values(["symbol", "_effective_from"]) + for symbol, versions in out.groupby("symbol", sort=False): + static_fields = ( + "asset_class", + "product_type", + "venue", + "listing_date", + "listing_date_basis", + "listing_evidence_id", + "master_scope", + ) + if any(versions[field].nunique(dropna=False) != 1 for field in static_fields): + raise ValueError(f"{symbol} changes static instrument fields across rule versions") + starts = pd.to_datetime(versions["effective_from"], format="mixed", utc=True).tolist() + ends = pd.to_datetime(versions["effective_to"], format="mixed", utc=True).tolist() + for previous_end, next_start in zip(ends, starts[1:], strict=False): + if previous_end != next_start: + raise ValueError(f"{symbol} has a rule-version gap or overlap") + return out.drop(columns="_effective_from").reset_index(drop=True) + + +def resolve_instrument_catalog( + catalog: pd.DataFrame, + *, + symbols: list[str], + start: pd.Timestamp, + end: pd.Timestamp, +) -> pd.DataFrame: + """Create a flat consumer view only when all covered versions are execution-equivalent.""" + first_open = ( + start.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=9, minutes=25) + ).tz_convert("UTC") + last_close = (end.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=15)).tz_convert("UTC") + rows: list[dict[str, Any]] = [] + value_fields = ( + "asset_class", + "product_type", + "venue", + "price_scale", + "price_tick", + "quantity_step", + "lot_size", + "commission_rate", + "stamp_duty_rate", + "fee_fields_scope", + "master_scope", + "listing_date", + "listing_date_basis", + ) + for symbol in symbols: + versions = catalog[catalog["symbol"].eq(symbol)].copy() + if versions.empty: + raise ValueError(f"Historical instrument master is missing symbol {symbol}") + versions["_start"] = pd.to_datetime(versions["effective_from"], format="mixed", utc=True) + versions["_end"] = pd.to_datetime(versions["effective_to"], format="mixed", utc=True) + versions = versions[(versions["_end"] > first_open) & (versions["_start"] <= last_close)] + versions = versions.sort_values("_start") if ( - price_scale < 0 - or price_tick <= 0 - or quantity_step <= 0 - or lot_size <= 0 - or price_tick * (Decimal(10) ** price_scale) % 1 + versions.empty + or versions.iloc[0]["_start"] > first_open + or versions.iloc[-1]["_end"] <= last_close ): - raise ValueError(f"{symbol} has invalid tick/lot fields") - listing_id = _nonempty(row["listing_evidence_id"], f"{symbol}.listing_evidence_id") - listing = documents.get(listing_id) - if listing is None or listing["kind"] != "listing": - raise ValueError(f"{symbol} listing evidence is missing or has the wrong kind") - if listing["published_at"] > available_at: - raise ValueError(f"{symbol} was marked available before its listing evidence") - rule_ids = _evidence_ids( - row["trading_rule_evidence_ids"], f"{symbol}.trading_rule_evidence_ids" - ) - limit_ids = _evidence_ids( - row["price_limit_evidence_ids"], f"{symbol}.price_limit_evidence_ids" - ) - first_rules = [ - documents[item] - for item in rule_ids - if documents[item]["effective_from"] <= effective_from < documents[item]["effective_to"] - ] - first_limits = [ - documents[item] - for item in limit_ids - if documents[item]["effective_from"] <= effective_from < documents[item]["effective_to"] - ] - if not first_rules or not first_limits: - raise ValueError(f"{symbol} lacks evidence effective at catalog start") - if any(document["published_at"] > available_at for document in first_rules + first_limits): - raise ValueError(f"{symbol} was marked available before its initial rules") - _validate_interval_coverage( - row, - ids=rule_ids, - documents=documents, - kinds={"trading_rule"}, - field=f"{symbol}.trading_rule_evidence_ids", + raise ValueError( + f"Historical instrument master does not cover {symbol} dataset interval" + ) + for previous_end, next_start in zip( + versions["_end"].tolist(), versions["_start"].tolist()[1:], strict=False + ): + if previous_end != next_start: + raise ValueError(f"Historical instrument master has a {symbol} coverage gap") + changed = [field for field in value_fields if versions[field].nunique(dropna=False) != 1] + if changed: + raise ValueError( + f"Flat research consumer cannot represent {symbol} rule changes: {changed}" + ) + first = versions.iloc[0] + rows.append( + { + "catalog_schema_version": CONSUMER_CATALOG_SCHEMA, + "catalog_source_version": first["catalog_source_version"], + "symbol": symbol, + **{field: first[field] for field in value_fields}, + "effective_from": first["effective_from"], + "effective_to": versions.iloc[-1]["effective_to"], + "available_at": first["available_at"], + "rule_versions": ";".join( + f"{row.trading_rule_evidence_id}|{row.price_limit_evidence_id}" + for row in versions.itertuples(index=False) + ), + } ) - _validate_interval_coverage( - row, - ids=limit_ids, - documents=documents, - kinds={"trading_rule", "price_limit"}, - field=f"{symbol}.price_limit_evidence_ids", + return pd.DataFrame(rows).sort_values("symbol").reset_index(drop=True) + + +def instrument_master_prefix( + root: Path, + *, + cutoff: str | pd.Timestamp, +) -> dict[str, Any]: + """Return the canonical evidence state known at ``cutoff``. + + The same cutoff can compare two independently captured bundles. Extending + an unchanged current observation beyond the cutoff has no effect, while a + changed historical value, availability time, version boundary, evidence + identity, file hash or typed assertion remains visible. + """ + manifest, catalog = load_instrument_master(root) + cutoff_at = _timestamp(cutoff, "cutoff") + captured_at = _timestamp(manifest["captured_at"], "captured_at") + if cutoff_at > captured_at: + raise ValueError("Instrument-master prefix cutoff exceeds the verified capture") + + known = catalog[ + pd.to_datetime(catalog["available_at"], format="mixed", utc=True) <= cutoff_at + ].copy() + rows: list[dict[str, Any]] = [] + referenced_documents: set[str] = set() + for row in known.sort_values(["symbol", "effective_from"]).to_dict(orient="records"): + start = _timestamp(row["effective_from"], "effective_from") + end = _timestamp(row["effective_to"], "effective_to") + normalized = { + key: value + for key, value in row.items() + if key not in {"catalog_schema_version", "catalog_source_version", "effective_to"} + } + if start < cutoff_at: + normalized["effective_to"] = min(end, cutoff_at).isoformat() + normalized["knowledge_state"] = "effective-by-cutoff" + else: + normalized["effective_to"] = None + normalized["knowledge_state"] = "scheduled-after-cutoff" + rows.append(normalized) + referenced_documents.update( + evidence_id + for evidence_id in ( + row["listing_evidence_id"], + row["trading_rule_evidence_id"], + row["price_limit_evidence_id"], + row["fee_evidence_id"], + ) + if evidence_id ) - out.loc[index, "symbol"] = symbol - return out.sort_values("symbol").reset_index(drop=True) + + documents = [] + for document in sorted(manifest["documents"], key=lambda item: item["document_id"]): + if document["document_id"] not in referenced_documents: + continue + available_at = _timestamp(document["available_at"], "document.available_at") + if available_at > cutoff_at: + raise ValueError("Known catalog version references evidence unavailable at cutoff") + effective_from = _timestamp(document["effective_from"], "document.effective_from") + effective_to = _timestamp(document["effective_to"], "document.effective_to") + normalized = { + key: value + for key, value in document.items() + if key not in {"file", "effective_to", "validity_basis"} + } + if effective_from < cutoff_at: + normalized["effective_to"] = min(effective_to, cutoff_at).isoformat() + normalized["knowledge_state"] = "effective-by-cutoff" + else: + normalized["effective_to"] = None + normalized["knowledge_state"] = "scheduled-after-cutoff" + documents.append(normalized) + return { + "schema_version": "qdk.historical-instrument-master-prefix/v1", + "cutoff": cutoff_at.isoformat(), + "catalog_schema_version": CATALOG_SCHEMA, + "rows": rows, + "documents": documents, + } def import_instrument_master(source_root: Path, output: Path) -> dict[str, Any]: - """Publish an immutable, hash-bound source bundle from declared official documents.""" + """Publish an immutable, self-contained bundle from declared official documents.""" source_root = source_root.resolve() output = output.resolve() if output.exists(): raise FileExistsError(output) declaration_path = source_root / "declaration.json" declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + _validate_source_schema(declaration) documents = _document_index(declaration, source_root) + captured_at = _timestamp(declaration["captured_at"], "captured_at") catalog_path = _relative_file(source_root, declaration.get("catalog_file"), "catalog_file") expected_catalog_hash = _nonempty(declaration.get("catalog_sha256"), "catalog_sha256").lower() if _digest(catalog_path) != expected_catalog_hash: raise ValueError("Instrument catalog source hash mismatch") - catalog = validate_instrument_catalog(pd.read_csv(catalog_path, dtype=str), documents) + catalog = validate_instrument_catalog( + pd.read_csv(catalog_path, dtype=str, keep_default_na=False), + documents, + captured_at=captured_at, + ) staging = output.with_name(f".{output.name}.tmp-{uuid4().hex}") staging.mkdir(parents=True) try: normalized_catalog = staging / "catalog.csv" catalog.to_csv(normalized_catalog, index=False) + source_directory = staging / "source" + source_directory.mkdir() + shutil.copyfile(declaration_path, source_directory / "declaration.json") + shutil.copyfile(catalog_path, source_directory / "catalog.csv") copied_documents = [] for document_id, document in sorted(documents.items()): suffix = document["source_file"].suffix.lower() destination = staging / "documents" / f"{document_id}{suffix}" destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(document["source_file"], destination) + original_destination = source_directory / Path(document["file"]) + original_destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(document["source_file"], original_destination) copied_documents.append( { key: value @@ -304,7 +693,9 @@ def import_instrument_master(source_root: Path, output: Path) -> dict[str, Any]: if key not in { "source_file", + "assertion_index", "published_at", + "available_at", "effective_from", "effective_to", "file", @@ -313,22 +704,33 @@ def import_instrument_master(source_root: Path, output: Path) -> dict[str, Any]: | { "file": destination.relative_to(staging).as_posix(), "published_at": document["published_at"].isoformat(), + "available_at": document["available_at"].isoformat(), "effective_from": document["effective_from"].isoformat(), "effective_to": document["effective_to"].isoformat(), } ) manifest = { "schema_version": BUNDLE_SCHEMA, - "captured_at": _timestamp(declaration["captured_at"], "captured_at").isoformat(), + "captured_at": captured_at.isoformat(), + "refresh_required_after": captured_at.isoformat(), "rights_note": _nonempty(declaration.get("rights_note"), "rights_note"), "coverage": declaration.get("coverage", {}), - "source_declaration_sha256": _digest(declaration_path), - "source_catalog_sha256": expected_catalog_hash, + "semantic_validation": { + "assertion_origin": "human-extracted-with-locator-and-short-excerpt", + "attachment_integrity": "sha256-verified", + "attachment_text_semantically_verified_by_code": False, + }, + "source": { + "declaration_file": "source/declaration.json", + "declaration_sha256": _digest(source_directory / "declaration.json"), + "catalog_file": "source/catalog.csv", + "catalog_sha256": _digest(source_directory / "catalog.csv"), + }, "catalog": { "file": "catalog.csv", "sha256": _digest(normalized_catalog), "rows": len(catalog), - "symbols": sorted(catalog["symbol"].tolist()), + "symbols": sorted(catalog["symbol"].unique().tolist()), }, "documents": copied_documents, } @@ -342,28 +744,62 @@ def import_instrument_master(source_root: Path, output: Path) -> dict[str, Any]: def load_instrument_master(root: Path) -> tuple[dict[str, Any], pd.DataFrame]: - """Load a published bundle after verifying every file and semantic time constraint.""" + """Load a bundle after verifying its source, documents and typed assertions.""" root = root.resolve() manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + if set(manifest) != BUNDLE_COLUMNS: + missing = sorted(BUNDLE_COLUMNS - set(manifest)) + extra = sorted(set(manifest) - BUNDLE_COLUMNS) + raise ValueError( + f"Instrument-master bundle schema mismatch: missing={missing}, extra={extra}" + ) if manifest.get("schema_version") != BUNDLE_SCHEMA: raise ValueError("Unsupported historical instrument-master bundle schema") identity = {key: value for key, value in manifest.items() if key != "bundle_sha256"} if manifest.get("bundle_sha256") != _digest_bytes(_canonical_bytes(identity)): raise ValueError("Historical instrument-master manifest identity mismatch") + captured_at = _timestamp(manifest.get("captured_at"), "captured_at") + if _timestamp(manifest.get("refresh_required_after"), "refresh_required_after") != captured_at: + raise ValueError("Historical instrument-master refresh boundary mismatch") + for field in ("declaration", "catalog"): + path = _relative_file(root, manifest["source"][f"{field}_file"], f"source.{field}_file") + if _digest(path) != manifest["source"][f"{field}_sha256"]: + raise ValueError(f"Historical instrument-master source {field} hash mismatch") + source_declaration = json.loads( + (root / manifest["source"]["declaration_file"]).read_text(encoding="utf-8") + ) + _validate_source_schema(source_declaration) + if _timestamp(source_declaration.get("captured_at"), "source.captured_at") != captured_at: + raise ValueError("Historical instrument-master source capture changed") + if source_declaration.get("catalog_sha256") != manifest["source"]["catalog_sha256"]: + raise ValueError("Historical instrument-master source catalog identity changed") + source_documents = _document_index( + source_declaration, (root / manifest["source"]["declaration_file"]).parent + ) + source_catalog = validate_instrument_catalog( + pd.read_csv(root / manifest["source"]["catalog_file"], dtype=str, keep_default_na=False), + source_documents, + captured_at=captured_at, + ) catalog_path = _relative_file(root, manifest["catalog"]["file"], "catalog.file") if _digest(catalog_path) != manifest["catalog"]["sha256"]: raise ValueError("Historical instrument-master catalog hash mismatch") - documents = {} declaration = { "schema_version": SOURCE_SCHEMA, "captured_at": manifest["captured_at"], "documents": manifest["documents"], } documents = _document_index(declaration, root) - catalog = validate_instrument_catalog(pd.read_csv(catalog_path, dtype=str), documents) + catalog = validate_instrument_catalog( + pd.read_csv(catalog_path, dtype=str, keep_default_na=False), + documents, + captured_at=captured_at, + ) + if not source_catalog.equals(catalog): + raise ValueError("Historical instrument-master normalized catalog differs from source") if ( len(catalog) != manifest["catalog"]["rows"] - or sorted(catalog["symbol"].tolist()) != manifest["catalog"]["symbols"] + or sorted(catalog["symbol"].unique().tolist()) != manifest["catalog"]["symbols"] ): raise ValueError("Historical instrument-master manifest differs from catalog") return manifest, catalog diff --git a/src/quant_data_kit/research_dataset.py b/src/quant_data_kit/research_dataset.py index e1607d9..1261e8a 100644 --- a/src/quant_data_kit/research_dataset.py +++ b/src/quant_data_kit/research_dataset.py @@ -26,7 +26,13 @@ from quant_data_kit._version import __version__ from quant_data_kit.calendar import load_sse_trade_dates -from quant_data_kit.instrument_master import load_instrument_master +from quant_data_kit.instrument_master import ( + BUNDLE_SCHEMA as INSTRUMENT_MASTER_SCHEMA, +) +from quant_data_kit.instrument_master import ( + load_instrument_master, + resolve_instrument_catalog, +) from quant_data_kit.providers._symbols import normalize_symbol from quant_data_kit.providers.benchmark import fetch_hs300_benchmark from quant_data_kit.providers.corporate_actions import fetch_etf_corporate_actions @@ -713,13 +719,20 @@ def _publish( parent_snapshot_id: str | None, update_evidence: dict[str, Any], instrument_master_root: Path | None = None, + preserved_history_root: Path | None = None, ) -> dict[str, Any]: root = root.resolve() root.mkdir(parents=True, exist_ok=True) staging = root / f".snapshot.tmp-{uuid4().hex}" staging.mkdir() try: - history = _history(frames["actions"], symbols, start, captured_at) + if preserved_history_root is None: + history = _history(frames["actions"], symbols, start, captured_at) + else: + preserved_history_root = preserved_history_root.resolve() + history = validate_history( + pd.read_parquet(preserved_history_root / FILE_NAMES["history"]) + ) master_manifest = None if instrument_master_root is None: catalog_frame = _instrument_catalog( @@ -731,24 +744,17 @@ def _publish( source=source, ) else: - master_manifest, catalog_frame = load_instrument_master(instrument_master_root) - if sorted(catalog_frame["symbol"].tolist()) != sorted(symbols): + master_destination = staging / "instrument_master" + shutil.copytree(instrument_master_root.resolve(), master_destination) + master_manifest, versioned_catalog = load_instrument_master(master_destination) + if sorted(versioned_catalog["symbol"].unique().tolist()) != sorted(symbols): raise ValueError("Historical instrument master must exactly match dataset symbols") - first_open = ( - start.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=9, minutes=25) - ).tz_convert("UTC") - last_close = (end.tz_localize("Asia/Shanghai") + pd.Timedelta(hours=15)).tz_convert( - "UTC" + catalog_frame = resolve_instrument_catalog( + versioned_catalog, + symbols=symbols, + start=start, + end=end, ) - effective_from = pd.to_datetime(catalog_frame["effective_from"], utc=True) - effective_to = pd.to_datetime(catalog_frame["effective_to"], utc=True) - available_at = pd.to_datetime(catalog_frame["available_at"], utc=True) - if not ( - (effective_from <= first_open).all() - and (effective_to > last_close).all() - and (available_at <= first_open).all() - ): - raise ValueError("Historical instrument master does not causally cover the dataset") validation = { **validation, "instrument_master": { @@ -772,7 +778,7 @@ def _publish( "catalog": ( "qdk.etf-instrument-catalog/v1" if master_manifest is None - else "qdk.historical-instrument-master/v1" + else INSTRUMENT_MASTER_SCHEMA ), "history": "derived_from_actions_captured_view", } @@ -780,7 +786,9 @@ def _publish( path = staging / filename path.parent.mkdir(parents=True, exist_ok=True) file_format = path.suffix.lstrip(".") - if file_format == "parquet": + if name == "history" and preserved_history_root is not None: + shutil.copyfile(preserved_history_root / filename, path) + elif file_format == "parquet": frames[name].to_parquet(path, index=False) elif file_format == "csv": frames[name].to_csv(path, index=False) @@ -795,28 +803,34 @@ def _publish( } history_root = staging / "history" history_source = history_root / "source.json" - history_source.write_bytes( - _history_source_payload(frames["actions"], symbols=symbols, source=source) - ) - history_manifest = { - "schema_version": HISTORY_SCHEMA, - "provider": source.get("provider", source["mode"]), - "source_uri": source.get("source_uri", "akshare://sina-etf+eastmoney-fund-f10"), - "license_note": source.get( - "license_note", "Public web sources; rights remain with source providers." - ), - "scope": "captured-current-source-declaration", - "original": {"file": history_source.name, "sha256": _sha256(history_source)}, - "history": { - "file": Path(FILE_NAMES["history"]).name, - "sha256": files["history"]["sha256"], - }, - "rows": len(history), - "domains": sorted(history["domain"].unique()), - "symbols": sorted(history["symbol"].unique()), - } history_manifest_path = history_root / "manifest.json" - history_manifest_path.write_bytes(_canonical_bytes(history_manifest)) + if preserved_history_root is None: + history_source.write_bytes( + _history_source_payload(frames["actions"], symbols=symbols, source=source) + ) + history_manifest = { + "schema_version": HISTORY_SCHEMA, + "provider": source.get("provider", source["mode"]), + "source_uri": source.get("source_uri", "akshare://sina-etf+eastmoney-fund-f10"), + "license_note": source.get( + "license_note", "Public web sources; rights remain with source providers." + ), + "scope": "captured-current-source-declaration", + "original": {"file": history_source.name, "sha256": _sha256(history_source)}, + "history": { + "file": Path(FILE_NAMES["history"]).name, + "sha256": files["history"]["sha256"], + }, + "rows": len(history), + "domains": sorted(history["domain"].unique()), + "symbols": sorted(history["symbol"].unique()), + } + history_manifest_path.write_bytes(_canonical_bytes(history_manifest)) + else: + shutil.copyfile(preserved_history_root / "history" / "source.json", history_source) + shutil.copyfile( + preserved_history_root / "history" / "manifest.json", history_manifest_path + ) evidence_files = { "history_manifest": { "file": "history/manifest.json", @@ -828,8 +842,6 @@ def _publish( }, } if instrument_master_root is not None: - master_destination = staging / "instrument_master" - shutil.copytree(instrument_master_root.resolve(), master_destination) for path in sorted(master_destination.rglob("*")): if path.is_file(): relative = path.relative_to(staging).as_posix() @@ -1037,7 +1049,7 @@ def update_dataset( merged = _normalize_frames(merged, symbols=symbols, start=start_day, end=end_day) validation = validate_dataset_frames(merged, symbols=symbols, start=start_day, end=end_day) master_root = None - if previous["files"]["catalog"].get("provider") == "qdk.historical-instrument-master/v1": + if previous["files"]["catalog"].get("provider") == INSTRUMENT_MASTER_SCHEMA: master_root = root.resolve() / "snapshots" / previous["snapshot_id"] / "instrument_master" return _publish( root, @@ -1132,6 +1144,7 @@ def bind_instrument_master( "instrument_master_bundle_sha256": master_manifest["bundle_sha256"], }, instrument_master_root=instrument_master_root, + preserved_history_root=root / "snapshots" / previous["snapshot_id"], ) diff --git a/tests/test_instrument_master.py b/tests/test_instrument_master.py index b8aa8aa..2319bbf 100644 --- a/tests/test_instrument_master.py +++ b/tests/test_instrument_master.py @@ -7,16 +7,39 @@ from quant_data_kit.instrument_master import ( CATALOG_SCHEMA, + CURRENT_VALIDITY, + MASTER_SCOPE, + SOURCE_SCHEMA, + UNAVAILABLE_FEE_SCOPE, import_instrument_master, + instrument_master_prefix, load_instrument_master, + resolve_instrument_catalog, ) +CAPTURED_AT = "2026-09-26T00:00:00Z" + def _hash(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) -> Path: +def _assertion(symbol: str, **fields: str) -> dict: + return { + "symbol": symbol, + "fields": { + field: { + "value": value, + "locator": f"fixture:{field}", + "excerpt": f"fixture declares {field}={value}", + "method": "direct", + } + for field, value in fields.items() + }, + } + + +def _source(root: Path, *, late_rule: bool = False) -> Path: root.mkdir() documents = root / "documents" documents.mkdir() @@ -27,41 +50,73 @@ def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) } for name, payload in payloads.items(): (documents / f"{name}.txt").write_bytes(payload) + + common = { + "catalog_schema_version": CATALOG_SCHEMA, + "catalog_source_version": "official-test-v2", + "symbol": "510300", + "asset_class": "etf", + "product_type": "etf", + "venue": "SSE", + "price_scale": "3", + "price_tick": "0.001", + "quantity_step": "1", + "lot_size": "100", + "commission_rate": "", + "stamp_duty_rate": "", + "fee_fields_scope": UNAVAILABLE_FEE_SCOPE, + "fee_evidence_id": "", + "listing_date": "2012-05-28", + "listing_date_basis": "official-published-document", + "listing_evidence_id": "listing", + "price_limit_rate": "0.10", + "master_scope": MASTER_SCOPE, + } catalog = pd.DataFrame( [ { - "catalog_schema_version": CATALOG_SCHEMA, - "catalog_source_version": "official-test-v1", - "symbol": "510300", - "asset_class": "etf", - "product_type": "etf", - "venue": "SSE", - "price_scale": "3", - "price_tick": "0.001", - "quantity_step": "1", - "lot_size": "100", - "commission_rate": "0", - "stamp_duty_rate": "0", - "effective_from": "2025-01-02T00:00:00Z", - "effective_to": "2026-09-25T00:00:00Z", - "available_at": available_at, - "listing_date": "2012-05-28", - "listing_date_basis": "official-published-document", - "listing_evidence_id": "listing", - "trading_rule_evidence_ids": "rule-2023;rule-2026", - "price_limit_evidence_ids": "rule-2023;rule-2026", - "price_limit_rate": "0.10", - } + **common, + "effective_from": "2023-04-10T00:00:00Z", + "effective_to": "2026-07-06T00:00:00Z", + "available_at": "2023-02-17T08:00:00Z", + "trading_rule_evidence_id": "rule-2023", + "price_limit_evidence_id": "rule-2023", + }, + { + **common, + "effective_from": "2026-07-06T00:00:00Z", + "effective_to": CAPTURED_AT, + "available_at": "2026-04-25T00:00:00Z", + "trading_rule_evidence_id": "rule-2026", + "price_limit_evidence_id": "rule-2026", + }, ] ) catalog_path = root / "catalog.csv" catalog.to_csv(catalog_path, index=False) + listing_assertion = _assertion( + "510300", + asset_class="etf", + product_type="etf", + venue="SSE", + listing_date="2012-05-28", + ) + rule_assertion = _assertion( + "510300", + venue="SSE", + price_scale="3", + price_tick="0.001", + quantity_step="1", + lot_size="100", + price_limit_rate="0.10", + ) declaration = { - "schema_version": "qdk.historical-instrument-master-source/v1", - "captured_at": "2026-09-26T00:00:00Z", + "schema_version": SOURCE_SCHEMA, + "captured_at": CAPTURED_AT, "rights_note": "test fixture", "coverage": { - "instrument_master": "field-evidence-complete", + "instrument_master": "typed-field-evidence-complete", + "fees": UNAVAILABLE_FEE_SCOPE, "daily_trading_status": "unavailable-no-positive-state-inference", }, "catalog_file": "catalog.csv", @@ -73,11 +128,15 @@ def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) "publisher": "official exchange", "title": "listing notice", "source_uri": "https://exchange.example/listing", - "published_at": "2012-05-23T08:00:00Z", + "published_at": "2012-05-23T00:00:00Z", + "available_at": "2012-05-23T08:00:00Z", "effective_from": "2012-05-28T00:00:00Z", - "effective_to": "2100-01-01T00:00:00Z", + "effective_to": CAPTURED_AT, + "validity_basis": CURRENT_VALIDITY, + "availability_basis": "official-page-time", "file": "documents/listing.txt", "sha256": _hash(documents / "listing.txt"), + "assertions": [listing_assertion], }, { "document_id": "rule-2023", @@ -85,11 +144,15 @@ def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) "publisher": "official exchange", "title": "2023 rule", "source_uri": "https://exchange.example/rule-2023", - "published_at": "2023-02-17T08:00:00Z", + "published_at": "2023-02-17T00:00:00Z", + "available_at": "2023-02-17T08:00:00Z", "effective_from": "2023-04-10T00:00:00Z", "effective_to": "2026-07-06T00:00:00Z", + "validity_basis": "superseded-by-next-version", + "availability_basis": "official-page-time", "file": "documents/rule-2023.txt", "sha256": _hash(documents / "rule-2023.txt"), + "assertions": [rule_assertion], }, { "document_id": "rule-2026", @@ -97,11 +160,15 @@ def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) "publisher": "official exchange", "title": "2026 rule", "source_uri": "https://exchange.example/rule-2026", - "published_at": ("2026-07-07T08:00:00Z" if late_rule else "2026-04-24T08:00:00Z"), + "published_at": "2026-04-24T00:00:00Z", + "available_at": ("2026-07-07T00:00:00Z" if late_rule else "2026-04-25T00:00:00Z"), "effective_from": "2026-07-06T00:00:00Z", - "effective_to": "2100-01-01T00:00:00Z", + "effective_to": CAPTURED_AT, + "validity_basis": CURRENT_VALIDITY, + "availability_basis": "official-page-date-next-utc-day", "file": "documents/rule-2026.txt", "sha256": _hash(documents / "rule-2026.txt"), + "assertions": [rule_assertion], }, ], } @@ -111,14 +178,25 @@ def _source(root: Path, *, available_at="2023-02-17T08:00:00Z", late_rule=False) return root -def test_import_verifies_source_identity_hashes_and_rule_intervals(tmp_path): +def _update_catalog_hash(source: Path) -> None: + declaration_path = source / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + declaration["catalog_sha256"] = _hash(source / "catalog.csv") + declaration_path.write_text(json.dumps(declaration, ensure_ascii=False), encoding="utf-8") + + +def test_import_verifies_source_identity_assertions_and_rule_versions(tmp_path): source = _source(tmp_path / "source") bundle = tmp_path / "bundle" manifest = import_instrument_master(source, bundle) loaded, catalog = load_instrument_master(bundle) assert loaded == manifest - assert catalog.loc[0, "symbol"] == "510300" - assert loaded["coverage"]["daily_trading_status"].startswith("unavailable") + assert len(catalog) == 2 + assert catalog["symbol"].tolist() == ["510300", "510300"] + assert loaded["refresh_required_after"] == "2026-09-26T00:00:00+00:00" + assert (bundle / "source" / "declaration.json").read_bytes() == ( + source / "declaration.json" + ).read_bytes() document = bundle / loaded["documents"][0]["file"] document.write_bytes(b"tampered") @@ -137,29 +215,150 @@ def test_manifest_source_identity_change_is_rejected(tmp_path): load_instrument_master(bundle) -def test_catalog_rejects_future_availability_and_late_rule_publication(tmp_path): - with pytest.raises(ValueError, match="before its initial rules"): +def test_catalog_rejects_late_rule_and_unobserved_future_validity(tmp_path): + with pytest.raises(ValueError, match="not available before"): import_instrument_master( - _source(tmp_path / "early", available_at="2023-02-16T08:00:00Z"), - tmp_path / "early-bundle", - ) - with pytest.raises(ValueError, match="not public before it became effective"): - import_instrument_master( - _source(tmp_path / "late", late_rule=True), - tmp_path / "late-bundle", + _source(tmp_path / "late", late_rule=True), tmp_path / "late-bundle" ) + source = _source(tmp_path / "future") + declaration_path = source / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + declaration["documents"][-1]["effective_to"] = "2099-01-01T00:00:00Z" + declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + with pytest.raises(ValueError, match="extends beyond its observed capture horizon"): + import_instrument_master(source, tmp_path / "future-bundle") + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("asset_class", "equity"), + ("product_type", "stock"), + ("venue", "FAKE"), + ("price_tick", "0.002"), + ("lot_size", "7"), + ("price_limit_rate", "9.99"), + ], +) +def test_catalog_rejects_values_not_bound_to_document_assertions(tmp_path, field, value): + source = _source(tmp_path / field) + catalog_path = source / "catalog.csv" + catalog = pd.read_csv(catalog_path, dtype=str, keep_default_na=False) + catalog.loc[:, field] = value + catalog.to_csv(catalog_path, index=False) + _update_catalog_hash(source) + with pytest.raises(ValueError): + import_instrument_master(source, tmp_path / f"{field}-bundle") + + +def test_catalog_schema_is_closed_and_unavailable_fees_remain_empty(tmp_path): + source = _source(tmp_path / "extra") + catalog_path = source / "catalog.csv" + catalog = pd.read_csv(catalog_path, dtype=str, keep_default_na=False) + catalog["unverified_field"] = "accepted-before-fix" + catalog.to_csv(catalog_path, index=False) + _update_catalog_hash(source) + with pytest.raises(ValueError, match="schema mismatch"): + import_instrument_master(source, tmp_path / "extra-bundle") -def test_etf_catalog_rejects_equity_asset_class(tmp_path): - source = _source(tmp_path / "wrong-class") + source = _source(tmp_path / "fees") catalog_path = source / "catalog.csv" - catalog = pd.read_csv(catalog_path, dtype=str) - catalog.loc[0, "asset_class"] = "equity" + catalog = pd.read_csv(catalog_path, dtype=str, keep_default_na=False) + catalog.loc[:, "commission_rate"] = "0" catalog.to_csv(catalog_path, index=False) + _update_catalog_hash(source) + with pytest.raises(ValueError, match="unavailable fee fields must remain empty"): + import_instrument_master(source, tmp_path / "fees-bundle") + + source = _source(tmp_path / "unlocated-claim") + declaration_path = source / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + del declaration["documents"][0]["assertions"][0]["fields"]["venue"]["locator"] + declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + with pytest.raises(ValueError, match="claim schema is invalid"): + import_instrument_master(source, tmp_path / "unlocated-claim-bundle") + + +def test_flat_view_preserves_initial_availability_and_rejects_rule_changes(tmp_path): + bundle = tmp_path / "bundle" + import_instrument_master(_source(tmp_path / "source"), bundle) + _, catalog = load_instrument_master(bundle) + flat = resolve_instrument_catalog( + catalog, + symbols=["510300"], + start=pd.Timestamp("2025-01-02"), + end=pd.Timestamp("2026-09-24"), + ) + assert flat.loc[0, "available_at"] == "2023-02-17T08:00:00Z" + assert flat.loc[0, "rule_versions"] == "rule-2023|rule-2023;rule-2026|rule-2026" + + source = _source(tmp_path / "changed") + catalog_path = source / "catalog.csv" + changed = pd.read_csv(catalog_path, dtype=str, keep_default_na=False) + changed.loc[1, "price_tick"] = "0.002" + changed.to_csv(catalog_path, index=False) + _update_catalog_hash(source) declaration_path = source / "declaration.json" declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + declaration["documents"][-1]["assertions"][0]["fields"]["price_tick"]["value"] = "0.002" + declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + changed_bundle = tmp_path / "changed-bundle" + import_instrument_master(source, changed_bundle) + _, changed_catalog = load_instrument_master(changed_bundle) + with pytest.raises(ValueError, match="cannot represent.*price_tick"): + resolve_instrument_catalog( + changed_catalog, + symbols=["510300"], + start=pd.Timestamp("2025-01-02"), + end=pd.Timestamp("2026-09-24"), + ) + + +def test_prefix_ignores_only_verified_current_horizon_extension(tmp_path): + first_source = _source(tmp_path / "first-source") + first_bundle = tmp_path / "first-bundle" + import_instrument_master(first_source, first_bundle) + + second_source = _source(tmp_path / "second-source") + declaration_path = second_source / "declaration.json" + declaration = json.loads(declaration_path.read_text(encoding="utf-8")) + declaration["captured_at"] = "2026-09-27T00:00:00Z" + for document in declaration["documents"]: + if document["effective_to"] == CAPTURED_AT: + document["effective_to"] = declaration["captured_at"] + catalog_path = second_source / "catalog.csv" + catalog = pd.read_csv(catalog_path, dtype=str, keep_default_na=False) + catalog.loc[catalog["effective_to"].eq(CAPTURED_AT), "effective_to"] = declaration[ + "captured_at" + ] + catalog.to_csv(catalog_path, index=False) declaration["catalog_sha256"] = _hash(catalog_path) declaration_path.write_text(json.dumps(declaration), encoding="utf-8") + second_bundle = tmp_path / "second-bundle" + import_instrument_master(second_source, second_bundle) - with pytest.raises(ValueError, match="ETF product_type requires ETF asset_class"): - import_instrument_master(source, tmp_path / "wrong-class-bundle") + cutoff = CAPTURED_AT + assert instrument_master_prefix(first_bundle, cutoff=cutoff) == instrument_master_prefix( + second_bundle, cutoff=cutoff + ) + + changed_source = _source(tmp_path / "changed-history-source") + changed_catalog_path = changed_source / "catalog.csv" + changed_catalog = pd.read_csv(changed_catalog_path, dtype=str, keep_default_na=False) + changed_catalog.loc[0, "price_tick"] = "0.002" + changed_catalog.to_csv(changed_catalog_path, index=False) + changed_declaration_path = changed_source / "declaration.json" + changed_declaration = json.loads(changed_declaration_path.read_text(encoding="utf-8")) + changed_declaration["catalog_sha256"] = _hash(changed_catalog_path) + changed_declaration["documents"][1]["assertions"][0]["fields"]["price_tick"][ + "value" + ] = "0.002" + changed_declaration_path.write_text(json.dumps(changed_declaration), encoding="utf-8") + changed_bundle = tmp_path / "changed-history-bundle" + import_instrument_master(changed_source, changed_bundle) + assert instrument_master_prefix(first_bundle, cutoff=cutoff) != instrument_master_prefix( + changed_bundle, cutoff=cutoff + ) + with pytest.raises(ValueError, match="cutoff exceeds"): + instrument_master_prefix(first_bundle, cutoff="2026-09-27T00:00:00Z") diff --git a/tests/test_research_dataset.py b/tests/test_research_dataset.py index 0d68bff..f08e1ea 100644 --- a/tests/test_research_dataset.py +++ b/tests/test_research_dataset.py @@ -145,6 +145,15 @@ def test_bind_historical_master_publishes_child_without_refetch(tmp_path): _source(source, dates, [10, 10, 9, 9.1, 9.2], [9, 9, 9, 9.1, 9.2]) root = tmp_path / "dataset" parent = _build(root, source) + parent_snapshot = root / "snapshots" / parent["snapshot_id"] + preserved_history = { + relative: (parent_snapshot / relative).read_bytes() + for relative in ( + "history/history.parquet", + "history/manifest.json", + "history/source.json", + ) + } master = tmp_path / "master" import_instrument_master(_master_source(tmp_path / "master-source"), master) @@ -155,8 +164,11 @@ def test_bind_historical_master_publishes_child_without_refetch(tmp_path): ) assert child["parent_snapshot_id"] == parent["snapshot_id"] assert child["update_evidence"]["market_data_refetched"] is False - assert child["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v1" + assert child["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v2" assert child["consumer_contract"]["instrument_master_availability"].startswith("official") + child_snapshot = root / "snapshots" / child["snapshot_id"] + for relative, expected in preserved_history.items(): + assert (child_snapshot / relative).read_bytes() == expected _, frames = load_research_snapshot(root, child["snapshot_id"]) assert frames["catalog"].loc[0, "available_at"] == "2023-02-17T08:00:00Z" @@ -175,7 +187,7 @@ def test_bind_historical_master_publishes_child_without_refetch(tmp_path): license_note="test fixture declaration", ) assert updated["parent_snapshot_id"] == child["snapshot_id"] - assert updated["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v1" + assert updated["files"]["catalog"]["provider"] == "qdk.historical-instrument-master/v2" assert updated["validation"]["instrument_master"]["passed"] is True From b5621379e1a15562371be31c03f354a6acf512e4 Mon Sep 17 00:00:00 2001 From: Neko65536 Date: Sat, 26 Sep 2026 17:06:31 +0800 Subject: [PATCH 3/6] Format instrument evidence regressions with pinned Ruff --- tests/test_instrument_master.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_instrument_master.py b/tests/test_instrument_master.py index 2319bbf..8201ee2 100644 --- a/tests/test_instrument_master.py +++ b/tests/test_instrument_master.py @@ -351,9 +351,7 @@ def test_prefix_ignores_only_verified_current_horizon_extension(tmp_path): changed_declaration_path = changed_source / "declaration.json" changed_declaration = json.loads(changed_declaration_path.read_text(encoding="utf-8")) changed_declaration["catalog_sha256"] = _hash(changed_catalog_path) - changed_declaration["documents"][1]["assertions"][0]["fields"]["price_tick"][ - "value" - ] = "0.002" + changed_declaration["documents"][1]["assertions"][0]["fields"]["price_tick"]["value"] = "0.002" changed_declaration_path.write_text(json.dumps(changed_declaration), encoding="utf-8") changed_bundle = tmp_path / "changed-history-bundle" import_instrument_master(changed_source, changed_bundle) From f4a3fe532b59ed4e5112ad9d188245c180773855 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:45:29 +0800 Subject: [PATCH 4/6] Add explicit US research snapshots and SEC point-in-time facts --- docs/us-research.md | 13 ++ pyproject.toml | 3 +- requirements.lock | 19 +- src/quant_data_kit/us_research/__init__.py | 6 + src/quant_data_kit/us_research/calendar.py | 36 +++ src/quant_data_kit/us_research/prices.py | 230 ++++++++++++++++++ src/quant_data_kit/us_research/sec.py | 239 +++++++++++++++++++ src/quant_data_kit/us_research/universe.py | 37 +++ tests/test_us_research.py | 258 +++++++++++++++++++++ 9 files changed, 837 insertions(+), 4 deletions(-) create mode 100644 docs/us-research.md create mode 100644 src/quant_data_kit/us_research/__init__.py create mode 100644 src/quant_data_kit/us_research/calendar.py create mode 100644 src/quant_data_kit/us_research/prices.py create mode 100644 src/quant_data_kit/us_research/sec.py create mode 100644 src/quant_data_kit/us_research/universe.py create mode 100644 tests/test_us_research.py diff --git a/docs/us-research.md b/docs/us-research.md new file mode 100644 index 0000000..2131d57 --- /dev/null +++ b/docs/us-research.md @@ -0,0 +1,13 @@ +# US research data v1 + +`quant_data_kit.us_research` is an explicit USD research namespace. It does not change the legacy six-digit domestic symbol adapters. + +- `calendar.schedule`: XNYS sessions with UTC opens/closes, DST and half-days. +- `calendar.settlement_session`: standard T+2/T+1 date regimes from 2018. +- `prices.validate_prices/write_bundle/load_bundle`: OHLC/action validation and immutable hashed CSV snapshots. +- `prices.fetch_yahoo`: opt-in Yahoo retrospective split-normalized prices, not raw as-traded prices or historical constituent certification. Dividend payment dates remain unknown. +- `universe.members_asof`: source-backed membership events selected by effective session and knowledge time. +- `sec.download_sec/read_sec_snapshot`: explicit SEC contact identity, rate-limited snapshots, full submissions pagination and hash checks. +- `sec.filing_table/fact_table/facts_asof/annual_quality`: accession acceptance plus modeled processing lag, retained restatements, period-matched annual quality metrics. Facts with unknown acceptance are excluded and audited. + +Install `.[us-research]` for optional calendar/Yahoo/SEC clients. Unit tests never require live data or contact credentials. `historical_pit` is source evidence supplied by the importer, not automatic certification by the package. diff --git a/pyproject.toml b/pyproject.toml index 3098d23..f821ad9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ ] [project.optional-dependencies] +us-research = ["exchange-calendars>=4.5,<5", "yfinance>=0.2.65,<1", "requests>=2.32,<3"] akshare = ["akshare>=1.12"] baostock = ["baostock>=0.8.9"] tushare = ["tushare>=1.4.24"] @@ -35,7 +36,7 @@ all-equity-providers = [ "yfinance>=0.2.65", "requests>=2.32,<3", ] -dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0"] +dev = ["coverage>=7.6", "psutil>=6.0,<8.0", "pytest>=7.4", "ruff>=0.9.0", "exchange-calendars>=4.5,<5"] [project.scripts] qdk-validate = "quant_data_kit.cli:main_validate" diff --git a/requirements.lock b/requirements.lock index 4908d9a..7e2192b 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --output-file=requirements.lock --strip-extras pyproject.toml +# pip-compile --allow-unsafe --build-deps-for=editable --constraint=requirements-constraints.txt --extra=dev --index-url=https://pypi.org/simple --no-index --output-file=requirements.lock --strip-extras pyproject.toml # attrs==26.1.0 # via @@ -14,6 +14,8 @@ coverage==7.16.0 # via quant-data-kit (pyproject.toml) duckdb==1.5.5 # via quant-data-kit (pyproject.toml) +exchange-calendars==4.13.2 + # via quant-data-kit (pyproject.toml) iniconfig==2.3.0 # via pytest jsonschema==4.26.0 @@ -22,8 +24,11 @@ jsonschema-rs==0.52.0 # via quant-data-kit (pyproject.toml) jsonschema-specifications==2025.9.1 # via jsonschema +korean-lunar-calendar==0.4.0 + # via exchange-calendars numpy==2.2.6 # via + # exchange-calendars # pandas # quant-data-kit (pyproject.toml) orjson==3.12.0 @@ -31,7 +36,9 @@ orjson==3.12.0 packaging==26.3 # via pytest pandas==2.3.3 - # via quant-data-kit (pyproject.toml) + # via + # exchange-calendars + # quant-data-kit (pyproject.toml) pluggy==1.6.0 # via pytest psutil==7.2.2 @@ -40,6 +47,8 @@ pyarrow==25.0.1 # via quant-data-kit (pyproject.toml) pygments==2.21.0 # via pytest +pyluach==2.3.0 + # via exchange-calendars pytest==9.1.1 # via quant-data-kit (pyproject.toml) python-dateutil==2.9.0.post0 @@ -61,12 +70,16 @@ ruff==0.16.5 # via quant-data-kit (pyproject.toml) six==1.17.0 # via python-dateutil +toolz==1.1.0 + # via exchange-calendars typing-extensions==4.16.0 # via # quant-data-kit (pyproject.toml) # referencing tzdata==2026.3 - # via pandas + # via + # exchange-calendars + # pandas websockets==15.0.1 # via quant-data-kit (pyproject.toml) diff --git a/src/quant_data_kit/us_research/__init__.py b/src/quant_data_kit/us_research/__init__.py new file mode 100644 index 0000000..e06caaa --- /dev/null +++ b/src/quant_data_kit/us_research/__init__.py @@ -0,0 +1,6 @@ +"""Explicit US research contracts, separate from legacy domestic adapters.""" + +from .calendar import schedule, settlement_session +from .prices import load_bundle, validate_prices, write_bundle + +__all__ = ["load_bundle", "schedule", "settlement_session", "validate_prices", "write_bundle"] diff --git a/src/quant_data_kit/us_research/calendar.py b/src/quant_data_kit/us_research/calendar.py new file mode 100644 index 0000000..b51076f --- /dev/null +++ b/src/quant_data_kit/us_research/calendar.py @@ -0,0 +1,36 @@ +"""Regular US equity sessions; UTC instants retain DST and early closes.""" + +from functools import lru_cache + +import pandas as pd + + +@lru_cache(maxsize=32) +def _calendar(start: str, end: str): + import exchange_calendars as xcals + + return xcals.get_calendar("XNYS", start=start, end=end) + + +def schedule(start: str, end: str) -> pd.DataFrame: + start_ts, end_ts = pd.Timestamp(start).normalize(), pd.Timestamp(end).normalize() + if start_ts.tz is not None or end_ts.tz is not None or start_ts > end_ts: + raise ValueError("session bounds must be ordered, timezone-naive dates") + cal = _calendar(str(start_ts.date()), str((end_ts + pd.Timedelta(days=15)).date())) + result = cal.schedule.loc[str(start_ts.date()) : str(end_ts.date()), ["open", "close"]].copy() + result.index = pd.DatetimeIndex(result.index).tz_localize(None) + result.index.name = "session" + return result + + +@lru_cache(maxsize=4096) +def settlement_session(session: str) -> str: + """Standard US equity cash settlement since 2018, not same-day resale rules.""" + day = pd.Timestamp(session).normalize() + if day < pd.Timestamp("2018-01-01"): + raise ValueError("US settlement model is supported from 2018 onward") + days = schedule(str(day.date()), str((day + pd.Timedelta(days=15)).date())).index + if not len(days) or days[0] != day: + raise ValueError("trade date is not a regular US equity session") + lag = 1 if day >= pd.Timestamp("2024-05-28") else 2 + return str(days[lag].date()) diff --git a/src/quant_data_kit/us_research/prices.py b/src/quant_data_kit/us_research/prices.py new file mode 100644 index 0000000..a3955b3 --- /dev/null +++ b/src/quant_data_kit/us_research/prices.py @@ -0,0 +1,230 @@ +"""Immutable USD daily research panels with explicit adjustment semantics.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import numpy as np +import pandas as pd + +from .calendar import schedule + +REQUIRED = { + "instrument_id", + "symbol", + "session", + "open", + "high", + "low", + "close", + "volume", + "dividend", + "split_ratio", + "currency", + "price_basis", + "available_at", + "source", +} + + +def utc(value, field: str = "timestamp") -> pd.Timestamp: + stamp = pd.Timestamp(value) + if pd.isna(stamp) or stamp.tz is None: + raise ValueError(f"{field} requires a timezone-aware timestamp") + return stamp.tz_convert("UTC") + + +def us_symbol(value: str) -> str: + symbol = value.strip().upper() + if not re.fullmatch(r"[A-Z][A-Z0-9]*(?:[.-][A-Z0-9]+)*", symbol): + raise ValueError(f"invalid explicit US equity symbol: {value!r}") + return symbol + + +def validate_prices(frame: pd.DataFrame) -> pd.DataFrame: + missing = REQUIRED - set(frame.columns) + if missing or frame.empty: + raise ValueError(f"nonempty US price panel required; missing {sorted(missing)}") + out = frame.copy() + for name in ("instrument_id", "symbol", "source"): + if out[name].isna().any() or out[name].astype(str).str.strip().eq("").any(): + raise ValueError(f"empty {name}") + out["symbol"] = out.symbol.map(us_symbol) + out["session"] = pd.to_datetime(out.session, errors="raise") + if out.session.dt.tz is not None or not out.session.eq(out.session.dt.normalize()).all(): + raise ValueError("session must contain plain dates") + if out.duplicated(["instrument_id", "session"]).any(): + raise ValueError("duplicate instrument/session") + if not out.currency.eq("USD").all(): + raise ValueError("US research requires explicit USD prices") + if not out.price_basis.isin(["raw", "split_normalized"]).all(): + raise ValueError("prices must be raw or split_normalized, never total-return prices") + if out.groupby("instrument_id").price_basis.nunique().gt(1).any(): + raise ValueError("price basis changed within an instrument") + numeric = ["open", "high", "low", "close", "volume", "dividend", "split_ratio"] + for name in numeric: + out[name] = pd.to_numeric(out[name], errors="raise") + if not np.isfinite(out[numeric].to_numpy(dtype=float)).all(): + raise ValueError("nonfinite price or action") + if (out[["open", "high", "low", "close", "split_ratio"]] <= 0).any().any(): + raise ValueError("prices and split ratios must be positive") + if (out[["volume", "dividend"]] < 0).any().any(): + raise ValueError("volume and dividends must be nonnegative") + if ( + (out.high < out[["open", "close", "low"]].max(axis=1)) + | (out.low > out[["open", "close", "high"]].min(axis=1)) + ).any(): + raise ValueError("inconsistent OHLC") + sessions = schedule(str(out.session.min().date()), str(out.session.max().date())) + if not out.session.isin(sessions.index).all(): + raise ValueError("price row outside regular US equity sessions") + for _, group in out.groupby("instrument_id"): + expected = sessions.loc[group.session.min() : group.session.max()].index + if len(group) != len(expected): + raise ValueError("missing interior daily bars; explicit status handling is required") + out["available_at"] = out.available_at.map(lambda value: utc(value, "available_at")) + closes = out.session.map(sessions.close) + if (out.available_at < closes).any(): + raise ValueError("complete daily bars cannot be available before session close") + if "pay_date" not in out: + out["pay_date"] = None + for row in out.loc[out.pay_date.notna() & out.pay_date.ne("")].itertuples(): + pay = pd.Timestamp(row.pay_date) + if pay.tz is not None or pay < row.session or pay != pay.normalize(): + raise ValueError("invalid dividend payment date") + return out.sort_values(["session", "instrument_id"]).reset_index(drop=True) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_bundle( + path: str | Path, + prices: pd.DataFrame, + provenance: dict, + *, + tables: dict[str, pd.DataFrame] | None = None, +) -> dict: + prices = validate_prices(prices) + required = {"source", "observed_at", "evidence_kind", "universe_kind"} + if not required.issubset(provenance): + raise ValueError(f"provenance needs {sorted(required)}") + utc(provenance["observed_at"], "observed_at") + if provenance["evidence_kind"] not in {"synthetic", "retrospective", "historical_pit"}: + raise ValueError("unknown evidence kind") + path = Path(path) + path.mkdir(parents=True, exist_ok=False) + payload = {"prices": prices, **(tables or {})} + if payload["prices"] is not prices: + raise ValueError("cannot replace validated prices") + files = {} + for name, table in payload.items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + raise ValueError("invalid table name") + target = path / f"{name}.csv" + table.to_csv(target, index=False, lineterminator="\n") + files[target.name] = {"sha256": sha256(target), "rows": len(table)} + manifest = {"schema": "puresaber.us-research/1", "provenance": provenance, "files": files} + (path / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2, allow_nan=False), encoding="utf-8" + ) + return manifest + + +def load_bundle(path: str | Path) -> tuple[dict, dict[str, pd.DataFrame]]: + path = Path(path) + manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema") != "puresaber.us-research/1": + raise ValueError("unsupported bundle schema") + tables = {} + for name, entry in manifest["files"].items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*\.csv", name): + raise ValueError("invalid manifest file name") + target = path / name + if sha256(target) != entry["sha256"]: + raise ValueError(f"bundle hash mismatch: {name}") + table = pd.read_csv(target, dtype={"instrument_id": str, "symbol": str}) + if len(table) != entry["rows"]: + raise ValueError(f"bundle row count mismatch: {name}") + tables[name[:-4]] = table + tables["prices"] = validate_prices(tables["prices"]) + return manifest, tables + + +def fetch_yahoo(symbols: list[str], start: str, end: str) -> tuple[pd.DataFrame, dict]: + """Retrospective split-normalized quotes. No historical-universe/PIT claim. + + Yahoo OHLC with auto_adjust=False is split-normalized, not as-traded raw. + Reported splits remain metadata; consumers must not multiply positions again. + Payment dates are not supplied, so dividends stay receivable in cash replay. + """ + import yfinance as yf + + if not symbols or len(set(symbols)) != len(symbols): + raise ValueError("unique symbols required") + sessions = schedule(start, end) + observed = pd.Timestamp.now(tz="UTC") + if sessions.empty or sessions.close.max() >= observed: + raise ValueError("request completed historical sessions only") + pieces = [] + for symbol in symbols: + symbol = us_symbol(symbol) + ticker = yf.Ticker(symbol) + history = ticker.history( + start=start, + end=str((pd.Timestamp(end) + pd.Timedelta(days=1)).date()), + auto_adjust=False, + back_adjust=False, + actions=True, + repair=False, + raise_errors=True, + ) + metadata = ticker.get_history_metadata() + if metadata.get("currency") != "USD": + raise ValueError(f"{symbol}: source did not confirm USD") + if history.empty or not {"Dividends", "Stock Splits"}.issubset(history): + raise ValueError(f"{symbol}: missing prices/actions") + local_dates = history.index.tz_convert("America/New_York").tz_localize(None).normalize() + for day, (_, row) in zip(local_dates, history.iterrows(), strict=True): + if day not in sessions.index: + raise ValueError(f"{symbol}: unexpected session {day}") + pieces.append( + { + "instrument_id": f"US:YAHOO:{symbol}", + "symbol": symbol, + "session": day, + "open": row.Open, + "high": row.High, + "low": row.Low, + "close": row.Close, + "volume": row.Volume, + "dividend": row.Dividends, + "split_ratio": row["Stock Splits"] or 1.0, + "currency": "USD", + "price_basis": "split_normalized", + "available_at": sessions.loc[day, "close"], + "source": f"yfinance:{yf.__version__}", + "pay_date": None, + } + ) + provenance = { + "source": "Yahoo Finance via yfinance", + "observed_at": observed.isoformat(), + "evidence_kind": "retrospective", + "universe_kind": "fixed_cohort", + "symbols": symbols, + "start": start, + "end": end, + "limitations": [ + "Current symbol identities; no historical constituent certification", + "Close timestamps are modeled availability, not archived PIT evidence", + "Split-normalized research share units, not as-traded shares", + "Dividend payment dates absent; cash replay retains receivables", + "Yahoo revisions and licensing apply; snapshot is not a redistribution grant", + ], + } + return validate_prices(pd.DataFrame(pieces)), provenance diff --git a/src/quant_data_kit/us_research/sec.py b/src/quant_data_kit/us_research/sec.py new file mode 100644 index 0000000..8fde43b --- /dev/null +++ b/src/quant_data_kit/us_research/sec.py @@ -0,0 +1,239 @@ +"""SEC snapshots and accession-level facts without backdating restatements.""" + +from __future__ import annotations + +import json +import re +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from .prices import sha256, utc + + +def download_sec(cik: str, destination: str | Path, user_agent: str) -> Path: + """Fetch companyfacts and all submissions pages with a real contact identity.""" + import requests + + if not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", user_agent): + raise ValueError("SEC_USER_AGENT must contain a real contact email") + if not re.fullmatch(r"\d{1,10}", str(cik)): + raise ValueError("CIK must contain 1 to 10 digits") + cik = str(cik).zfill(10) + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + manifest = { + "schema": "puresaber.sec-snapshot/1", + "cik": cik, + "observed_at": pd.Timestamp.now(tz="UTC").isoformat(), + "files": {}, + } + session = requests.Session() + session.headers["User-Agent"] = user_agent + + def fetch(relative: str, name: str): + time.sleep(0.25) + response = session.get("https://data.sec.gov/" + relative, timeout=60) + response.raise_for_status() + payload = response.json() + target = destination / name + target.write_bytes(response.content) + manifest["files"][name] = {"url": response.url, "sha256": sha256(target)} + return payload + + try: + submissions = fetch(f"submissions/CIK{cik}.json", "submissions.json") + for item in submissions["filings"].get("files", []): + name = item["name"] + if not re.fullmatch(r"CIK\d{10}-submissions-\d+\.json", name): + raise ValueError("unexpected SEC archive name") + fetch("submissions/" + name, name) + fetch(f"api/xbrl/companyfacts/CIK{cik}.json", "companyfacts.json") + (destination / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + finally: + session.close() + return destination + + +def read_sec_snapshot(path: str | Path) -> tuple[dict, list[dict], dict]: + path = Path(path) + manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema") != "puresaber.sec-snapshot/1": + raise ValueError("unsupported SEC snapshot") + records = {} + for name, entry in manifest["files"].items(): + if Path(name).name != name or "/" in name or "\\" in name: + raise ValueError("invalid SEC snapshot path") + if sha256(path / name) != entry["sha256"]: + raise ValueError(f"SEC hash mismatch: {name}") + records[name] = json.loads((path / name).read_text(encoding="utf-8")) + submissions = [records["submissions.json"]["filings"]["recent"]] + submissions += [ + value for name, value in records.items() if name.startswith("CIK") and "submissions" in name + ] + return records["companyfacts.json"], submissions, manifest + + +def filing_table( + submissions: list[dict], instrument_id: str, *, lag_minutes: int = 5 +) -> pd.DataFrame: + if isinstance(lag_minutes, bool) or not isinstance(lag_minutes, int) or lag_minutes < 0: + raise ValueError("lag_minutes must be a nonnegative integer") + rows = [] + for page in submissions: + columns = ("accessionNumber", "acceptanceDateTime", "form", "filingDate") + if not all(name in page for name in columns): + raise ValueError("SEC submissions missing acceptance timestamps") + if len({len(page[name]) for name in columns}) != 1: + raise ValueError("SEC submissions columns have different lengths") + for accession, accepted, form, filed in zip(*(page[name] for name in columns), strict=True): + at = utc(accepted, "SEC acceptanceDateTime") + rows.append( + { + "instrument_id": instrument_id, + "accession": accession, + "accepted_at": at, + "available_at": at + pd.Timedelta(minutes=lag_minutes), + "form": form, + "filed": filed, + "source": "SEC EDGAR", + } + ) + result = pd.DataFrame(rows) + if result.empty: + raise ValueError("no SEC filings") + result = result.drop_duplicates() + if result.accession.duplicated().any(): + raise ValueError("conflicting SEC accession metadata") + return result.sort_values(["available_at", "accession"]).reset_index(drop=True) + + +def fact_table( + companyfacts: dict, + filings: pd.DataFrame, + *, + concepts: tuple[str, ...] = ( + "Assets", + "NetIncomeLoss", + "NetCashProvidedByUsedInOperatingActivities", + ), + unit: str = "USD", +) -> tuple[pd.DataFrame, dict]: + """Preserve each filing vintage and exact period; exclude unknown acceptance.""" + index = filings.set_index("accession") + if index.index.duplicated().any() or filings.instrument_id.nunique() != 1: + raise ValueError("one entity and unique accessions required") + rows, missing = [], set() + for concept in concepts: + values = companyfacts.get("facts", {}).get("us-gaap", {}).get(concept, {}) + for fact in values.get("units", {}).get(unit, []): + accession = fact["accn"] + if accession not in index.index: + missing.add(accession) + continue + filing = index.loc[accession] + value = float(fact["val"]) + if not np.isfinite(value): + raise ValueError("nonfinite SEC fact") + end = pd.Timestamp(fact["end"]) + start = pd.Timestamp(fact["start"]) if "start" in fact else pd.NaT + if end.date() > filing.accepted_at.date() or (pd.notna(start) and start > end): + raise ValueError("SEC fact period is inconsistent with filing time") + rows.append( + { + "instrument_id": filing.instrument_id, + "concept": concept, + "unit": unit, + "period_start": start, + "period_end": end, + "value": value, + "accession": accession, + "form": filing.form, + "accepted_at": filing.accepted_at, + "available_at": filing.available_at, + } + ) + columns = [ + "instrument_id", + "concept", + "unit", + "period_start", + "period_end", + "value", + "accession", + "form", + "accepted_at", + "available_at", + ] + result = pd.DataFrame(rows, columns=columns).drop_duplicates() + keys = ["instrument_id", "concept", "unit", "period_start", "period_end", "accession"] + if result.duplicated(keys).any(): + raise ValueError("ambiguous same-accession financial facts") + return result, { + "excluded_unknown_accessions": sorted(missing), + "rows": len(result), + "availability": "SEC acceptance plus configured processing lag", + } + + +def facts_asof(facts: pd.DataFrame, at) -> pd.DataFrame: + facts = facts.copy() + facts["available_at"] = facts.available_at.map(lambda value: utc(value, "fact available_at")) + visible = facts.loc[facts.available_at <= utc(at)].copy() + keys = ["instrument_id", "concept", "unit", "period_start", "period_end"] + return visible.sort_values(["available_at", "accession"]).drop_duplicates(keys, keep="last") + + +def annual_quality(facts: pd.DataFrame, at, *, max_age_days: int = 550) -> pd.DataFrame: + """Annual net-income/assets and cash accruals; no mislabeled quarterly/TTM mixing.""" + visible = facts_asof(facts, at) + visible["period_start"] = pd.to_datetime(visible.period_start) + visible["period_end"] = pd.to_datetime(visible.period_end) + results = [] + for instrument, data in visible.groupby("instrument_id"): + income = data.loc[data.concept.eq("NetIncomeLoss") & data.unit.eq("USD")].copy() + duration = (income.period_end - income.period_start).dt.days + income = income.loc[duration.between(330, 380)].sort_values("period_end") + if income.empty: + continue + latest = income.iloc[-1] + if (utc(at).tz_localize(None) - latest.period_end).days > max_age_days: + continue + assets = data.loc[ + data.concept.eq("Assets") + & data.unit.eq("USD") + & data.period_start.isna() + & data.period_end.eq(latest.period_end) + ] + cash = data.loc[ + data.concept.eq("NetCashProvidedByUsedInOperatingActivities") + & data.unit.eq("USD") + & data.period_start.eq(latest.period_start) + & data.period_end.eq(latest.period_end) + ] + if len(assets) != 1 or len(cash) != 1 or assets.iloc[0].value <= 0: + continue + a, c = assets.iloc[0], cash.iloc[0] + results.append( + { + "instrument_id": instrument, + "period_end": latest.period_end, + "roa_annual": latest.value / a.value, + "accruals_annual": (latest.value - c.value) / a.value, + "available_at": max(latest.available_at, a.available_at, c.available_at), + "accessions": "|".join(sorted({latest.accession, a.accession, c.accession})), + } + ) + return pd.DataFrame( + results, + columns=[ + "instrument_id", + "period_end", + "roa_annual", + "accruals_annual", + "available_at", + "accessions", + ], + ) diff --git a/src/quant_data_kit/us_research/universe.py b/src/quant_data_kit/us_research/universe.py new file mode 100644 index 0000000..aeb7baa --- /dev/null +++ b/src/quant_data_kit/us_research/universe.py @@ -0,0 +1,37 @@ +"""Membership events selected by knowledge time and effective session.""" + +import pandas as pd + +from .prices import utc + + +def validate_membership(frame: pd.DataFrame) -> pd.DataFrame: + required = {"instrument_id", "effective_session", "known_at", "member", "source", "event_id"} + if frame.empty or not required.issubset(frame): + raise ValueError("nonempty historical membership events required") + result = frame.copy() + for field in ("instrument_id", "source", "event_id"): + if result[field].isna().any() or result[field].astype(str).str.strip().eq("").any(): + raise ValueError(f"membership has empty {field}") + if result.event_id.duplicated().any(): + raise ValueError("duplicate membership event") + result["effective_session"] = pd.to_datetime(result.effective_session) + result["known_at"] = result.known_at.map(lambda value: utc(value, "known_at")) + if not result.member.isin([True, False, 0, 1]).all(): + raise ValueError("membership must be boolean") + result["member"] = result.member.astype(bool) + if result.duplicated(["instrument_id", "effective_session", "known_at"]).any(): + raise ValueError("ambiguous membership events") + return result + + +def members_asof(events: pd.DataFrame, session: str, at) -> set[str]: + visible = events.loc[ + (events.known_at <= utc(at)) & (events.effective_session <= pd.Timestamp(session)) + ] + latest = ( + visible.sort_values(["effective_session", "known_at", "event_id"]) + .groupby("instrument_id", sort=False) + .tail(1) + ) + return set(latest.loc[latest.member, "instrument_id"]) diff --git a/tests/test_us_research.py b/tests/test_us_research.py new file mode 100644 index 0000000..93ef350 --- /dev/null +++ b/tests/test_us_research.py @@ -0,0 +1,258 @@ +import json +import sys +from types import SimpleNamespace + +import pandas as pd +import pytest + +from quant_data_kit.us_research.calendar import schedule, settlement_session +from quant_data_kit.us_research.prices import ( + fetch_yahoo, + load_bundle, + us_symbol, + validate_prices, + write_bundle, +) +from quant_data_kit.us_research.sec import ( + annual_quality, + download_sec, + fact_table, + facts_asof, + filing_table, +) +from quant_data_kit.us_research.universe import members_asof, validate_membership + + +@pytest.fixture +def panel(): + cal = schedule("2024-03-08", "2024-03-12") + return pd.DataFrame( + [ + { + "instrument_id": "US:AAPL", + "symbol": "AAPL", + "session": day, + "open": 100, + "high": 102, + "low": 99, + "close": 101, + "volume": 1000, + "dividend": 0, + "split_ratio": 1, + "currency": "USD", + "price_basis": "raw", + "available_at": row.close, + "source": "fixture", + } + for day, row in cal.iterrows() + ] + ) + + +def test_calendar_dst_halfday_settlement(): + cal = schedule("2024-03-08", "2024-03-11") + assert cal.iloc[0].open.hour == 14 + assert cal.iloc[1].open.hour == 13 + assert schedule("2024-11-29", "2024-11-29").iloc[0].close.hour == 18 + assert schedule("2024-07-04", "2024-07-04").empty + assert settlement_session("2024-05-24") == "2024-05-29" + assert settlement_session("2024-05-28") == "2024-05-29" + with pytest.raises(ValueError): + settlement_session("2024-05-27") + with pytest.raises(ValueError): + settlement_session("2017-01-03") + with pytest.raises(ValueError): + schedule("2024-05-30", "2024-05-01") + + +@pytest.mark.parametrize("value", ["AAPL", "BRK.B", "BRK-B", "SPY"]) +def test_us_symbols_are_not_domestic(value): + assert us_symbol(value) == value + + +@pytest.mark.parametrize("value", ["000001", "AAPL.SZ/", "../SPY", ""]) +def test_invalid_symbols(value): + with pytest.raises(ValueError): + us_symbol(value) + + +@pytest.mark.parametrize( + "column,value", + [ + ("close", float("nan")), + ("open", -1), + ("high", 1), + ("volume", -1), + ("dividend", -1), + ("split_ratio", 0), + ("currency", "CNY"), + ("price_basis", "adjusted"), + ("source", ""), + ("available_at", "2024-03-08T09:00:00Z"), + ("available_at", "2024-03-08"), + ("pay_date", "2020-01-01"), + ], +) +def test_bad_panel_rejected(panel, column, value): + panel[column] = panel[column].astype(object) if column in panel else None + panel.loc[0, column] = value + with pytest.raises((ValueError, TypeError)): + validate_prices(panel) + + +def test_bundle_immutable_and_hash_checked(panel, tmp_path): + path = tmp_path / "bundle" + provenance = { + "source": "fixture", + "observed_at": "2024-03-13T00:00:00Z", + "evidence_kind": "synthetic", + "universe_kind": "fixed_cohort", + } + write_bundle(path, panel, provenance) + manifest, tables = load_bundle(path) + assert tables["prices"].currency.eq("USD").all() + assert manifest["provenance"] == provenance + with pytest.raises(FileExistsError): + write_bundle(path, panel, provenance) + with (path / "prices.csv").open("a") as handle: + handle.write("corruption") + with pytest.raises(ValueError, match="hash"): + load_bundle(path) + + +def test_gaps_duplicates_and_basis_are_rejected(panel): + for bad in (panel.drop(index=1), pd.concat([panel, panel.iloc[[0]]])): + with pytest.raises(ValueError): + validate_prices(bad) + panel.loc[0, "price_basis"] = "split_normalized" + with pytest.raises(ValueError): + validate_prices(panel) + + +def test_yahoo_contract_has_no_double_split_or_pit_claim(monkeypatch): + hist = pd.DataFrame( + { + "Open": [50], + "High": [51], + "Low": [49], + "Close": [50], + "Volume": [2000], + "Dividends": [1], + "Stock Splits": [2], + }, + index=pd.DatetimeIndex(["2024-03-08"], tz="America/New_York"), + ) + calls = [] + + class Ticker: + def __init__(self, symbol): + calls.append(symbol) + + def history(self, **kwargs): + assert not kwargs["auto_adjust"] and kwargs["actions"] + return hist + + def get_history_metadata(self): + return {"currency": "USD"} + + monkeypatch.setitem(sys.modules, "yfinance", SimpleNamespace(Ticker=Ticker, __version__="test")) + prices, provenance = fetch_yahoo(["AAPL"], "2024-03-08", "2024-03-08") + assert calls == ["AAPL"] + assert prices.iloc[0].close == 50 + assert prices.iloc[0].price_basis == "split_normalized" + assert provenance["evidence_kind"] == "retrospective" + assert prices.pay_date.isna().all() + + +def test_membership_knowledge_time_not_current_constituents(): + events = validate_membership( + pd.DataFrame( + [ + { + "instrument_id": "OLD", + "effective_session": "2024-01-02", + "known_at": "2024-01-01T00:00Z", + "member": True, + "source": "archive", + "event_id": "1", + }, + { + "instrument_id": "OLD", + "effective_session": "2024-02-01", + "known_at": "2024-02-03T00:00Z", + "member": False, + "source": "revision", + "event_id": "2", + }, + ] + ) + ) + assert members_asof(events, "2024-02-02", "2024-02-02T12:00Z") == {"OLD"} + assert members_asof(events, "2024-02-04", "2024-02-04T12:00Z") == set() + with pytest.raises(ValueError): + validate_membership(pd.concat([events, events])) + + +def test_sec_restatement_vintages_and_periods(): + filings = filing_table( + [ + { + "accessionNumber": ["a", "b"], + "acceptanceDateTime": ["2023-02-01T22:00Z", "2024-02-01T22:00Z"], + "form": ["10-K", "10-K/A"], + "filingDate": ["2023-02-01", "2024-02-01"], + } + ], + "US:A", + ) + concepts = {} + for name, value in [ + ("Assets", 1000), + ("NetIncomeLoss", 100), + ("NetCashProvidedByUsedInOperatingActivities", 130), + ]: + values = [ + {"val": value, "end": "2022-12-31", "accn": "a"}, + {"val": value * 2, "end": "2022-12-31", "accn": "b"}, + ] + if name != "Assets": + for row in values: + row["start"] = "2022-01-01" + concepts[name] = {"units": {"USD": values}} + facts, audit = fact_table({"facts": {"us-gaap": concepts}}, filings) + assert audit["excluded_unknown_accessions"] == [] + assert facts_asof(facts, "2023-02-01T22:04Z").empty + assert set(facts_asof(facts, "2023-02-02T00:00Z").accession) == {"a"} + assert set(facts_asof(facts, "2024-02-02T00:00Z").accession) == {"b"} + quality = annual_quality(facts, "2023-02-02T00:00Z") + assert quality.iloc[0].roa_annual == pytest.approx(0.1) + assert quality.iloc[0].accruals_annual == pytest.approx(-0.03) + assert annual_quality(facts, "2026-01-01T00:00Z").empty + + +def test_sec_missing_acceptance_never_backdated(): + with pytest.raises(ValueError): + filing_table([{"accessionNumber": ["a"]}], "US:A") + with pytest.raises(ValueError, match="contact"): + download_sec("123", "unused", "anonymous") + with pytest.raises(ValueError, match="CIK"): + download_sec("../x", "unused", "test@example.com") + + +def test_manifest_traversal_rejected(panel, tmp_path): + path = tmp_path / "b" + write_bundle( + path, + panel, + { + "source": "x", + "observed_at": "2024-01-01T00:00Z", + "evidence_kind": "synthetic", + "universe_kind": "fixed_cohort", + }, + ) + manifest = json.loads((path / "manifest.json").read_text()) + manifest["files"]["../prices.csv"] = manifest["files"].pop("prices.csv") + (path / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + load_bundle(path) From 6c40fedfd01fb6cff795c3faeacb04ff7f094c83 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:58:48 +0800 Subject: [PATCH 5/6] Separate US settlement calendar and validate SEC identity before optional imports --- docs/us-research.md | 2 ++ src/quant_data_kit/us_research/calendar.py | 24 +++++++++++++++++++--- src/quant_data_kit/us_research/sec.py | 4 ++-- tests/test_us_research.py | 15 ++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/us-research.md b/docs/us-research.md index 2131d57..fe3653f 100644 --- a/docs/us-research.md +++ b/docs/us-research.md @@ -11,3 +11,5 @@ - `sec.filing_table/fact_table/facts_asof/annual_quality`: accession acceptance plus modeled processing lag, retained restatements, period-matched annual quality metrics. Facts with unknown acceptance are excluded and audited. Install `.[us-research]` for optional calendar/Yahoo/SEC clients. Unit tests never require live data or contact credentials. `historical_pit` is source evidence supplied by the importer, not automatic certification by the package. + +Settlement uses regular exchange holidays plus Columbus/Veterans bank holidays, independently of exchange-only ad-hoc closures. Saturday Veterans Day is not observed on Friday by Federal Reserve Banks. The daily research account models settled funds as available on the settlement date, without intraday clearing or broker-specific holds. Sources: [Federal Reserve bank holidays](https://www.federalreserve.gov/releases/k8/default.htm), [DTCC Veterans Day 2024](https://www.dtcc.com/-/media/Files/pdf/2024/10/11/a9503.pdf), [SIFMA January 2025 closure matrix](https://www.sifma.org/resources/guides-playbooks/unscheduled-close-market-matrix). Future or exceptional clearing changes require calendar review. diff --git a/src/quant_data_kit/us_research/calendar.py b/src/quant_data_kit/us_research/calendar.py index b51076f..2135d19 100644 --- a/src/quant_data_kit/us_research/calendar.py +++ b/src/quant_data_kit/us_research/calendar.py @@ -1,8 +1,18 @@ """Regular US equity sessions; UTC instants retain DST and early closes.""" from functools import lru_cache +from typing import ClassVar import pandas as pd +from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, USColumbusDay, sunday_to_monday + + +class _AdditionalBankHolidays(AbstractHolidayCalendar): + # Federal Reserve Banks do not move a Saturday Veterans Day to Friday. + rules: ClassVar = [ + USColumbusDay, + Holiday("Veterans Day", month=11, day=11, observance=sunday_to_monday), + ] @lru_cache(maxsize=32) @@ -29,8 +39,16 @@ def settlement_session(session: str) -> str: day = pd.Timestamp(session).normalize() if day < pd.Timestamp("2018-01-01"): raise ValueError("US settlement model is supported from 2018 onward") - days = schedule(str(day.date()), str((day + pd.Timedelta(days=15)).date())).index - if not len(days) or days[0] != day: + end = day + pd.Timedelta(days=15) + trading_days = schedule(str(day.date()), str(end.date())).index + if not len(trading_days) or trading_days[0] != day: raise ValueError("trade date is not a regular US equity session") + # Settlement has its own calendar: bank holidays prevent cash settlement, + # while exchange-only ad-hoc closures (e.g. national mourning) need not. + cal = _calendar(str(day.date()), str(end.date())) + holidays = cal.regular_holidays.holidays(day, end).union( + _AdditionalBankHolidays().holidays(day, end) + ) + days = pd.bdate_range(day + pd.Timedelta(days=1), end).difference(holidays) lag = 1 if day >= pd.Timestamp("2024-05-28") else 2 - return str(days[lag].date()) + return str(days[lag - 1].date()) diff --git a/src/quant_data_kit/us_research/sec.py b/src/quant_data_kit/us_research/sec.py index 8fde43b..d10ae37 100644 --- a/src/quant_data_kit/us_research/sec.py +++ b/src/quant_data_kit/us_research/sec.py @@ -15,12 +15,12 @@ def download_sec(cik: str, destination: str | Path, user_agent: str) -> Path: """Fetch companyfacts and all submissions pages with a real contact identity.""" - import requests - if not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", user_agent): raise ValueError("SEC_USER_AGENT must contain a real contact email") if not re.fullmatch(r"\d{1,10}", str(cik)): raise ValueError("CIK must contain 1 to 10 digits") + import requests + cik = str(cik).zfill(10) destination = Path(destination) destination.mkdir(parents=True, exist_ok=False) diff --git a/tests/test_us_research.py b/tests/test_us_research.py index 93ef350..e1876a1 100644 --- a/tests/test_us_research.py +++ b/tests/test_us_research.py @@ -65,6 +65,21 @@ def test_calendar_dst_halfday_settlement(): schedule("2024-05-30", "2024-05-01") +@pytest.mark.parametrize( + "trade,settles", + [ + ("2024-10-11", "2024-10-15"), + ("2024-11-08", "2024-11-12"), + ("2023-11-09", "2023-11-13"), + ("2024-03-28", "2024-04-02"), + ("2025-01-08", "2025-01-09"), + ("2018-12-04", "2018-12-06"), + ], +) +def test_settlement_uses_bank_holidays_and_keeps_exchange_only_closures(trade, settles): + assert settlement_session(trade) == settles + + @pytest.mark.parametrize("value", ["AAPL", "BRK.B", "BRK-B", "SPY"]) def test_us_symbols_are_not_domestic(value): assert us_symbol(value) == value From 271a65ee383158b3dd7570e19a3f2b4123f09f79 Mon Sep 17 00:00:00 2001 From: PureSaber <88314620+PureSaber@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:07:26 +0800 Subject: [PATCH 6/6] feat: add evidence-bound financial data foundations --- docs/FINANCIAL_FOUNDATIONS.md | 72 +++++ examples/financial/units.json | 18 ++ pyproject.toml | 1 + src/quant_data_kit/financial/__init__.py | 7 + src/quant_data_kit/financial/actions.py | 95 ++++++ src/quant_data_kit/financial/calendars.py | 91 ++++++ src/quant_data_kit/financial/cli.py | 81 +++++ src/quant_data_kit/financial/common.py | 48 +++ src/quant_data_kit/financial/holdings.py | 183 +++++++++++ src/quant_data_kit/financial/lifecycle.py | 139 ++++++++ src/quant_data_kit/financial/macro.py | 84 +++++ .../financial/reconciliation.py | 207 ++++++++++++ src/quant_data_kit/financial/returns.py | 65 ++++ src/quant_data_kit/financial/status.py | 69 ++++ src/quant_data_kit/financial/units.py | 49 +++ src/quant_data_kit/us_research/sec.py | 185 ++++++++++- .../financial/apple_cash_flow_extract.json | 29 ++ tests/test_financial_evidence.py | 122 +++++++ tests/test_financial_foundations.py | 305 ++++++++++++++++++ 19 files changed, 1849 insertions(+), 1 deletion(-) create mode 100644 docs/FINANCIAL_FOUNDATIONS.md create mode 100644 examples/financial/units.json create mode 100644 src/quant_data_kit/financial/__init__.py create mode 100644 src/quant_data_kit/financial/actions.py create mode 100644 src/quant_data_kit/financial/calendars.py create mode 100644 src/quant_data_kit/financial/cli.py create mode 100644 src/quant_data_kit/financial/common.py create mode 100644 src/quant_data_kit/financial/holdings.py create mode 100644 src/quant_data_kit/financial/lifecycle.py create mode 100644 src/quant_data_kit/financial/macro.py create mode 100644 src/quant_data_kit/financial/reconciliation.py create mode 100644 src/quant_data_kit/financial/returns.py create mode 100644 src/quant_data_kit/financial/status.py create mode 100644 src/quant_data_kit/financial/units.py create mode 100644 tests/fixtures/financial/apple_cash_flow_extract.json create mode 100644 tests/test_financial_evidence.py create mode 100644 tests/test_financial_foundations.py diff --git a/docs/FINANCIAL_FOUNDATIONS.md b/docs/FINANCIAL_FOUNDATIONS.md new file mode 100644 index 0000000..ea74836 --- /dev/null +++ b/docs/FINANCIAL_FOUNDATIONS.md @@ -0,0 +1,72 @@ +# Financial foundations v1 — 01–06 / 08–10 + +这是一组证据驱动的离线金融数据契约，不是全市场历史数据库，也不认证输入来源完整性。所有时间戳必须带时区；effective_at 是经济生效时间，available_at 是研究者获知时间。修订不能提前可见。不可将今天的证券名单或修订值直接回填过去。 + +## 01 单位与因子输入 + +`financial.units.normalize_trading_units(frame, volume_unit=..., amount_unit=..., currency=..., share_basis="raw", lot_size=...)` 显式把手转换为股、千元/万元转换为币种金额。手必须提供 lot_size；不允许复权股数、不推断流通股数。输出 free_float_shares 必须来自当时已知的原始股本口径。历史交易量均值不是换手率。 + +## 02 生命周期与历史证券池 + +`financial.lifecycle.COLUMNS` 是完整事件列清单。事件含稳定 instrument_id、symbol、venue、kind、effective_at、available_at、source、evidence_id、universe_id、successor_id；不适用字段保留空值。支持上市、退市、更名、转板、合并、入池、出池。 +`select_universe(events, effective_at, known_at, universe_id=None, holdings=())` 返回 eligible、retained_holdings、unknown_holdings。出池不等于退市；存量持仓必须继续估值/清算。冲突事件和同时重复的活跃代码/交易所映射拒绝处理。它是 instrument_master 的伴随历史，不替代交易规则主表。 + +## 03 公司行动与总收益 + +`financial.actions.ActionTerms` 统一拆并股、现金分红权益/支付、碎股现金、换股合并、分拆、供股权分派与行权、终止现金。换股/分拆/权利分派必须提供成本分配比例和目标估值，行权必须显式指定 election_quantity；跨币种、做空权益和税务自动推断不支持。 +`financial.returns.total_return_panel(prices, actions, timezone=..., price_basis="raw")` 仅接受原始报价，构造拆股/分红权益的信号指数。假设除息收盘理论再投资，不代表现金到账。复杂或同时多项行动拒绝进入此简化指数，须使用 QExec 账户回放。账本不得再次应用复权价格中的同一行动。 + +## 04 可交易状态 + +`financial.status.permission_asof(records, instrument_id, effective_at, known_at)` 返回买卖两侧许可。字段包含 effective_from、effective_to（右开）、available_at、buy_status/sell_status、reason、source、evidence_id。状态为 tradable/blocked/unknown；缺失、过期、冲突一律 unknown。`no_restriction` 不能作为 tradable 的证据。QExec 和港股适配负责将 unknown 变成订单阻断。 + +## 05 用途日历 + +`PurposeCalendar(calendar_id, purpose, version, available_at, valid_from, valid_to, open_days, source, evidence_kind)` 不隐式生成工作日。purpose 为 trading/settlement/banking/connect/dealing/confirmation；有效区间、版本获知时间和开日都必须显式。 +`CalendarBook([...]).asof(id, purpose, at, on)` 选择当时可见版本，`calendar.advance(on, lag, at=..., purpose=...)` 计算到期日。缺少覆盖直接失败。正 lag 从 on 之后计开日；lag=0 要求 on 本身开放。HK 与基金适配已消费该契约。 + +## 06 来源冲突与新快照 + +1. `discrepancies(observations, at=...)` 比较同一证券/字段/生效时间，先处理金额倍数及百分比；币种、复权口径不一致标为不可直接比较。 +2. `freeze_snapshot(chosen_records)` 创建每个经济键唯一的治理快照。 +3. `adjudicate(case, selected_observation_id=..., resolved_at=..., reviewer=..., rationale=..., evidence_uri=..., prior_snapshot=..., dependencies=...)` 要求人工选择已有证据，计算受影响节点的传递闭包。 +4. `publish_decision(decision, new_path)` 独占创建裁决文件；`apply_decision_snapshot(parent, decision, new_path)` 实际生成修正数据快照，不改旧文件，修正值从裁决时间起可见。 +5. 重算受影响的因子/订单/报告是单独动作；发布裁决不等于重算完成。quant-report-hub 可核验并展示待重算清单。 + +快照和裁决有内容哈希，不等于数字签名/审批权限控制。调用者仍须保管完整来源、旧版本和运行依赖图。 + +## 08 SEC 独立季度、TTM、重述 + +`us_research.sec.quarterly_facts(facts, at)` 按实际 fiscal period 边界识别直接季度、同年度累计差分；不按自然季度硬切，不把累计数字当独立季度。保留 accession 与 vintage_consistent。 +`ttm_facts(..., allow_mixed_vintages=False, max_age_days=200)` 要求四个连续季度，兼容 53 周年，拒绝缺口及未确认可比的跨文件差分。允许混合版本必须显式 opt-in；这不构成重述可比性认证。 +`ttm_quality` 要求同一截止期 USD 净利润/经营现金流和期末资产；ROA 分母是期末资产，不是平均资产。美股 quality_ttm 单独启用，旧 annual 不更名。 +真实小样本位于 tests/fixtures/financial/apple_cash_flow_extract.json：Apple 三份 2024 财年 10-Q 的公开现金流与申报索引事实摘录。它验证累计差分及 acceptance+5min 边界，不是下载快照或全市场回测。SEC 下载仍需用户自己的真实联系身份，未写入凭据。 + +## 09 宏观 vintage + +`financial.macro.from_alfred(observations, series_id=..., unit=..., release_times=..., source=...)` 接受存档 ALFRED JSON。realtime_start 只有日期精度，不能猜成午夜或默认 08:30；release_times 必须显式提供有证据的带时区发布时间。同一系列不能隐式改变单位。`macro_asof` 按当时已知版本选择，缺失点不填充。quant-regime 输出上下文、缺失和陈旧状态，不暗中改变原状态分类器。 + +## 10 基金/ETF 穿透 + +`financial.holdings.COLUMNS` 定义 disclosure_id、fund_id、holding_date、available_at、instrument_id、asset_type、currency、weight、source、evidence_id。披露是完整快照，不把后来缺失的持仓与旧披露拼接。 +`look_through(disclosures, {fund_id: "portfolio_weight"}, at, max_depth=8, max_age_days=180)` 使用 Decimal 并保留每条路径。未披露余额、陈旧、缺失、循环和深度上限都变为 UNKNOWN，不丢弃权重；exposure_summary 聚合重叠证券与币种。现金必须使用 CASH:。 +仅支持 long-only 权重；不自动推断衍生品 delta、杠杆、汇率或债券发行人集团敞口。披露穿透不是实时持仓。金额输入须先用已知 FX 转为统一组合权重。 + +## 使用与回归 + +安装本分支后执行： +```powershell +python -m quant_data_kit.financial.cli --input examples/financial/units.json --output units-result.json +python -m pytest tests/test_financial_foundations.py tests/test_financial_evidence.py +``` +CLI schema 为 puresaber.financial-foundations/1，operation 支持 units/lifecycle/universe/status/calendar/reconcile/sec-ttm/macro/lookthrough。结果明确标注 supplied_facts_not_market_coverage，拒绝覆盖输出文件。 + +## 参考与边界 + +- [SEC 官方 API 与 XBRL](https://www.sec.gov/search-filings/edgar-application-programming-interfaces)：申报版本、单位、期间与 acceptance 来源。 +- [ALFRED realtime period](https://fred.stlouisfed.org/docs/api/fred/realtime_period.html)：知识时点与修订，不等于日内发布时间。 +- [HKEX 结算](https://www.hkex.com.hk/Services/Settlement-and-Depository/Settlement?sc_lang=en)：交易和交收用途分离。 +- [LEAN corporate actions](https://www.quantconnect.com/docs/v2/writing-algorithms/securities/asset-classes/us-equity/corporate-actions)：行动事件与复权口径分离，未复制第三方实现。 +- [Apple IR dividend/split history](https://investor.apple.com/dividend-history/)：4-for-1 拆股条款；测试价格/持仓及保守获知时间为场景值，不冒充真实逐笔证据。 + +这些模块不提供收益承诺、投资建议或实盘下单能力；真实数据授权、完整覆盖及历史可获得性仍需使用者验证。 diff --git a/examples/financial/units.json b/examples/financial/units.json new file mode 100644 index 0000000..1eba561 --- /dev/null +++ b/examples/financial/units.json @@ -0,0 +1,18 @@ +{ + "schema": "puresaber.financial-foundations/1", + "operation": "units", + "records": [ + { + "volume": 12, + "amount": 36, + "free_float_shares": 1000000 + } + ], + "options": { + "volume_unit": "lots", + "lot_size": 100, + "amount_unit": "thousand_currency", + "currency": "CNY", + "share_basis": "raw" + } +} diff --git a/pyproject.toml b/pyproject.toml index fd26dbb..1b06d95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ qdk-capture = "quant_data_kit.capture_v2.cli:main_capture" qdk-providers = "quant_data_kit.cli:main_providers" qdk-research-inputs = "quant_data_kit.research_inputs:main" qdk-research-dataset = "quant_data_kit.research_dataset:main" +qdk-financial = "quant_data_kit.financial.cli:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/quant_data_kit/financial/__init__.py b/src/quant_data_kit/financial/__init__.py new file mode 100644 index 0000000..f3b8171 --- /dev/null +++ b/src/quant_data_kit/financial/__init__.py @@ -0,0 +1,7 @@ +"""Versioned financial facts; explicit knowledge time, provenance and coverage. + +These contracts validate supplied evidence, not completeness of market history. +No provider is contacted implicitly and missing facts are never guessed. +""" + +SCHEMA_VERSION = "puresaber.financial-foundations/1" diff --git a/src/quant_data_kit/financial/actions.py b/src/quant_data_kit/financial/actions.py new file mode 100644 index 0000000..fcea4d5 --- /dev/null +++ b/src/quant_data_kit/financial/actions.py @@ -0,0 +1,95 @@ +"""Explicit corporate-action instructions; no inferred elections or adjustment mixes.""" + +from dataclasses import asdict, dataclass +from decimal import Decimal + +from .common import day, number, utc +from .reconciliation import canonical + +KINDS = { + "split", + "dividend_entitlement", + "dividend_payment", + "cash_in_lieu", + "merger", + "spin_off", + "rights_distribution", + "rights_exercise", + "terminal_cash", +} + + +@dataclass(frozen=True) +class ActionTerms: + event_id: str + instrument_id: str + kind: str + effective_at: str + available_at: str + currency: str + source: str + evidence_id: str + ratio: str = "1" + cash_per_unit: str = "0" + target_id: str | None = None + cost_fraction: str | None = None + election_quantity: str | None = None + retired_quantity: str | None = None + target_mark: str | None = None + parent_mark: str | None = None + entitlement_date: str | None = None + price_basis: str = "raw" + + def __post_init__(self): + if self.kind not in KINDS or self.price_basis != "raw": + raise ValueError("explicit raw-price action with a supported kind required") + if not all( + isinstance(x, str) and x.strip() + for x in ( + self.event_id, + self.instrument_id, + self.currency, + self.source, + self.evidence_id, + ) + ): + raise ValueError("corporate action identity, currency and source evidence required") + if len(self.currency) != 3 or not self.currency.isupper(): + raise ValueError("explicit action currency required") + utc(self.effective_at) + utc(self.available_at) + if self.cost_fraction is None: + if self.kind in {"merger", "spin_off", "rights_distribution"}: + raise ValueError("conversion/distribution requires explicit cost allocation") + object.__setattr__(self, "cost_fraction", "0") + for name in ("ratio", "cash_per_unit", "cost_fraction"): + number(getattr(self, name), nonnegative=True) + if number(self.cost_fraction) > 1: + raise ValueError("cost allocation fraction must be in [0,1]") + if self.kind in {"merger", "spin_off", "rights_distribution", "rights_exercise"}: + if not self.target_id or self.target_id == self.instrument_id: + raise ValueError("conversion requires a different target instrument") + if self.target_mark is None or number(self.target_mark) <= 0 or number(self.ratio) <= 0: + raise ValueError("conversion requires positive ratio and evidenced target mark") + if self.kind in {"spin_off", "rights_distribution"} and ( + self.parent_mark is None or number(self.parent_mark) <= 0 + ): + raise ValueError("distribution requires an explicit ex-event parent mark") + if self.kind == "rights_exercise" and ( + self.election_quantity is None or number(self.election_quantity) <= 0 + ): + raise ValueError("rights exercise requires an explicit positive election") + if self.kind.startswith("dividend_"): + if self.entitlement_date is None: + raise ValueError("dividend phase requires the original entitlement date") + if day(self.entitlement_date).date() > utc(self.effective_at).date(): + raise ValueError("dividend cannot precede entitlement date") + if self.kind == "split" and number(self.ratio) <= Decimal(0): + raise ValueError("split ratio must be positive") + if self.kind == "cash_in_lieu" and ( + self.retired_quantity is None or number(self.retired_quantity) <= 0 + ): + raise ValueError("cash-in-lieu requires explicit retired share quantity") + + def fingerprint(self): + return canonical(asdict(self)) diff --git a/src/quant_data_kit/financial/calendars.py b/src/quant_data_kit/financial/calendars.py new file mode 100644 index 0000000..a112d37 --- /dev/null +++ b/src/quant_data_kit/financial/calendars.py @@ -0,0 +1,91 @@ +"""Evidence-versioned purpose calendars, never implicit weekday fallbacks.""" + +from dataclasses import dataclass + +import pandas as pd + +from .common import day, utc + +PURPOSES = {"trading", "settlement", "banking", "connect", "dealing", "confirmation"} + + +@dataclass(frozen=True) +class PurposeCalendar: + calendar_id: str + purpose: str + version: str + available_at: str + valid_from: str + valid_to: str + open_days: tuple[str, ...] + source: str + evidence_kind: str = "historical_pit" + + def __post_init__(self): + if ( + self.purpose not in PURPOSES + or not self.calendar_id + or not self.version + or not self.source + ): + raise ValueError("calendar identity, purpose, version and source are required") + if self.evidence_kind not in {"synthetic", "retrospective", "historical_pit"}: + raise ValueError("unknown calendar evidence kind") + utc(self.available_at) + lo, hi = day(self.valid_from), day(self.valid_to) + dates = tuple(day(x) for x in self.open_days) + if lo > hi or tuple(sorted(set(dates))) != dates: + raise ValueError("calendar bounds and unique ordered days required") + if any(x < lo or x > hi for x in dates): + raise ValueError("calendar open day outside declared coverage") + object.__setattr__(self, "open_days", tuple(str(x.date()) for x in dates)) + + def check(self, value, *, at, purpose=None): + value = day(value) + if purpose is not None and purpose != self.purpose: + raise ValueError("calendar purpose mismatch") + if utc(at) < utc(self.available_at): + raise ValueError("calendar version was not yet known") + if not day(self.valid_from) <= value <= day(self.valid_to): + raise ValueError("calendar coverage exhausted") + return value + + def is_open(self, value, *, at, purpose=None): + value = self.check(value, at=at, purpose=purpose) + return str(value.date()) in self.open_days + + def advance(self, value, lag, *, at, purpose=None): + """lag=0 requires an open day; positive lag counts opens strictly AFTER value.""" + value = self.check(value, at=at, purpose=purpose) + if type(lag) is not int or lag < 0: + raise ValueError("lag must be a nonnegative integer") + days = pd.DatetimeIndex(self.open_days) + if lag == 0: + if value not in days: + raise ValueError("same-day settlement requires an open purpose day") + return value + future = days[days > value] + if len(future) < lag: + raise ValueError("calendar coverage exhausted before due date") + return future[lag - 1] + + +class CalendarBook: + def __init__(self, calendars): + self.calendars = tuple(calendars) + keys = [(x.calendar_id, x.purpose, utc(x.available_at)) for x in self.calendars] + if len(keys) != len(set(keys)): + raise ValueError("ambiguous calendar publication") + + def asof(self, calendar_id, purpose, at, on): + candidates = [ + x + for x in self.calendars + if x.calendar_id == calendar_id + and x.purpose == purpose + and utc(x.available_at) <= utc(at) + and day(x.valid_from) <= day(on) <= day(x.valid_to) + ] + if not candidates: + raise ValueError("no visible calendar with requested purpose and coverage") + return max(candidates, key=lambda x: utc(x.available_at)) diff --git a/src/quant_data_kit/financial/cli.py b/src/quant_data_kit/financial/cli.py new file mode 100644 index 0000000..66051c2 --- /dev/null +++ b/src/quant_data_kit/financial/cli.py @@ -0,0 +1,81 @@ +"""Offline financial-data operations with immutable JSON request/result artifacts.""" + +import argparse +import json +from dataclasses import asdict +from pathlib import Path + +import pandas as pd + +from . import SCHEMA_VERSION +from .calendars import CalendarBook, PurposeCalendar +from .holdings import exposure_summary, look_through +from .lifecycle import lifecycle_asof, select_universe +from .macro import macro_asof +from .reconciliation import canonical, digest, discrepancies +from .status import permission_asof +from .units import normalize_trading_units + + +def run_request(request): + if request.get("schema") != SCHEMA_VERSION: + raise ValueError("unsupported financial request schema") + operation = request["operation"] + frame = pd.DataFrame(request.get("records", [])) + options = request.get("options", {}) + if operation == "lifecycle": + result = lifecycle_asof(frame, **options).to_dict("records") + elif operation == "universe": + selected = select_universe(frame, **options) + result = {k: sorted(v) for k, v in asdict(selected).items()} + elif operation == "status": + result = asdict(permission_asof(frame, **options)) + elif operation == "calendar": + book = CalendarBook([PurposeCalendar(**x) for x in request["calendars"]]) + cal = book.asof(options["calendar_id"], options["purpose"], options["at"], options["on"]) + result = { + "calendar": asdict(cal), + "due": cal.advance( + options["on"], options["lag"], at=options["at"], purpose=options["purpose"] + ) + .date() + .isoformat(), + } + elif operation == "macro": + result = macro_asof(frame, **options).to_dict("records") + elif operation == "lookthrough": + leaves = look_through(frame, **options) + result = {"summary": exposure_summary(leaves), "leaves": leaves.to_dict("records")} + elif operation == "reconcile": + result = discrepancies(frame, **options) + elif operation == "units": + result = normalize_trading_units(frame, **options).to_dict("records") + elif operation == "sec-ttm": + from quant_data_kit.us_research.sec import ttm_facts + + result = ttm_facts(frame, **options).to_dict("records") + else: + raise ValueError("unknown financial operation") + return { + "schema": SCHEMA_VERSION, + "operation": operation, + "request_sha256": digest(request), + "result": result, + "validation_scope": "supplied_facts_not_market_coverage", + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args(argv) + request = json.loads(Path(args.input).read_text(encoding="utf-8")) + result = run_request(request) + with Path(args.output).open("xb") as stream: + stream.write(canonical(result)) + print(f"{result['operation']}: {args.output}; no market-coverage certification") + + +if __name__ == "__main__": + main() diff --git a/src/quant_data_kit/financial/common.py b/src/quant_data_kit/financial/common.py new file mode 100644 index 0000000..ac3ef18 --- /dev/null +++ b/src/quant_data_kit/financial/common.py @@ -0,0 +1,48 @@ +"""Strict boundary validation shared by financial data domains.""" + +from decimal import Decimal, InvalidOperation + +import pandas as pd + + +def utc(value, field="timestamp") -> pd.Timestamp: + result = pd.Timestamp(value) + if pd.isna(result) or result.tzinfo is None: + raise ValueError(f"{field} requires a nonmissing timezone-aware timestamp") + return result.tz_convert("UTC") + + +def day(value) -> pd.Timestamp: + result = pd.Timestamp(value) + if pd.isna(result) or result.tzinfo is not None or result != result.normalize(): + raise ValueError("expected a nonmissing plain calendar date") + return result + + +def number(value, *, nonnegative=False) -> Decimal: + if isinstance(value, bool): + raise TypeError("boolean is not a financial number") + try: + result = Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"invalid financial number: {value!r}") from exc + if not result.is_finite() or (nonnegative and result < 0): + raise ValueError("financial numbers must be finite and respect sign constraints") + return result + + +def table(frame, columns, *, text=(), timestamps=(), dates=(), unique=()): + missing = set(columns) - set(frame.columns) + if missing: + raise ValueError(f"missing columns: {sorted(missing)}") + out = frame.copy(deep=True) + for col in text: + if out[col].map(lambda x: not isinstance(x, str) or not x.strip()).any(): + raise ValueError(f"{col} must contain nonempty strings") + for col in timestamps: + out[col] = pd.to_datetime(out[col].map(lambda x, field=col: utc(x, field)), utc=True) + for col in dates: + out[col] = pd.to_datetime(out[col].map(day)) + if unique and out.duplicated(list(unique)).any(): + raise ValueError(f"ambiguous duplicate facts: {list(unique)}") + return out diff --git a/src/quant_data_kit/financial/holdings.py b/src/quant_data_kit/financial/holdings.py new file mode 100644 index 0000000..e4c5ccc --- /dev/null +++ b/src/quant_data_kit/financial/holdings.py @@ -0,0 +1,183 @@ +"""Disclosure-aware fund/ETF look-through, retaining every unknown weight.""" + +from decimal import Decimal + +import pandas as pd + +from .common import number, table, utc + +COLUMNS = [ + "disclosure_id", + "fund_id", + "holding_date", + "available_at", + "instrument_id", + "asset_type", + "currency", + "weight", + "source", + "evidence_id", +] +ONE = Decimal(1) + + +def validate_holdings(frame): + out = table( + frame, + COLUMNS, + text=( + "disclosure_id", + "fund_id", + "instrument_id", + "asset_type", + "currency", + "source", + "evidence_id", + ), + dates=("holding_date",), + timestamps=("available_at",), + unique=("disclosure_id", "instrument_id"), + ) + if not out.asset_type.isin(["fund", "security", "cash"]).all(): + raise ValueError("unsupported holding asset type") + if not out.currency.str.fullmatch(r"[A-Z]{3}").all(): + raise ValueError("holding currency must be explicit") + for row in out.itertuples(): + if row.asset_type == "cash" and row.instrument_id != f"CASH:{row.currency}": + raise ValueError("cash identity must be CASH:") + if row.asset_type != "cash" and row.instrument_id.startswith("CASH:"): + raise ValueError("cash identity cannot label a security or fund") + if out.groupby("instrument_id")[["currency", "asset_type"]].nunique().gt(1).any().any(): + raise ValueError("stable holding identity has conflicting currency or asset type") + out["weight"] = out.weight.map(lambda x: number(x, nonnegative=True)) + if (out.holding_date > out.available_at.dt.tz_localize(None).dt.normalize()).any(): + raise ValueError("holdings cannot be disclosed before their holding date") + for _, rows in out.groupby("disclosure_id"): + if any(rows[x].nunique() != 1 for x in ("fund_id", "holding_date", "available_at")): + raise ValueError("one disclosure must be one complete dated snapshot") + if sum(rows.weight, Decimal(0)) > ONE: + raise ValueError("long-only disclosed weights exceed 100%; leverage is unsupported") + snapshots = out[["disclosure_id", "fund_id", "holding_date", "available_at"]].drop_duplicates() + if snapshots.duplicated(["fund_id", "holding_date", "available_at"]).any(): + raise ValueError("ambiguous fund disclosure; reconcile sources first") + return out + + +def holdings_asof(frame, at): + out = validate_holdings(frame) + visible = out.loc[out.available_at <= utc(at)] + snapshots = visible[["disclosure_id", "fund_id", "holding_date", "available_at"]] + ids = ( + snapshots.drop_duplicates() + .sort_values(["holding_date", "available_at"]) + .groupby("fund_id") + .tail(1) + .disclosure_id + ) + return visible.loc[visible.disclosure_id.isin(ids)].reset_index(drop=True) + + +def look_through(frame, allocations, at, *, max_depth=8, max_age_days=180): + """Return weighted leaves with paths; sum(weights) equals supplied allocations. + + Allocations are fund -> portfolio WEIGHT, not amounts in different currencies. + Missing, stale, cyclic, depth-limited and undisclosed portions stay UNKNOWN. + Shares may be retained as optional source columns, never silently treated as weights. + """ + if ( + type(max_depth) is not int + or max_depth < 1 + or type(max_age_days) is not int + or max_age_days < 0 + ): + raise ValueError("positive max_depth and nonnegative max_age_days required") + allocations = {key: number(value, nonnegative=True) for key, value in allocations.items()} + if sum(allocations.values(), Decimal(0)) > ONE: + raise ValueError("portfolio allocations exceed 100%") + visible = holdings_asof(frame, at) + groups = {key: value for key, value in visible.groupby("fund_id")} + leaves = [] + + def leaf(instrument, weight, currency, path, reason, holding_date=None, evidence=None): + if weight: + leaves.append( + { + "instrument_id": instrument, + "weight": weight, + "currency": currency, + "path": tuple(path), + "reason": reason, + "holding_date": holding_date, + "evidence_id": evidence, + "known": reason == "disclosed", + } + ) + + def visit(fund, weight, path): + reason = None + if fund in path: + reason = "cycle" + elif len(path) >= max_depth: + reason = "depth_limit" + elif fund not in groups: + reason = "missing_disclosure" + if reason: + leaf(f"UNKNOWN:{fund}", weight, None, [*path, fund], reason) + return + rows = groups[fund] + held = rows.iloc[0].holding_date + if (utc(at).tz_localize(None).normalize() - held).days > max_age_days: + leaf(f"UNKNOWN:{fund}", weight, None, [*path, fund], "stale", held) + return + for row in rows.itertuples(): + child_weight = weight * row.weight + if row.asset_type == "fund": + visit(row.instrument_id, child_weight, [*path, fund]) + else: + leaf( + row.instrument_id, + child_weight, + row.currency, + [*path, fund], + "disclosed", + held, + row.evidence_id, + ) + uncovered = weight * (ONE - sum(rows.weight, Decimal(0))) + leaf(f"UNKNOWN:{fund}", uncovered, None, [*path, fund], "undisclosed", held) + + for fund, weight in sorted(allocations.items()): + visit(fund, weight, []) + result = pd.DataFrame( + leaves, + columns=[ + "instrument_id", + "weight", + "currency", + "path", + "reason", + "holding_date", + "evidence_id", + "known", + ], + ) + if sum(result.weight, Decimal(0)) != sum(allocations.values(), Decimal(0)): + raise ArithmeticError("look-through failed portfolio weight conservation") + return result + + +def exposure_summary(leaves): + """Aggregate stable security identity without conflating currencies or unknowns.""" + if leaves.empty: + return {"known_weight": Decimal(0), "unknown_weight": Decimal(0), "exposures": []} + exposures = ( + leaves.loc[leaves.known].groupby(["instrument_id", "currency"], sort=True).weight.sum() + ) + return { + "known_weight": sum(leaves.loc[leaves.known, "weight"], Decimal(0)), + "unknown_weight": sum(leaves.loc[~leaves.known, "weight"], Decimal(0)), + "exposures": [ + {"instrument_id": key[0], "currency": key[1], "weight": value} + for key, value in exposures.items() + ], + } diff --git a/src/quant_data_kit/financial/lifecycle.py b/src/quant_data_kit/financial/lifecycle.py new file mode 100644 index 0000000..3d710be --- /dev/null +++ b/src/quant_data_kit/financial/lifecycle.py @@ -0,0 +1,139 @@ +"""Stable security identities and knowledge-time lifecycle/universe selection. + +Companion to instrument_master's evidence-bound rule catalog, not a replacement +for that catalog. Ticker changes never change instrument_id. Universe exits do +not imply delisting and holdings remain valuation/exit obligations. +""" + +from dataclasses import dataclass + +import pandas as pd + +from .common import table, utc + +KINDS = {"listing", "delisting", "symbol_change", "venue_change", "merger", "entry", "exit"} +COLUMNS = [ + "event_id", + "instrument_id", + "kind", + "effective_at", + "available_at", + "source", + "evidence_id", + "symbol", + "venue", + "universe_id", + "successor_id", +] + + +def validate_lifecycle(events: pd.DataFrame) -> pd.DataFrame: + out = table( + events, + COLUMNS, + text=("event_id", "instrument_id", "kind", "source", "evidence_id"), + timestamps=("effective_at", "available_at"), + unique=("event_id",), + ) + if not out.kind.isin(KINDS).all(): + raise ValueError("unsupported lifecycle event") + for row in out.itertuples(): + required = { + "listing": ("symbol", "venue"), + "symbol_change": ("symbol",), + "venue_change": ("venue",), + "entry": ("universe_id",), + "exit": ("universe_id",), + "merger": ("successor_id",), + }.get(row.kind, ()) + if any( + not isinstance(getattr(row, k), str) or not getattr(row, k).strip() for k in required + ): + raise ValueError(f"{row.kind} requires {required}") + if row.kind == "merger" and row.successor_id == row.instrument_id: + raise ValueError("merger successor must be a different stable identity") + keys = ["instrument_id", "kind", "effective_at", "available_at", "universe_id"] + if out.duplicated(keys).any(): + raise ValueError("ambiguous lifecycle version") + return out + + +def lifecycle_asof(events, effective_at, known_at) -> pd.DataFrame: + out = validate_lifecycle(events) + visible = out.loc[(out.effective_at <= utc(effective_at)) & (out.available_at <= utc(known_at))] + # A later correction supersedes the same economic event only after publication. + visible = visible.sort_values(["effective_at", "available_at", "event_id"]) + visible = visible.drop_duplicates( + ["instrument_id", "kind", "effective_at", "universe_id"], keep="last" + ) + for _, simultaneous in visible.groupby(["instrument_id", "effective_at"]): + kinds = set(simultaneous.kind) + if "listing" in kinds and kinds & {"delisting", "merger"}: + raise ValueError("conflicting simultaneous lifecycle states") + for _, membership in simultaneous.groupby("universe_id"): + if {"entry", "exit"} <= set(membership.kind): + raise ValueError("conflicting simultaneous membership states") + records = [] + for instrument, rows in visible.groupby("instrument_id", sort=True): + state = { + "instrument_id": instrument, + "listed": None, + "symbol": None, + "venue": None, + "successor_id": None, + "memberships": set(), + "evidence_ids": [], + } + for row in rows.itertuples(): + state["evidence_ids"].append(row.evidence_id) + if row.kind == "listing": + state.update(listed=True, symbol=row.symbol, venue=row.venue) + elif row.kind in {"delisting", "merger"}: + state["listed"] = False + if row.kind == "merger": + state["successor_id"] = row.successor_id + elif row.kind == "symbol_change": + state["symbol"] = row.symbol + elif row.kind == "venue_change": + state["venue"] = row.venue + elif row.kind == "entry": + state["memberships"].add(row.universe_id) + elif row.kind == "exit": + state["memberships"].discard(row.universe_id) + state["memberships"] = tuple(sorted(state["memberships"])) + state["evidence_ids"] = tuple(state["evidence_ids"]) + records.append(state) + result = pd.DataFrame( + records, + columns=[ + "instrument_id", + "listed", + "symbol", + "venue", + "successor_id", + "memberships", + "evidence_ids", + ], + ) + if result.loc[result.listed.eq(True)].duplicated(["symbol", "venue"]).any(): + raise ValueError("active ticker/venue maps to multiple stable identities") + return result + + +@dataclass(frozen=True) +class UniverseSelection: + eligible: frozenset[str] + retained_holdings: frozenset[str] + unknown_holdings: frozenset[str] + + +def select_universe(events, effective_at, known_at, *, universe_id=None, holdings=()): + states = lifecycle_asof(events, effective_at, known_at) + eligible = frozenset( + row.instrument_id + for row in states.itertuples() + if row.listed is True and (universe_id is None or universe_id in row.memberships) + ) + held = frozenset(holdings) + known = set(states.loc[states.listed.notna(), "instrument_id"]) + return UniverseSelection(eligible, held - eligible, held - known) diff --git a/src/quant_data_kit/financial/macro.py b/src/quant_data_kit/financial/macro.py new file mode 100644 index 0000000..8743383 --- /dev/null +++ b/src/quant_data_kit/financial/macro.py @@ -0,0 +1,84 @@ +"""Macro observations with vintage dates AND intraday release evidence.""" + +import pandas as pd + +from .common import number, table, utc + +COLUMNS = [ + "series_id", + "observation_date", + "vintage_date", + "released_at", + "available_at", + "value", + "unit", + "source", + "evidence_id", +] + + +def validate_macro(frame): + out = table( + frame, + COLUMNS, + text=("series_id", "unit", "source", "evidence_id"), + timestamps=("released_at", "available_at"), + dates=("observation_date", "vintage_date"), + unique=("series_id", "observation_date", "available_at"), + ) + if (out.released_at > out.available_at).any(): + raise ValueError("macro value cannot precede release") + release_dates = ( + out.released_at.dt.tz_convert("America/New_York").dt.tz_localize(None).dt.normalize() + ) + if (out.vintage_date > release_dates).any(): + raise ValueError("macro vintage cannot be visible before its publication date") + if (out.observation_date > out.released_at.dt.tz_localize(None).dt.normalize()).any(): + raise ValueError("macro observation cannot be in the future") + # One series keeps one unit; transformations must get new series identities. + if out.groupby("series_id").unit.nunique().gt(1).any(): + raise ValueError("macro unit drift requires a new series identity") + out["value"] = out.value.map(lambda x: None if pd.isna(x) or x == "." else number(x)) + return out + + +def macro_asof(frame, at, *, latest_observation=False): + out = validate_macro(frame) + out = out.loc[out.available_at <= utc(at)].sort_values( + ["series_id", "observation_date", "available_at"] + ) + out = out.drop_duplicates(["series_id", "observation_date"], keep="last") + if latest_observation: + out = out.groupby("series_id", sort=True).tail(1) + return out.reset_index(drop=True) + + +def from_alfred(observations, *, series_id, unit, release_times, source): + """Parse archived FRED JSON; release_times maps vintage DATE to evidenced UTC time. + + FRED realtime_start is date precision, not an 08:30 release timestamp. Missing + release evidence fails closed instead of making a revision visible at midnight. + API fetching/keys stay with the caller; missing '.' values remain missing. + """ + rows = [] + for i, obs in enumerate(observations): + vintage = obs["realtime_start"] + if vintage not in release_times: + raise ValueError(f"missing evidenced release timestamp for vintage {vintage}") + released = utc(release_times[vintage]) + if released.tz_convert("America/New_York").date().isoformat() != vintage: + raise ValueError("release timestamp and ALFRED vintage date disagree") + rows.append( + { + "series_id": series_id, + "observation_date": obs["date"], + "vintage_date": vintage, + "released_at": released, + "available_at": released, + "value": obs["value"], + "unit": unit, + "source": source, + "evidence_id": f"{series_id}:{vintage}:{i}", + } + ) + return validate_macro(pd.DataFrame(rows, columns=COLUMNS)) diff --git a/src/quant_data_kit/financial/reconciliation.py b/src/quant_data_kit/financial/reconciliation.py new file mode 100644 index 0000000..b4dca5a --- /dev/null +++ b/src/quant_data_kit/financial/reconciliation.py @@ -0,0 +1,207 @@ +"""Immutable source discrepancy decisions and explicit downstream invalidation.""" + +import hashlib +import json +from pathlib import Path + +from .common import number, table, utc + +COLUMNS = [ + "observation_id", + "instrument_id", + "field", + "effective_at", + "available_at", + "value", + "unit", + "currency", + "basis", + "source", + "evidence_id", +] +UNITS = { + "currency": ("currency", 1), + "thousand_currency": ("currency", 1000), + "shares": ("shares", 1), + "ratio": ("ratio", 1), + "percent": ("ratio", "0.01"), +} + + +def canonical(value): + return json.dumps( + value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str + ).encode("utf-8") + + +def digest(value): + return hashlib.sha256(canonical(value)).hexdigest() + + +def validate_observations(frame): + out = table( + frame, + COLUMNS, + text=tuple(c for c in COLUMNS if c not in {"value", "effective_at", "available_at"}), + timestamps=("effective_at", "available_at"), + unique=("observation_id",), + ) + if not out.unit.isin(UNITS).all(): + raise ValueError("unsupported unit: normalize with explicit evidence before comparison") + out["value"] = out.value.map(number) + out["normalized_value"] = [row.value * number(UNITS[row.unit][1]) for row in out.itertuples()] + out["normalized_unit"] = out.unit.map(lambda x: UNITS[x][0]) + return out + + +def discrepancies(frame, *, at, tolerance="0"): + out = validate_observations(frame) + out = out.loc[out.available_at <= utc(at)] + tolerance = number(tolerance, nonnegative=True) + cases = [] + for key, rows in out.groupby(["instrument_id", "field", "effective_at"], sort=True): + rows = rows.sort_values(["available_at", "observation_id"]).groupby("source").tail(1) + if len(rows) < 2: + continue + semantics = rows[["normalized_unit", "currency", "basis"]].drop_duplicates() + reason = "incomparable_semantics" if len(semantics) > 1 else "value_disagreement" + if ( + len(semantics) == 1 + and max(rows.normalized_value) - min(rows.normalized_value) <= tolerance + ): + continue + observations = rows.sort_values("observation_id")[COLUMNS].to_dict("records") + case = { + "instrument_id": key[0], + "field": key[1], + "effective_at": str(key[2]), + "reason": reason, + "observations": observations, + } + case["case_id"] = digest(case) + cases.append(case) + return cases + + +def impacted_nodes(changed_observation_ids, dependencies): + """dependencies: artifact/node -> direct input ids. Cycles terminate safely.""" + affected = set(changed_observation_ids) + while True: + extra = {node for node, inputs in dependencies.items() if set(inputs) & affected} + if extra <= affected: + break + affected |= extra + return sorted(affected - set(changed_observation_ids)) + + +def adjudicate( + case, + *, + selected_observation_id, + resolved_at, + reviewer, + rationale, + evidence_uri, + prior_snapshot, + dependencies, +): + original = {k: v for k, v in case.items() if k != "case_id"} + if digest(original) != case.get("case_id"): + raise ValueError("case content hash mismatch") + if not all( + isinstance(x, str) and x.strip() + for x in (reviewer, rationale, evidence_uri, prior_snapshot) + ): + raise ValueError("adjudication requires reviewer, rationale, evidence and parent snapshot") + at = utc(resolved_at) + ids = [x["observation_id"] for x in case["observations"]] + if selected_observation_id not in ids: + raise ValueError("decision must select an evidenced observation, not invent a replacement") + if any(utc(x["available_at"]) > at for x in case["observations"]): + raise ValueError("cannot adjudicate observations before they were known") + decision = { + "schema": "qdk.source-adjudication/1", + "case": case, + "selected_observation_id": selected_observation_id, + "resolved_at": str(at), + "reviewer": reviewer, + "rationale": rationale, + "evidence_uri": evidence_uri, + "prior_snapshot": prior_snapshot, + "impacted": impacted_nodes(ids, dependencies), + } + decision["snapshot_id"] = digest(decision) + return decision + + +def publish_decision(decision, destination): + """Exclusive create; never mutate the parent snapshot or existing resolution.""" + expected = digest({k: v for k, v in decision.items() if k != "snapshot_id"}) + if decision.get("snapshot_id") != expected: + raise ValueError("decision hash mismatch") + destination = Path(destination) + with destination.open("xb") as stream: + stream.write(canonical(decision)) + return destination + + +def freeze_snapshot(records, *, parent_snapshot=None, decision_id=None): + """A governed observation snapshot: exactly one chosen fact per economic key.""" + import pandas as pd + + frame = validate_observations(pd.DataFrame(records)) + if frame.duplicated(["instrument_id", "field", "effective_at"]).any(): + raise ValueError("snapshot must contain one selected observation per economic key") + snapshot = { + "schema": "qdk.resolved-observations/1", + "parent_snapshot": parent_snapshot, + "decision_id": decision_id, + "records": frame[COLUMNS].sort_values("observation_id").to_dict("records"), + } + snapshot["snapshot_id"] = digest(snapshot) + return snapshot + + +def apply_decision_snapshot(parent, decision, destination): + """Create a new data snapshot, retaining the immutable parent and knowledge boundary. + + Rerunning downstream artifacts is a separate explicit operation, never implied + by publishing a source adjudication. Revised facts become known at resolution. + """ + for artifact in (parent, decision): + if digest({k: v for k, v in artifact.items() if k != "snapshot_id"}) != artifact.get( + "snapshot_id" + ): + raise ValueError("snapshot/decision hash mismatch") + if parent.get("schema") != "qdk.resolved-observations/1": + raise ValueError("unsupported parent snapshot") + if decision["prior_snapshot"] != parent["snapshot_id"]: + raise ValueError("decision targets a different parent snapshot") + case = decision["case"] + if digest({k: v for k, v in case.items() if k != "case_id"}) != case["case_id"]: + raise ValueError("case hash mismatch") + selected = dict( + next( + x + for x in case["observations"] + if x["observation_id"] == decision["selected_observation_id"] + ) + ) + key = lambda row: (row["instrument_id"], row["field"], utc(row["effective_at"])) + matches = [row for row in parent["records"] if key(row) == key(selected)] + if len(matches) != 1 or matches[0]["observation_id"] not in { + x["observation_id"] for x in case["observations"] + }: + raise ValueError("parent observation does not match the adjudicated case") + resolved = utc(decision["resolved_at"]) + if resolved < max(utc(x["available_at"]) for x in case["observations"]): + raise ValueError("resolution precedes source knowledge") + selected["available_at"] = str(resolved) + selected["observation_id"] += ":resolution:" + decision["snapshot_id"] + selected["evidence_id"] += ":decision:" + decision["snapshot_id"] + records = [row for row in parent["records"] if key(row) != key(selected)] + [selected] + result = freeze_snapshot( + records, parent_snapshot=parent["snapshot_id"], decision_id=decision["snapshot_id"] + ) + publish_decision(result, destination) + return result diff --git a/src/quant_data_kit/financial/returns.py b/src/quant_data_kit/financial/returns.py new file mode 100644 index 0000000..85e0387 --- /dev/null +++ b/src/quant_data_kit/financial/returns.py @@ -0,0 +1,65 @@ +"""Simple shareholder-return signal index, separate from spendable cash accounting.""" + +import numpy as np +import pandas as pd + +from .actions import ActionTerms +from .common import utc + + +def total_return_panel(prices, actions, *, timezone, price_basis="raw"): + """Raw date/symbol/close panel plus explicit split and dividend entitlements. + + Index assumes theoretical reinvestment at ex-day close, NOT cash availability. + Execution must separately carry dividend receivables until payment. Complex + conversions or simultaneous share/cash terms require an account-based return, + and are rejected rather than silently omitted. + """ + if price_basis != "raw": + raise ValueError("explicit actions require raw quotes; do not double-adjust returns") + for field in ("adjustment", "price_basis"): + if field in prices and not prices[field].isin(["none", "raw"]).all(): + raise ValueError("explicit actions require raw quotes; adjusted input detected") + if not np.isfinite(pd.to_numeric(prices.close, errors="raise")).all(): + raise ValueError("finite raw prices required") + terms = [x if isinstance(x, ActionTerms) else ActionTerms(**x) for x in actions] + economic = [x for x in terms if x.kind != "dividend_payment"] + if any(x.kind not in {"split", "dividend_entitlement"} for x in economic): + raise ValueError("complex actions need account-based total return") + indexed = {} + for action in economic: + effective = utc(action.effective_at) + if utc(action.available_at) > effective: + raise ValueError("late-known corporate action cannot backfill historical signal") + key = (action.instrument_id, effective.tz_convert(timezone).date()) + if key in indexed: + raise ValueError("simultaneous actions need an explicit combined entitlement basis") + indexed[key] = action + pieces = [] + for symbol, group in prices.groupby("symbol", sort=False): + group = group.sort_values("date").copy() + if group.date.duplicated().any() or (group.close <= 0).any(): + raise ValueError("unique dates and positive raw prices required") + dates = set(pd.to_datetime(group.date).dt.date) + if any( + sym == symbol and min(dates) <= date <= max(dates) and date not in dates + for sym, date in indexed + ): + raise ValueError("corporate action date has no raw quote") + values = [] + previous, index = None, 100.0 + for row in group.itertuples(): + action = indexed.get((symbol, pd.Timestamp(row.date).date())) + ratio, cash = 1.0, 0.0 + if action: + if action.kind == "split": + ratio = float(action.ratio) + else: + cash = float(action.cash_per_unit) + if previous is not None: + index *= (row.close * ratio + cash) / previous + values.append(index) + previous = row.close + group["return_close"] = values + pieces.append(group) + return pd.concat(pieces, ignore_index=True) if pieces else prices.assign(return_close=[]) diff --git a/src/quant_data_kit/financial/status.py b/src/quant_data_kit/financial/status.py new file mode 100644 index 0000000..57cbee5 --- /dev/null +++ b/src/quant_data_kit/financial/status.py @@ -0,0 +1,69 @@ +"""Side-specific, expiring status evidence; absence is UNKNOWN, not permission.""" + +from dataclasses import dataclass + +from .common import table, utc + +COLUMNS = [ + "instrument_id", + "effective_from", + "effective_to", + "available_at", + "buy_status", + "sell_status", + "reason", + "source", + "evidence_id", +] +STATES = {"tradable", "blocked", "unknown"} + + +def validate_status(frame): + out = table( + frame, + COLUMNS, + text=("instrument_id", "reason", "source", "evidence_id"), + timestamps=("effective_from", "effective_to", "available_at"), + unique=("instrument_id", "effective_from", "available_at", "source"), + ) + if (out.effective_to <= out.effective_from).any(): + raise ValueError("status evidence requires a bounded positive validity interval") + if not out.buy_status.isin(STATES).all() or not out.sell_status.isin(STATES).all(): + raise ValueError("status must be tradable, blocked or unknown") + # A statement that no restriction was found is not positive trading evidence. + if ( + out.reason.eq("no_restriction") + & (out.buy_status.eq("tradable") | out.sell_status.eq("tradable")) + ).any(): + raise ValueError("no_restriction is not affirmative tradability evidence") + return out + + +@dataclass(frozen=True) +class TradingPermission: + buy: str + sell: str + reason: str + evidence_ids: tuple[str, ...] = () + + +def permission_asof(frame, instrument_id, effective_at, known_at): + out = validate_status(frame) + at, known = utc(effective_at), utc(known_at) + rows = out.loc[ + out.instrument_id.eq(instrument_id) + & (out.effective_from <= at) + & (at < out.effective_to) + & (out.available_at <= known) + ] + if rows.empty: + return TradingPermission("unknown", "unknown", "missing_or_expired_evidence") + # Each source can correct itself; disagreement across sources remains unknown. + rows = rows.sort_values(["available_at", "effective_from"]).groupby("source").tail(1) + evidence = tuple(sorted(rows.evidence_id)) + if rows[["buy_status", "sell_status"]].drop_duplicates().shape[0] != 1: + return TradingPermission("unknown", "unknown", "conflicting_sources", evidence) + row = rows.iloc[0] + return TradingPermission( + row.buy_status, row.sell_status, "|".join(sorted(set(rows.reason))), evidence + ) diff --git a/src/quant_data_kit/financial/units.py b/src/quant_data_kit/financial/units.py new file mode 100644 index 0000000..80dc168 --- /dev/null +++ b/src/quant_data_kit/financial/units.py @@ -0,0 +1,49 @@ +"""Normalize vendor trading units without combining adjusted and raw shares.""" + +import numpy as np +import pandas as pd + + +def normalize_trading_units( + frame: pd.DataFrame, + *, + volume_unit: str, + amount_unit: str, + currency: str, + share_basis: str, + lot_size: int | None = None, +) -> pd.DataFrame: + """Produce shares and currency units; a lot multiplier must be explicitly supplied. + + ``free_float_shares`` must already use the same raw share basis as volume. + Split-normalized historical volume must not be divided by raw historical float. + The caller supplies historical/PIT float, never a current float backfill. + """ + if share_basis != "raw" or volume_unit not in {"shares", "lots"}: + raise ValueError("explicit raw shares/lots required") + if amount_unit not in {"currency", "thousand_currency", "ten_thousand_currency"}: + raise ValueError("unsupported traded amount unit") + if not isinstance(currency, str) or len(currency) != 3 or not currency.isupper(): + raise ValueError("explicit three-letter currency required") + if volume_unit == "lots" and (type(lot_size) is not int or lot_size <= 0): + raise ValueError("lot volume requires an explicit positive lot_size") + if "volume" not in frame or "amount" not in frame: + raise ValueError("volume and actual traded amount are required") + out = frame.copy() + for col in ("volume", "amount", "free_float_shares"): + if col not in out: + continue + out[col] = pd.to_numeric(out[col], errors="raise") + values = out[col].dropna() + if not np.isfinite(values).all() or (values < 0).any(): + raise ValueError(f"invalid {col}") + if col == "free_float_shares" and values.eq(0).any(): + raise ValueError("known free float must be positive") + out["volume"] *= lot_size if volume_unit == "lots" else 1 + out["amount"] *= {"currency": 1, "thousand_currency": 1000, "ten_thousand_currency": 10000}[ + amount_unit + ] + out["volume_unit"], out["amount_unit"] = "shares", "currency" + out["share_basis"], out["currency"] = "raw", currency + out.attrs["financial_units_version"] = "raw-trading-units/1" + return out diff --git a/src/quant_data_kit/us_research/sec.py b/src/quant_data_kit/us_research/sec.py index d10ae37..9223087 100644 --- a/src/quant_data_kit/us_research/sec.py +++ b/src/quant_data_kit/us_research/sec.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools import json import re import time @@ -153,6 +154,8 @@ def fact_table( "form": filing.form, "accepted_at": filing.accepted_at, "available_at": filing.available_at, + "fiscal_year": fact.get("fy"), + "fiscal_period": fact.get("fp"), } ) columns = [ @@ -166,6 +169,8 @@ def fact_table( "form", "accepted_at", "available_at", + "fiscal_year", + "fiscal_period", ] result = pd.DataFrame(rows, columns=columns).drop_duplicates() keys = ["instrument_id", "concept", "unit", "period_start", "period_end", "accession"] @@ -180,9 +185,25 @@ def fact_table( def facts_asof(facts: pd.DataFrame, at) -> pd.DataFrame: facts = facts.copy() - facts["available_at"] = facts.available_at.map(lambda value: utc(value, "fact available_at")) + facts["value"] = pd.to_numeric(facts.value, errors="raise") + if not np.isfinite(facts.value).all(): + raise ValueError("nonfinite SEC fact") + facts["period_start"] = pd.to_datetime(facts.period_start) + facts["period_end"] = pd.to_datetime(facts.period_end) + facts["available_at"] = pd.to_datetime( + facts.available_at.map(lambda value: utc(value, "fact available_at")), utc=True + ) + if "accepted_at" in facts: + accepted = facts.accepted_at.map(lambda value: utc(value, "SEC accepted_at")) + if (facts.available_at < accepted).any(): + raise ValueError("SEC fact cannot be known before acceptance") + if (facts.period_end > facts.available_at.dt.tz_localize(None)).any(): + raise ValueError("SEC fact period cannot be in the future") visible = facts.loc[facts.available_at <= utc(at)].copy() keys = ["instrument_id", "concept", "unit", "period_start", "period_end"] + versions = visible.groupby([*keys, "available_at"], dropna=False).value.nunique() + if versions.gt(1).any(): + raise ValueError("ambiguous same-time SEC facts") return visible.sort_values(["available_at", "accession"]).drop_duplicates(keys, keep="last") @@ -237,3 +258,165 @@ def annual_quality(facts: pd.DataFrame, at, *, max_age_days: int = 550) -> pd.Da "accessions", ], ) + + +def quarterly_facts(facts: pd.DataFrame, at) -> pd.DataFrame: + """Independent fiscal quarters, not calendar quarters or blindly divided YTD. + + 53-week fiscal years are accepted through duration/contiguity checks. Fiscal + year labels in companyfacts describe the FILING, so period boundaries govern + derivation. Direct disclosed quarters take precedence over differences. + Cross-accession differences remain flagged: restated YTD and old quarters + are not automatically certified as comparable. + """ + visible = facts_asof(facts, at).copy() + visible["period_start"] = pd.to_datetime(visible.period_start) + visible["period_end"] = pd.to_datetime(visible.period_end) + rows = [] + for _, group in visible.groupby(["instrument_id", "concept", "unit"]): + group = group.loc[group.period_start.notna()].sort_values("period_end") + direct = {} + for row in group.to_dict("records"): + days = (row["period_end"] - row["period_start"]).days + 1 + if 60 <= days <= 110: + key = (row["period_start"], row["period_end"]) + direct[key] = dict( + row, + method="reported_quarter", + accessions=row["accession"], + vintage_consistent=True, + ) + derived = {} + for row in group.to_dict("records"): + duration = (row["period_end"] - row["period_start"]).days + 1 + if not 150 <= duration <= 380: + continue + prior = group.loc[ + (group.period_start == row["period_start"]) & (group.period_end < row["period_end"]) + ] + if prior.empty: + continue + previous = prior.iloc[-1] + start = previous.period_end + pd.Timedelta(days=1) + days = (row["period_end"] - start).days + 1 + if not 60 <= days <= 110: + continue + key = (start, row["period_end"]) + if key in direct: + continue + candidate = dict( + row, + period_start=start, + value=row["value"] - previous.value, + method="cumulative_difference", + available_at=max(row["available_at"], previous.available_at), + accessions="|".join(sorted({row["accession"], previous.accession})), + vintage_consistent=row["accession"] == previous.accession, + ) + if key in derived and derived[key]["value"] != candidate["value"]: + raise ValueError("conflicting cumulative derivations; reconcile SEC facts") + derived[key] = candidate + rows.extend([*direct.values(), *derived.values()]) + columns = [*facts.columns, "method", "accessions", "vintage_consistent"] + return pd.DataFrame(rows, columns=columns).sort_values( + ["instrument_id", "concept", "unit", "period_end"] + ) + + +def ttm_facts(facts: pd.DataFrame, at, *, allow_mixed_vintages=False, max_age_days=200): + """Four contiguous independent quarters, with unit and vintage lineage intact.""" + if type(allow_mixed_vintages) is not bool or type(max_age_days) is not int or max_age_days < 0: + raise ValueError("explicit vintage policy and nonnegative max_age_days required") + quarters = quarterly_facts(facts, at) + result = [] + for key, rows in quarters.groupby(["instrument_id", "concept", "unit"]): + last = rows.sort_values("period_end").tail(4) + if len(last) != 4 or (not allow_mixed_vintages and not last.vintage_consistent.all()): + continue + records = last.to_dict("records") + if any( + b["period_start"] != a["period_end"] + pd.Timedelta(days=1) + for a, b in itertools.pairwise(records) + ): + continue + span = (records[-1]["period_end"] - records[0]["period_start"]).days + 1 + if not 330 <= span <= 380: + continue + if (utc(at).tz_localize(None) - records[-1]["period_end"]).days > max_age_days: + continue + result.append( + { + "instrument_id": key[0], + "concept": key[1], + "unit": key[2], + "period_start": records[0]["period_start"], + "period_end": records[-1]["period_end"], + "value": sum(last.value), + "available_at": max(last.available_at), + "accessions": "|".join(sorted(set("|".join(last.accessions).split("|")))), + "vintage_consistent": bool(last.vintage_consistent.all()), + } + ) + return pd.DataFrame( + result, + columns=[ + "instrument_id", + "concept", + "unit", + "period_start", + "period_end", + "value", + "available_at", + "accessions", + "vintage_consistent", + ], + ) + + +def ttm_quality(facts, at, *, allow_mixed_vintages=False): + flows = ttm_facts(facts, at, allow_mixed_vintages=allow_mixed_vintages) + visible = facts_asof(facts, at) + result = [] + for instrument, rows in flows.groupby("instrument_id"): + income = rows.loc[rows.concept.eq("NetIncomeLoss") & rows.unit.eq("USD")] + cash = rows.loc[ + rows.concept.eq("NetCashProvidedByUsedInOperatingActivities") & rows.unit.eq("USD") + ] + if len(income) != 1 or len(cash) != 1: + continue + i, c = income.iloc[0], cash.iloc[0] + if i.period_start != c.period_start or i.period_end != c.period_end: + continue + assets = visible.loc[ + visible.instrument_id.eq(instrument) + & visible.concept.eq("Assets") + & visible.unit.eq("USD") + & visible.period_start.isna() + & pd.to_datetime(visible.period_end).eq(i.period_end) + ] + if len(assets) != 1 or assets.iloc[0].value <= 0: + continue + a = assets.iloc[0] + result.append( + { + "instrument_id": instrument, + "period_end": i.period_end, + "roa_ttm": i.value / a.value, + "accruals_ttm": (i.value - c.value) / a.value, + "available_at": max(i.available_at, c.available_at, a.available_at), + "accessions": "|".join( + sorted(set((i.accessions + "|" + c.accessions + "|" + a.accession).split("|"))) + ), + } + ) + return pd.DataFrame( + result, + columns=[ + "instrument_id", + "period_end", + "roa_ttm", + "accruals_ttm", + "available_at", + "accessions", + ], + ) diff --git a/tests/fixtures/financial/apple_cash_flow_extract.json b/tests/fixtures/financial/apple_cash_flow_extract.json new file mode 100644 index 0000000..0b71fce --- /dev/null +++ b/tests/fixtures/financial/apple_cash_flow_extract.json @@ -0,0 +1,29 @@ +{ + "classification": "manual_public_fact_extract_not_a_downloaded_snapshot", + "issuer": "Apple Inc.", + "concept": "NetCashProvidedByUsedInOperatingActivities", + "source_unit": "USD millions", + "period_start": "2023-10-01", + "locator": "Condensed Consolidated Statements of Cash Flows, cash generated by operating activities", + "acceptance_timezone": "America/New_York (EDGAR filing index local time, explicitly converted below)", + "facts": [ + { + "period_end": "2023-12-30", "value_millions": 39895, + "accession": "0000320193-24-000006", "accepted_at": "2024-02-01T18:03:38-05:00", + "statement_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/aapl-20231230.htm", + "acceptance_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/0000320193-24-000006-index.htm" + }, + { + "period_end": "2024-03-30", "value_millions": 62585, + "accession": "0000320193-24-000069", "accepted_at": "2024-05-02T18:04:25-04:00", + "statement_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/aapl-20240330.htm", + "acceptance_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000069/0000320193-24-000069-index.htm" + }, + { + "period_end": "2024-06-29", "value_millions": 91443, + "accession": "0000320193-24-000081", "accepted_at": "2024-08-01T18:03:34-04:00", + "statement_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000081/aapl-20240629.htm", + "acceptance_url": "https://www.sec.gov/Archives/edgar/data/320193/000032019324000081/0000320193-24-000081-index.htm" + } + ] +} diff --git a/tests/test_financial_evidence.py b/tests/test_financial_evidence.py new file mode 100644 index 0000000..0d992a4 --- /dev/null +++ b/tests/test_financial_evidence.py @@ -0,0 +1,122 @@ +import json +from pathlib import Path + +import pandas as pd +import pytest + +from quant_data_kit.financial import reconciliation as r +from quant_data_kit.financial.returns import total_return_panel +from quant_data_kit.us_research.sec import fact_table, filing_table, quarterly_facts + + +def test_real_apple_cumulative_cash_flow_and_acceptance_lag(): + data = json.loads( + (Path(__file__).parent / "fixtures/financial/apple_cash_flow_extract.json").read_text() + ) + rows = data["facts"] + filings = filing_table( + [ + { + "accessionNumber": [x["accession"] for x in rows], + "acceptanceDateTime": [x["accepted_at"] for x in rows], + "form": ["10-Q"] * 3, + "filingDate": [x["accepted_at"][:10] for x in rows], + } + ], + "CIK:0000320193", + ) + source = { + "facts": { + "us-gaap": { + data["concept"]: { + "units": { + "USD": [ + { + "start": data["period_start"], + "end": x["period_end"], + "val": x["value_millions"] * 1_000_000, + "accn": x["accession"], + } + for x in rows + ] + } + } + } + } + } + facts, _ = fact_table(source, filings) + boundary = pd.Timestamp(rows[2]["accepted_at"]) + pd.Timedelta(minutes=5) + assert len(quarterly_facts(facts, boundary - pd.Timedelta(seconds=1))) == 2 + quarters = quarterly_facts(facts, boundary) + assert quarters.value.tolist() == [39_895e6, 22_690e6, 28_858e6] + assert quarters.vintage_consistent.tolist() == [True, False, False] + # Fiscal Q2 starts in December: calendar-quarter bucketing would be wrong. + assert quarters.iloc[1].period_start == pd.Timestamp("2023-12-31") + + +def test_resolution_materializes_new_snapshot_without_retroactive_knowledge(tmp_path): + base = { + "instrument_id": "A", + "field": "close", + "effective_at": "2024-01-01T00:00:00Z", + "available_at": "2024-01-01T00:00:00Z", + "currency": "USD", + "basis": "raw", + "unit": "currency", + "evidence_id": "synthetic", + } + records = [ + dict(base, observation_id="a", source="exchange", value="10"), + dict(base, observation_id="b", source="vendor", value="20"), + ] + parent = r.freeze_snapshot([records[1]]) + old_bytes = r.canonical(parent) + case = r.discrepancies(pd.DataFrame(records), at="2024-02-01T00:00:00Z")[0] + decision = r.adjudicate( + case, + selected_observation_id="a", + resolved_at="2024-02-01T00:00:00Z", + reviewer="test", + rationale="exchange evidence", + evidence_uri="synthetic:test", + prior_snapshot=parent["snapshot_id"], + dependencies={"factor": ["b"]}, + ) + path = tmp_path / "corrected.json" + result = r.apply_decision_snapshot(parent, decision, path) + assert r.canonical(parent) == old_bytes + assert result["records"][0]["value"] == 10 + assert result["records"][0]["available_at"] == pd.Timestamp("2024-02-01T00:00:00Z") + assert result["parent_snapshot"] == parent["snapshot_id"] + with pytest.raises(FileExistsError): + r.apply_decision_snapshot(parent, decision, path) + + +def test_split_total_return_is_not_raw_price_return_and_refuses_adjusted_quotes(): + # Apple's 2020 4-for-1 split terms; prices here are synthetic, not observed quotes. + # Source: https://investor.apple.com/dividend-history/ + bars = pd.DataFrame( + { + "date": ["2020-08-28", "2020-08-31"], + "symbol": ["AAPL"] * 2, + "close": [400, 100], + "adjustment": ["none"] * 2, + } + ) + action = { + "event_id": "apple-split", + "instrument_id": "AAPL", + "kind": "split", + "ratio": "4", + "effective_at": "2020-08-31T13:30:00Z", + "available_at": "2020-07-31T23:59:59Z", + "currency": "USD", + "source": "Apple IR", + "evidence_id": "split-2020", + } + result = total_return_panel(bars, [action], timezone="America/New_York") + assert result.return_close.iloc[-1] == result.return_close.iloc[0] + with pytest.raises(ValueError): + total_return_panel( + bars.assign(adjustment="total_return"), [action], timezone="America/New_York" + ) diff --git a/tests/test_financial_foundations.py b/tests/test_financial_foundations.py new file mode 100644 index 0000000..e14668e --- /dev/null +++ b/tests/test_financial_foundations.py @@ -0,0 +1,305 @@ +from dataclasses import replace +from decimal import Decimal + +import pandas as pd +import pytest + +from quant_data_kit.financial import calendars, holdings, lifecycle, macro, reconciliation, status +from quant_data_kit.financial.actions import ActionTerms +from quant_data_kit.financial.units import normalize_trading_units +from quant_data_kit.us_research.sec import quarterly_facts, ttm_facts + +T0 = "2024-01-01T00:00:00Z" +T1 = "2024-02-01T00:00:00Z" +T2 = "2024-03-01T00:00:00Z" + + +def lifecycle_row(kind, effective=T0, known=T0, **kw): + return dict( + event_id=f"{kind}:{effective}", + instrument_id="stable-A", + kind=kind, + effective_at=effective, + available_at=known, + source="synthetic", + evidence_id="fixture", + symbol="OLD", + venue="TEST", + universe_id="U", + successor_id="stable-B", + **kw, + ) + + +def test_lifecycle_future_correction_exit_and_delisting_keep_obligations(): + events = pd.DataFrame( + [ + lifecycle_row("listing"), + lifecycle_row("entry"), + lifecycle_row("exit", T1), + lifecycle_row("delisting", T2), + ] + ) + before = lifecycle.select_universe(events, T0, T0, universe_id="U") + assert before.eligible == {"stable-A"} + after = lifecycle.select_universe( + events, T1, T1, universe_id="U", holdings=["stable-A", "missing"] + ) + assert not after.eligible and after.retained_holdings == {"stable-A", "missing"} + assert after.unknown_holdings == {"missing"} + assert lifecycle.lifecycle_asof(events, T1, T1).iloc[0].listed + assert not lifecycle.lifecycle_asof(events, T2, T2).iloc[0].listed + pd.testing.assert_frame_equal( + lifecycle.lifecycle_asof(events, T0, T0), lifecycle.lifecycle_asof(events.iloc[:2], T0, T0) + ) + + +def test_lifecycle_symbol_change_has_stable_id_and_delayed_knowledge(): + change = lifecycle_row("symbol_change", T1, T2) + change["symbol"] = "NEW" + frame = pd.DataFrame([lifecycle_row("listing"), change]) + assert lifecycle.lifecycle_asof(frame, T1, T1).iloc[0].symbol == "OLD" + latest = lifecycle.lifecycle_asof(frame, T2, T2).iloc[0] + assert latest.symbol == "NEW" and latest.instrument_id == "stable-A" + + +def status_frame(): + return pd.DataFrame( + [ + { + "instrument_id": "A", + "effective_from": T0, + "effective_to": T2, + "available_at": T0, + "buy_status": "blocked", + "sell_status": "tradable", + "reason": "limit_up", + "source": "exchange", + "evidence_id": "fixture", + } + ] + ) + + +def test_status_expiry_side_specific_unknown_and_source_conflict(): + frame = status_frame() + assert status.permission_asof(frame, "A", T1, T1).sell == "tradable" + assert status.permission_asof(frame, "A", T2, T2).buy == "unknown" + assert status.permission_asof(frame, "absent", T1, T1).buy == "unknown" + conflict = frame.iloc[0].to_dict() + conflict.update(source="vendor", buy_status="tradable") + result = status.permission_asof(pd.concat([frame, pd.DataFrame([conflict])]), "A", T1, T1) + assert result.reason == "conflicting_sources" and result.sell == "unknown" + with pytest.raises(ValueError): + status.validate_status(frame.assign(reason="no_restriction")) + + +def calendar(purpose="settlement", days=("2024-01-02", "2024-01-04", "2024-01-05")): + return calendars.PurposeCalendar( + "TEST", purpose, "v1", T0, "2024-01-01", "2024-01-10", days, "synthetic", "synthetic" + ) + + +def test_purpose_calendar_holiday_coverage_publication_and_revision(): + cal = calendar() + assert cal.advance("2024-01-02", 1, at=T1).date().isoformat() == "2024-01-04" + with pytest.raises(ValueError, match="purpose"): + cal.advance("2024-01-02", 1, at=T1, purpose="trading") + with pytest.raises(ValueError, match="coverage"): + cal.advance("2024-01-05", 1, at=T1) + with pytest.raises(ValueError, match="known"): + cal.advance("2024-01-02", 1, at="2023-12-31T00:00:00Z") + revised = replace(cal, version="v2", available_at=T1, open_days=("2024-01-02", "2024-01-05")) + book = calendars.CalendarBook([cal, revised]) + assert book.asof("TEST", "settlement", T0, "2024-01-02").version == "v1" + assert book.asof("TEST", "settlement", T2, "2024-01-02").version == "v2" + + +def test_macro_release_boundary_missing_revision_and_unit_drift(): + payload = [ + {"date": "2023-12-01", "realtime_start": "2024-01-05", "value": "2.1"}, + {"date": "2023-12-01", "realtime_start": "2024-02-02", "value": "2.3"}, + ] + frame = macro.from_alfred( + payload, + series_id="TEST", + unit="percent", + source="synthetic", + release_times={"2024-01-05": "2024-01-05T13:30:00Z", "2024-02-02": "2024-02-02T13:30:00Z"}, + ) + assert macro.macro_asof(frame, "2024-01-05T13:29:59Z").empty + assert macro.macro_asof(frame, "2024-01-05T13:30:00Z").iloc[0].value == Decimal("2.1") + assert macro.macro_asof(frame, T2).iloc[0].value == Decimal("2.3") + with pytest.raises(ValueError, match="release timestamp"): + macro.from_alfred( + payload, series_id="TEST", unit="percent", source="synthetic", release_times={} + ) + bad = frame.copy() + bad.loc[1, "unit"] = "fraction" + with pytest.raises(ValueError, match="unit drift"): + macro.validate_macro(bad) + + +def holding(fund, instrument, weight, kind="security", **kw): + row = { + "disclosure_id": f"{fund}:1", + "fund_id": fund, + "holding_date": "2023-12-31", + "available_at": T0, + "instrument_id": instrument, + "asset_type": kind, + "currency": "CNY", + "weight": weight, + "source": "synthetic", + "evidence_id": f"{fund}:{instrument}", + } + return {**row, **kw} + + +def test_lookthrough_nested_overlap_missing_cash_currency_and_conservation(): + frame = pd.DataFrame( + [ + holding("F1", "F2", ".5", "fund"), + holding("F1", "A", ".3"), + holding("F2", "A", ".5"), + holding("F2", "CASH:USD", ".5", "cash", currency="USD"), + ] + ) + leaves = holdings.look_through(frame, {"F1": 1}, T1) + summary = holdings.exposure_summary(leaves) + assert summary["known_weight"] == Decimal(".8") + assert summary["unknown_weight"] == Decimal(".2") + assert next(x for x in summary["exposures"] if x["instrument_id"] == "A")["weight"] == Decimal( + ".55" + ) + assert sum(leaves.weight) == 1 + assert ( + holdings.exposure_summary(holdings.look_through(frame, {"F1": 1}, T2, max_age_days=1))[ + "unknown_weight" + ] + == 1 + ) + + +def test_holdings_cycle_depth_and_future_disclosure_not_silently_dropped(): + frame = pd.DataFrame([holding("F1", "F2", "1", "fund"), holding("F2", "F1", "1", "fund")]) + assert holdings.look_through(frame, {"F1": 1}, T1).iloc[0].reason == "cycle" + assert holdings.look_through(frame, {"F1": 1}, T1, max_depth=1).iloc[0].reason == "depth_limit" + future = frame.assign(available_at=T2) + assert holdings.look_through(future, {"F1": 1}, T1).iloc[0].reason == "missing_disclosure" + with pytest.raises(ValueError): + holdings.validate_holdings(frame.assign(weight="1.1")) + + +def test_reconciliation_normalizes_units_preserves_snapshot_and_propagates_impact(tmp_path): + base = { + "instrument_id": "A", + "field": "amount", + "effective_at": T0, + "available_at": T0, + "currency": "CNY", + "basis": "raw", + "evidence_id": "fixture", + } + frame = pd.DataFrame( + [ + dict(base, observation_id="a", source="exchange", value="1000", unit="currency"), + dict(base, observation_id="b", source="vendor", value="1", unit="thousand_currency"), + ] + ) + assert not reconciliation.discrepancies(frame, at=T1) + frame.loc[1, "value"] = "2" + case = reconciliation.discrepancies(frame, at=T1)[0] + decision = reconciliation.adjudicate( + case, + selected_observation_id="a", + resolved_at=T1, + reviewer="test", + rationale="exchange evidence", + evidence_uri="synthetic:document", + prior_snapshot="old", + dependencies={"factor": ["b"], "orders": ["factor"], "report": ["orders"]}, + ) + assert decision["impacted"] == ["factor", "orders", "report"] + path = reconciliation.publish_decision(decision, tmp_path / "new.json") + with pytest.raises(FileExistsError): + reconciliation.publish_decision(decision, path) + with pytest.raises(ValueError, match="hash"): + reconciliation.publish_decision({**decision, "reviewer": "tampered"}, tmp_path / "bad.json") + + +def sec_fact(start, end, value, accession, known, concept="NetIncomeLoss"): + return { + "instrument_id": "A", + "concept": concept, + "unit": "USD", + "period_start": pd.Timestamp(start), + "period_end": pd.Timestamp(end), + "value": value, + "accession": accession, + "form": "10-Q", + "accepted_at": pd.Timestamp(known), + "available_at": pd.Timestamp(known), + } + + +def test_sec_cumulative_quarters_and_mixed_vintage_fail_closed(): + facts = pd.DataFrame( + [ + sec_fact("2023-01-01", "2023-03-31", 10, "q1", "2023-05-01T00:00:00Z"), + sec_fact("2023-01-01", "2023-06-30", 25, "q2", "2023-08-01T00:00:00Z"), + sec_fact("2023-01-01", "2023-09-30", 45, "q3", "2023-11-01T00:00:00Z"), + sec_fact("2023-01-01", "2023-12-31", 70, "fy", T1), + ] + ) + quarters = quarterly_facts(facts, T2) + assert quarters.value.tolist() == [10, 15, 20, 25] + assert ttm_facts(facts, T2).empty # No silent cross-filing comparability assumption. + assert ttm_facts(facts, T2, allow_mixed_vintages=True).iloc[0].value == 70 + revised = pd.DataFrame([sec_fact("2023-01-01", "2023-06-30", 30, "amended", T2)]) + extended = pd.concat([facts, revised], ignore_index=True) + pd.testing.assert_frame_equal(quarterly_facts(facts, T1), quarterly_facts(extended, T1)) + + +def test_sec_53_week_year_and_gap_not_fabricated(): + bounds = [ + ("2022-12-26", "2023-03-26"), + ("2023-03-27", "2023-06-25"), + ("2023-06-26", "2023-09-24"), + ("2023-09-25", "2023-12-31"), + ] + facts = pd.DataFrame([sec_fact(a, b, 10, str(i), T1) for i, (a, b) in enumerate(bounds)]) + assert ttm_facts(facts, T2).iloc[0].value == 40 + assert ttm_facts(facts.iloc[1:], T2).empty + + +@pytest.mark.parametrize("unit,basis,lot", [("lots", "raw", None), ("shares", "adjusted", None)]) +def test_units_reject_implicit_multiplier_or_adjusted_shares(unit, basis, lot): + with pytest.raises(ValueError): + normalize_trading_units( + pd.DataFrame({"volume": [1], "amount": [2]}), + volume_unit=unit, + amount_unit="currency", + currency="CNY", + share_basis=basis, + lot_size=lot, + ) + + +def test_actions_require_explicit_election_and_reject_adjustment_mix(): + args = { + "event_id": "x", + "instrument_id": "A", + "kind": "rights_exercise", + "effective_at": T1, + "available_at": T0, + "currency": "USD", + "source": "synthetic", + "evidence_id": "fixture", + "target_id": "B", + "target_mark": "10", + } + with pytest.raises(ValueError, match="election"): + ActionTerms(**args) + with pytest.raises(ValueError, match="raw"): + ActionTerms(**{**args, "price_basis": "total_return"})