From ab17ee7b8c417aa72a9707106f5455beba965b76 Mon Sep 17 00:00:00 2001 From: PureSaber Date: Sun, 11 Oct 2026 07:34:00 +0800 Subject: [PATCH] Add bounded Cboe NSE and Deribit research ingestion --- docs/free-derivatives.md | 63 +++ src/quant_data_kit/derivatives/bundle.py | 35 ++ src/quant_data_kit/derivatives/cli.py | 33 +- src/quant_data_kit/derivatives/free_common.py | 160 ++++++++ src/quant_data_kit/derivatives/free_daily.py | 300 ++++++++++++++ .../derivatives/free_deribit.py | 266 +++++++++++++ .../derivatives/free_sources.py | 93 +++++ tests/test_free_derivatives.py | 376 ++++++++++++++++++ 8 files changed, 1325 insertions(+), 1 deletion(-) create mode 100644 docs/free-derivatives.md create mode 100644 src/quant_data_kit/derivatives/free_common.py create mode 100644 src/quant_data_kit/derivatives/free_daily.py create mode 100644 src/quant_data_kit/derivatives/free_deribit.py create mode 100644 src/quant_data_kit/derivatives/free_sources.py create mode 100644 tests/test_free_derivatives.py diff --git a/docs/free-derivatives.md b/docs/free-derivatives.md new file mode 100644 index 0000000..565ba5b --- /dev/null +++ b/docs/free-derivatives.md @@ -0,0 +1,63 @@ +# Public derivative research sources + +`qdk-derivatives free-fetch` adds bounded official-source ingestion. No API key, +broker connection or live order is involved. A successful request publishes an +immutable contracts/quotes/manifest bundle with source URLs, response hashes, +normalization assumptions, excluded records and allowed research modes. + +| Provider | Explicit selection | Supported data | Boundary | +| --- | --- | --- | --- | +| `cboe-vix` | Actual VX expiry dates, comma separated | Cboe per-contract daily OHLC, settlement, volume and OI | VX futures only; select actual expiry, including holiday adjustments | +| `nse-fo` | `FinInstrmNm`, e.g. `NIFTY25JAN23500PE` | NSE UDiFF daily index futures/options | Historical `NewBrdLotQty` and actual expiry come from each row; stock/physical delivery unsupported | +| `deribit` | BTC/ETH option names | `trades`: historical public trades aggregated daily; `snapshot`: current vendor marks | USD analytical projection of native coin premiums; **analysis only** | + +Example, run from an installed locked environment: + +```powershell +qdk-derivatives free-fetch --provider cboe-vix --kind future --selection 2025-02-19,2025-03-18 --start 2025-01-08 --end 2025-01-10 --output D:/research/vx-v1 --rights-note "Personal local research; no raw redistribution" +qdk-derivatives free-fetch --provider nse-fo --kind option --selection NIFTY25JAN23500PE --start 2025-01-08 --end 2025-01-10 --output D:/research/nse-v1 --rights-note "Personal local research; no raw redistribution" +qdk-derivatives free-fetch --provider deribit --kind option --selection BTC-31JAN25-100000-C --source-mode trades --start 2025-01-08 --end 2025-01-10 --output D:/research/btc-v1 --rights-note "Personal local research; no raw redistribution" +``` + +Each request covers 1–32 inclusive dates and explicit contracts (20 maximum; +Deribit 10). A snapshot requires today's UTC date and unexpired options. The +source's daily session is retained in the normalized quote; date requests are +calendar bounds, not a claim of complete exchange-calendar coverage. + +Daily zero-volume records are disclosed and excluded, not replaced by fabricated +OHLC. Missing NSE HTTP 404 dates are disclosed without calling them holidays. +Access denials stop the request without bypass or automatic retry. No incomplete +download is published. Deribit pagination is capped at 20 pages per instrument; +sequence gaps, overlaps or an ambiguous boundary fail closed. Same-millisecond +trades are ordered by their official sequence number. + +For official files downloaded by the user, add `--files-directory PATH`. Use the +original `VX_YYYY-MM-DD.csv` or +`BhavCopy_NSE_FO_0_0_0_YYYYMMDD_F_0000.csv.zip` names. Files are read locally; +untrusted ZIP paths and oversized decompression are rejected. + +Research defaults are analysis only. Cboe/NSE may enable modeled replay with both +`--initial-margin` and `--maintenance-margin` (per contract, contract currency). +These and `--tick` are **explicit research assumptions**, not historical exchange +margin certification. Samples must end before expiry; no final settlement is +inferred. Publication times and first-observed listing times in daily files are +conservative retrospective assumptions, not point-in-time evidence. + +Deribit trade premiums use each trade's USD index. Snapshot marks use one shared +index fixing within a bounded capture interval; the source timestamps are kept. +These are calculation snapshots, not synchronized executable quotes. Native coin +settlement, inverse payoff, collateral and variable tick rules are not modeled +by the USD ledger. Bundle validation and the research/replay layers must enforce +`allowed_modes`. Multi-leg historical analysis also requires synchronized last +observations; using different trades as if simultaneous is rejected. + +Public access does not grant redistribution rights. Raw downloaded responses and +customer data stay outside Git. Unit tests use synthetic official-shaped records. + +Official references: + +- [Cboe historical files](https://www.cboe.com/markets/us/futures/market-statistics/historical-data/futures) +- [VX specifications](https://www.cboe.com/tradable-products/vix/vix-futures/specifications) +- [NSE derivative reports](https://www.nseindia.com/all-reports-derivatives) +- [NSE index contract specifications](https://www.nseindia.com/static/products-services/equity-derivatives-contract-specifications) +- [Deribit public API](https://docs.deribit.com/) diff --git a/src/quant_data_kit/derivatives/bundle.py b/src/quant_data_kit/derivatives/bundle.py index 0296fe2..822224a 100644 --- a/src/quant_data_kit/derivatives/bundle.py +++ b/src/quant_data_kit/derivatives/bundle.py @@ -84,6 +84,21 @@ class DerivativeBundle: quotes: tuple[Quote, ...] identity: str + @property + def allowed_modes(self): + source = self.manifest.get("source", {}) + return ( + source.get("allowed_modes", ["analysis", "replay"]) + if isinstance(source, dict) + else ["analysis", "replay"] + ) + + def require_mode(self, mode): + if mode not in self.allowed_modes: + raise ValueError( + f"input permits {self.allowed_modes}; {mode} is blocked by source usage restrictions" + ) + def contract(self, instrument_id): try: return next(c for c in self.contracts if c.instrument_id == instrument_id) @@ -123,6 +138,7 @@ def summary(self): "venues": sorted({c.venue for c in self.contracts}), "input_sha256": self.identity, "limits": self.manifest["limits"], + "allowed_modes": self.allowed_modes, } @@ -144,6 +160,25 @@ def load_bundle(root): if not isinstance(manifest.get(name), str) or not manifest[name].strip(): raise ValueError(f"{name} must be declared") utc(manifest["captured_at"]) + source = manifest.get("source", {}) + if manifest["provider"] in {"cboe-vix", "nse-fo", "deribit"}: + if ( + not isinstance(source, dict) + or source.get("schema") != "qdk.free-derivatives/v1" + or "allowed_modes" not in source + ): + raise ValueError("public derivative input must declare its usage restrictions") + if manifest["provider"] == "deribit" and source["allowed_modes"] != ["analysis"]: + raise ValueError("Deribit USD projections are analysis-only") + if isinstance(source, dict) and "allowed_modes" in source: + modes = source["allowed_modes"] + if ( + not isinstance(modes, list) + or not modes + or any(m not in {"analysis", "replay"} for m in modes) + or len(set(modes)) != len(modes) + ): + raise ValueError("invalid input usage restrictions") files = manifest.get("files", {}) if set(files) != {"contracts.json", "quotes.csv"}: raise ValueError("unexpected bundle file set") diff --git a/src/quant_data_kit/derivatives/cli.py b/src/quant_data_kit/derivatives/cli.py index 854a042..c16be4f 100644 --- a/src/quant_data_kit/derivatives/cli.py +++ b/src/quant_data_kit/derivatives/cli.py @@ -27,6 +27,30 @@ def main(argv=None): fetch.add_argument("--end", required=True) fetch.add_argument("--dataset") fetch.add_argument("--max-cost-usd") + free = sub.add_parser("free-fetch", help="Cboe VX, NSE index F&O, Deribit coin-option analysis") + free.add_argument("--provider", choices=["cboe-vix", "nse-fo", "deribit"], required=True) + free.add_argument("--kind", choices=["future", "option"], required=True) + free.add_argument( + "--selection", + required=True, + help="comma-separated VX expiry dates, NSE FinInstrmNm or Deribit symbols", + ) + free.add_argument("--start", required=True) + free.add_argument("--end", required=True) + free.add_argument("--source-mode", choices=["daily", "trades", "snapshot"], default="daily") + free.add_argument( + "--tick", + default="0.05", + help="explicit research tick assumption, not historical rule certification", + ) + free.add_argument("--initial-margin") + free.add_argument("--maintenance-margin") + free.add_argument( + "--files-directory", + help="offline official files; expected exact filenames appear in documentation/receipts", + ) + free.add_argument("--output", required=True) + free.add_argument("--rights-note", required=True) for p in (imp, fetch): p.add_argument("--contracts", required=True, help="explicit contract rules JSON") p.add_argument("--output", required=True) @@ -37,6 +61,13 @@ def main(argv=None): result = load_bundle(args.bundle) elif args.command == "demo": result = write_demo(args.output, args.kind) + elif args.command == "free-fetch": + from .free_sources import collect_free + + options = vars(args).copy() + options.pop("command") + output = options.pop("output") + result = collect_free(output, **options) else: contracts = [ record(Contract, c) @@ -67,7 +98,7 @@ def main(argv=None): rights_note=args.rights_note, **extra, ) - print(json.dumps(result.summary(), ensure_ascii=False, indent=2)) + print(json.dumps(result.summary(), ensure_ascii=True, indent=2)) return 0 except (ValueError, OSError, KeyError, TypeError) as exc: parser.exit(2, f"derivative input rejected: {exc}\n") diff --git a/src/quant_data_kit/derivatives/free_common.py b/src/quant_data_kit/derivatives/free_common.py new file mode 100644 index 0000000..9b7898f --- /dev/null +++ b/src/quant_data_kit/derivatives/free_common.py @@ -0,0 +1,160 @@ +"""Bounded public-file/API ingestion; never bypass a source access rejection.""" + +import csv +import io +import json +import re +from datetime import date, datetime, time, timedelta, timezone +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen +from zoneinfo import ZoneInfo + +from .bundle import digest +from .models import decimal + +MAX_BYTES = 32 * 1024 * 1024 +PROVIDERS = {"cboe-vix", "nse-fo", "deribit"} + + +class PublicSourceError(ValueError): + def __init__(self, status): + self.status = status + super().__init__(f"public source HTTP {status}; use official files if access is restricted") + + +def public_request(url): + try: + req = Request(url, headers={"User-Agent": "QuantResearch/1.0 (read-only research)"}) + with urlopen(req, timeout=30) as response: + raw = response.read(MAX_BYTES + 1) + except HTTPError as exc: + raise PublicSourceError(exc.code) from None + except (URLError, OSError, TimeoutError): + raise ValueError( + "public data source unavailable; no automatic retry or partial publication" + ) from None + if len(raw) > MAX_BYTES: + raise ValueError("public response exceeds 32 MiB; narrow the request") + return raw + + +def selections(value): + values = value.split(",") if isinstance(value, str) else list(value) + values = [v.strip() for v in values] + if not 1 <= len(values) <= 20 or len(set(values)) != len(values): + raise ValueError("select 1-20 distinct contracts, separated by commas") + if any(not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", v) for v in values): + raise ValueError("invalid contract selection") + return values + + +def validate_request(provider, kind, selection, start, end, source_mode="daily"): + if provider not in PROVIDERS or kind not in {"future", "option"}: + raise ValueError("unsupported free source or instrument kind") + values = selections(selection) + first, last = date.fromisoformat(start), date.fromisoformat(end) + if not 0 <= (last - first).days <= 31 or last > datetime.now(timezone.utc).date(): + raise ValueError("select 1-32 inclusive completed/current UTC dates") + if provider == "cboe-vix": + if kind != "future" or source_mode != "daily": + raise ValueError("Cboe adapter supports VX futures daily contract files") + for value in values: + expiry = date.fromisoformat(value) + if last >= expiry: + raise ValueError( + "Cboe sample must end before the selected expiry; final fixing is separate" + ) + elif provider == "nse-fo" and source_mode != "daily": + raise ValueError("NSE adapter supports daily UDiFF files") + elif provider == "deribit": + if kind != "option" or source_mode not in {"trades", "snapshot"}: + raise ValueError("Deribit supports BTC/ETH option trades or analytical snapshots") + if len(values) > 10: + raise ValueError("Deribit collection is bounded to ten explicit option contracts") + if any( + not re.fullmatch(r"(?:BTC|ETH)-\d{1,2}[A-Z]{3}\d{2}-\d+(?:d\d+)?-[CP]", v) + for v in values + ): + raise ValueError("select explicit BTC/ETH coin-quoted vanilla option symbols") + if len({v.split("-")[0] for v in values}) != 1: + raise ValueError("one underlying currency per Deribit request") + if source_mode == "snapshot" and ( + first != last or last != datetime.now(timezone.utc).date() + ): + raise ValueError("a current snapshot requires today's UTC date; it cannot be backdated") + return values, first, last + + +def csv_rows(raw, required): + if len(raw) > MAX_BYTES: + raise ValueError("CSV exceeds 32 MiB") + reader = csv.DictReader(io.StringIO(raw.decode("utf-8-sig"), newline="")) + if ( + not reader.fieldnames + or len(set(reader.fieldnames)) != len(reader.fieldnames) + or not set(required) <= set(reader.fieldnames) + ): + raise ValueError("public CSV schema changed or required fields are absent") + rows = list(reader) + if any(None in row for row in rows): + raise ValueError("malformed public CSV record") + return rows + + +def daily_time(day, zone, hour, minute=0): + at = datetime.combine(day, time(hour, minute), ZoneInfo(zone)).astimezone(timezone.utc) + # Explicit conservative research assumption, not historical publication evidence. + available = datetime.combine(day + timedelta(days=1), time(9), ZoneInfo(zone)).astimezone( + timezone.utc + ) + return at, available + + +def margin_values(initial=None, maintenance=None): + if initial is None and maintenance is None: + return None, None + if initial is None or maintenance is None: + raise ValueError("declare both research margin amounts or neither") + initial, maintenance = decimal(initial), decimal(maintenance) + if not 0 <= maintenance <= initial or initial <= 0: + raise ValueError("margin requires 0 <= maintenance <= positive initial") + return initial, maintenance + + +class Resources: + def __init__(self, *, fetch=public_request, directory=None): + self.fetch = fetch + self.directory = Path(directory).resolve() if directory else None + self.receipts = [] + + def read(self, filename, url): + if self.directory: + path = (self.directory / filename).resolve() + if ( + path.parent != self.directory + or not path.is_file() + or path.stat().st_size > MAX_BYTES + ): + raise ValueError(f"missing, escaped or oversized official source file: {filename}") + raw = path.read_bytes() + else: + raw = self.fetch(url) + if len(raw) > MAX_BYTES: + raise ValueError("public source response exceeds 32 MiB") + self.receipts.append( + { + "file": filename, + "url": url, + "bytes": len(raw), + "sha256": digest(raw), + "transport": "local-file" if self.directory else "https", + } + ) + return raw + + def rpc(self, filename, url): + data = json.loads(self.read(filename, url)) + if not isinstance(data, dict) or "error" in data or "result" not in data: + raise ValueError("public API returned a JSON-RPC error or changed schema") + return data["result"] diff --git a/src/quant_data_kit/derivatives/free_daily.py b/src/quant_data_kit/derivatives/free_daily.py new file mode 100644 index 0000000..e1419b2 --- /dev/null +++ b/src/quant_data_kit/derivatives/free_daily.py @@ -0,0 +1,300 @@ +"""Cboe VX contract CSV and NSE index F&O UDiFF adapters.""" + +import io +import zipfile +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from zoneinfo import ZoneInfo + +from .free_common import MAX_BYTES, PublicSourceError, csv_rows, daily_time +from .models import Contract, Quote, decimal + +CBOE = "https://cdn.cboe.com/data/us/futures/market_statistics/historical_data/VX/" +NSE = "https://nsearchives.nseindia.com/content/fo/" +INDEX_PRODUCTS = {"NIFTY", "BANKNIFTY", "FINNIFTY", "MIDCPNIFTY", "NIFTYNXT50"} + + +def cboe_vix(resources, selected, first, last, *, tick, margins): + contracts, quotes, skipped = [], [], [] + for expiration in selected: + expiry_day = date.fromisoformat(expiration) + filename = f"VX_{expiration}.csv" + rows = csv_rows( + resources.read(filename, CBOE + filename), + [ + "Trade Date", + "Futures", + "Open", + "High", + "Low", + "Close", + "Settle", + "Total Volume", + "Open Interest", + ], + ) + instrument = f"CFE:VX:{expiration}" + contract_quotes = [] + seen = set() + for row in rows: + day = date.fromisoformat(row["Trade Date"]) + if not first <= day <= last: + continue + if day in seen: + raise ValueError("duplicate Cboe contract session") + seen.add(day) + month = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + ][expiry_day.month - 1] + label = f"({month} {expiry_day.year})" + if label not in row["Futures"]: + raise ValueError("Cboe contract month does not match selected expiry") + volume = decimal(row["Total Volume"]) + prices = [decimal(row[k]) for k in ("Open", "High", "Low", "Close")] + if volume == 0: + skipped.append( + { + "symbol": instrument, + "session": day.isoformat(), + "reason": "zero trading volume; reference/settlement-only row excluded", + } + ) + continue + if min(prices) <= 0: + raise ValueError("Cboe traded bar has missing/nonpositive OHLC; no fabricated fill") + at, available = daily_time(day, "America/Chicago", 16) + contract_quotes.append( + Quote( + instrument, + at, + available, + day.isoformat(), + *prices, + volume, + open_interest=row["Open Interest"], + settlement=row["Settle"], + ) + ) + if not contract_quotes: + raise ValueError(f"no traded bars for selected Cboe expiry {expiration}") + listed = min(q.at for q in contract_quotes) - timedelta(days=1) + expiry = datetime.combine(expiry_day, time(8), ZoneInfo("America/Chicago")) + contracts.append( + Contract( + instrument, + "VX-" + expiration, + "VX", + "CFE", + "USD", + "future", + Decimal(1000), + tick, + listed, + expiry, + expiry, + listed, + "America/Chicago", + "cash", + initial_margin=margins[0], + maintenance_margin=margins[1], + rules_source="Cboe VX specifications; explicit expiry selection; first-observed lifecycle and user tick/margin assumptions", + ) + ) + quotes.extend(contract_quotes) + return ( + contracts, + quotes, + { + "excluded_rows": skipped, + "unavailable_days": [], + "contract_rules_url": "https://www.cboe.com/tradable-products/vix/vix-futures/specifications", + "date_basis": "Chicago daily session; publication assumed next local day 09:00; not PIT evidence", + }, + ) + + +def nse_csv(raw): + if not raw.startswith(b"PK"): + raise ValueError("NSE response must be an official ZIP, not an HTML/error page") + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + items = archive.infolist() + if ( + len(items) != 1 + or not items[0].filename.endswith(".csv") + or items[0].file_size > MAX_BYTES + or items[0].flag_bits & 1 + ): + raise ValueError("NSE ZIP must contain one unencrypted CSV within 32 MiB") + item = items[0] + if ( + "/" in item.filename + or "\\" in item.filename + or item.file_size > max(1, item.compress_size) * 200 + ): + raise ValueError("unsafe or excessive NSE archive expansion") + return archive.read(item) + + +def nse_fo(resources, selected, first, last, *, kind, tick, margins): + quotes, definitions, missing, skipped = [], {}, [], [] + day = first + expected_type = "IDO" if kind == "option" else "IDF" + required = [ + "TradDt", + "Src", + "Sgmt", + "FinInstrmTp", + "FinInstrmId", + "FinInstrmNm", + "TckrSymb", + "FininstrmActlXpryDt", + "NewBrdLotQty", + "OpnPric", + "HghPric", + "LwPric", + "ClsPric", + "SttlmPric", + "TtlTradgVol", + "OpnIntrst", + "UndrlygPric", + "StrkPric", + "OptnTp", + ] + while day <= last: + filename = f"BhavCopy_NSE_FO_0_0_0_{day:%Y%m%d}_F_0000.csv.zip" + try: + raw = resources.read(filename, NSE + filename) + except PublicSourceError as exc: + if exc.status != 404: + raise + missing.append(day.isoformat()) + day += timedelta(days=1) + continue + rows = csv_rows(nse_csv(raw), required) + seen = set() + for row in rows: + name = row["FinInstrmNm"] + if name not in selected: + continue + if name in seen: + raise ValueError("duplicate NSE contract session") + seen.add(name) + if row["TradDt"] != day.isoformat() or row["Src"] != "NSE" or row["Sgmt"] != "FO": + raise ValueError("NSE source/date/segment does not match request") + if row["FinInstrmTp"] != expected_type or row["TckrSymb"] not in INDEX_PRODUCTS: + raise ValueError("adapter supports selected NSE index futures/options only") + expiry_day = date.fromisoformat(row["FininstrmActlXpryDt"]) + if last >= expiry_day: + raise ValueError("NSE sample must end before expiry; final fixing is separate") + lot = decimal(row["NewBrdLotQty"]) + if lot <= 0 or lot != int(lot): + raise ValueError("NSE historical lot size must be a positive integer") + definition = { + k: row[k] + for k in ( + "FinInstrmId", + "TckrSymb", + "FininstrmActlXpryDt", + "NewBrdLotQty", + "StrkPric", + "OptnTp", + ) + } + if name in definitions and definitions[name] != definition: + raise ValueError("NSE contract rules changed within window; split the input bundle") + definitions[name] = definition + volume = decimal(row["TtlTradgVol"]) + if volume == 0: + skipped.append( + { + "symbol": name, + "session": day.isoformat(), + "reason": "zero-volume daily bar excluded", + } + ) + continue + prices = [decimal(row[k]) for k in ("OpnPric", "HghPric", "LwPric", "ClsPric")] + if min(prices) <= 0: + raise ValueError("NSE traded bar has nonpositive OHLC") + at, available = daily_time(day, "Asia/Kolkata", 15, 30) + underlying = decimal(row["UndrlygPric"]) if row["UndrlygPric"] else None + if kind == "option" and (underlying is None or underlying <= 0): + raise ValueError("NSE option row lacks a positive source underlying price") + quotes.append( + Quote( + "NSE:" + name, + at, + available, + day.isoformat(), + *prices, + volume, + open_interest=decimal(row["OpnIntrst"]) / lot, + settlement=row["SttlmPric"] or None, + underlying_price=underlying, + ) + ) + day += timedelta(days=1) + contracts = [] + for name in selected: + observed = [q for q in quotes if q.instrument_id == "NSE:" + name] + if not observed: + raise ValueError(f"no traded observations for requested NSE symbol {name}") + row = definitions[name] + expiry_day = date.fromisoformat(row["FininstrmActlXpryDt"]) + expiry = datetime.combine(expiry_day, time(15, 30), ZoneInfo("Asia/Kolkata")) + listed = min(q.at for q in observed) - timedelta(days=1) + option = {} + if kind == "option": + if row["OptnTp"] not in {"CE", "PE"}: + raise ValueError("NSE index option must declare CE or PE") + option = { + "underlying": row["TckrSymb"], + "option_right": "call" if row["OptnTp"] == "CE" else "put", + "strike": row["StrkPric"], + "exercise_style": "european", + } + contracts.append( + Contract( + "NSE:" + name, + name, + row["TckrSymb"], + "NSE", + "INR", + kind, + row["NewBrdLotQty"], + tick, + listed, + expiry, + expiry, + listed, + "Asia/Kolkata", + "cash", + initial_margin=margins[0], + maintenance_margin=margins[1], + rules_source="NSE UDiFF actual expiry and historical lot; index-only cash settlement; explicit user tick/margin assumptions", + **option, + ) + ) + return ( + contracts, + quotes, + { + "excluded_rows": skipped, + "unavailable_days": missing, + "definitions": definitions, + "volume_unit": "contracts; source open interest divided by historical NewBrdLotQty", + "date_basis": "NSE TradDt, assumed close 15:30 IST and publication next local day 09:00; not PIT certified", + "missing_days_note": "HTTP 404 dates are disclosed, not asserted to be exchange holidays or complete coverage", + }, + ) diff --git a/src/quant_data_kit/derivatives/free_deribit.py b/src/quant_data_kit/derivatives/free_deribit.py new file mode 100644 index 0000000..c2407ca --- /dev/null +++ b/src/quant_data_kit/derivatives/free_deribit.py @@ -0,0 +1,266 @@ +"""Coin-quoted options projected to USD for analysis, never ledger replay.""" + +from collections import defaultdict +from datetime import datetime, time, timedelta, timezone +from decimal import Decimal +from urllib.parse import urlencode + +from .models import Contract, Quote, decimal + +HISTORY = "https://history.deribit.com/api/v2/public/" +CURRENT = "https://www.deribit.com/api/v2/public/" + + +def timestamp(ms): + return datetime.fromtimestamp(int(ms) / 1000, timezone.utc) + + +def instrument(resources, name, *, history): + base = HISTORY if history else CURRENT + raw = resources.rpc( + f"instrument-{name}.json", base + "get_instrument?" + urlencode({"instrument_name": name}) + ) + coin = name.split("-")[0] + if ( + raw.get("instrument_name") != name + or raw.get("kind") != "option" + or raw.get("base_currency") != coin + or raw.get("quote_currency") != coin + or raw.get("settlement_currency") != coin + or raw.get("counter_currency") != "USD" + ): + raise ValueError("Deribit metadata is not the selected coin-quoted USD-strike option") + if raw.get("option_type") not in {"call", "put"}: + raise ValueError("invalid Deribit option right") + return raw + + +def contract(raw, tick): + name = raw["instrument_name"] + expiry, listed = timestamp(raw["expiration_timestamp"]), timestamp(raw["creation_timestamp"]) + return Contract( + "DERIBIT:" + name, + name, + raw["base_currency"], + "DERIBIT", + "USD", + "option", + raw["contract_size"], + tick, + listed, + expiry, + expiry, + listed, + "UTC", + "cash", + underlying=raw["base_currency"], + option_right=raw["option_type"], + strike=raw["strike"], + exercise_style="european", + rules_source="Deribit metadata; ANALYTICAL USD projection of coin-settled contract, not a USD cash contract; tick is analytical only", + ) + + +def historical_trades(resources, names, first, last, *, tick): + contracts, quotes, metadata, statistics = [], [], {}, {} + start = int(datetime.combine(first, time(), timezone.utc).timestamp() * 1000) + end = ( + int(datetime.combine(last + timedelta(days=1), time(), timezone.utc).timestamp() * 1000) - 1 + ) + for name in names: + info = instrument(resources, name, history=True) + c = contract(info, tick) + if end >= info["expiration_timestamp"]: + raise ValueError("historical option analysis must end before expiry") + metadata[name] = info + contracts.append(c) + rows, previous, finished = [], None, False + params = { + "instrument_name": name, + "start_timestamp": start, + "end_timestamp": end, + "count": 1000, + "sorting": "asc", + "include_old": "true", + } + method = "get_last_trades_by_instrument_and_time" + for page in range(20): + result = resources.rpc( + f"trades-{name}-{page:02d}.json", HISTORY + method + "?" + urlencode(params) + ) + trades = result.get("trades") + if not isinstance(trades, list) or not isinstance(result.get("has_more"), bool): + raise ValueError("Deribit trade page schema changed") # noqa: TRY004 # noqa: TRY004 + # Time-sorted responses can reorder trades sharing a millisecond. + # Sequence order is deterministic; gaps still fail closed, including + # an ambiguous page boundary that cuts through such a group. + trades = sorted(trades, key=lambda t: int(t["trade_seq"])) + for trade in trades: + if trade.get("instrument_name") != name: + raise ValueError("Deribit returned an unrequested instrument") + sequence = int(trade["trade_seq"]) + if previous is not None and sequence != previous + 1: + raise ValueError( + "Deribit trade sequence gap/overlap; incomplete input rejected" + ) + previous = sequence + if int(trade["timestamp"]) < start: + raise ValueError("Deribit returned data preceding requested interval") + if int(trade["timestamp"]) > end: + finished = True + break + rows.append(trade) + if finished or not result["has_more"]: + finished = True + break + if not trades or previous is None: + raise ValueError("Deribit pagination made no progress") + method = "get_last_trades_by_instrument" + params = { + "instrument_name": name, + "start_seq": previous + 1, + "count": 1000, + "sorting": "asc", + "include_old": "true", + } + if not finished: + raise ValueError("Deribit history exceeded 20-page bound; narrow the date window") + if not rows: + raise ValueError(f"no trades for selected option {name}; no zero bars manufactured") + grouped = defaultdict(list) + for row in rows: + grouped[timestamp(row["timestamp"]).date()].append(row) + for day, trades in sorted(grouped.items()): + prices, amounts = [], [] + previous_time = None + for row in trades: + observed = timestamp(row["timestamp"]) + if previous_time is not None and observed < previous_time: + raise ValueError("Deribit trade timestamps moved backwards") + previous_time = observed + index, premium, amount = ( + decimal(row["index_price"]), + decimal(row["price"]), + decimal(row["amount"]), + ) + if index <= 0 or premium < 0 or amount <= 0: + raise ValueError("invalid Deribit trade price/index/amount") + prices.append(premium * index) + amounts.append(amount / c.multiplier) + at = timestamp(trades[-1]["timestamp"]) + available = datetime.combine(day + timedelta(days=1), time(), timezone.utc) + quotes.append( + Quote( + c.instrument_id, + at, + available, + day.isoformat(), + prices[0], + max(prices), + min(prices), + prices[-1], + sum(amounts), + underlying_price=trades[-1]["index_price"], + ) + ) + statistics[name] = { + "trades": len(rows), + "first_sequence": rows[0]["trade_seq"], + "last_sequence": rows[-1]["trade_seq"], + "sessions": len(grouped), + } + return ( + contracts, + quotes, + { + "native_instruments": metadata, + "trade_coverage": statistics, + "price_basis": "observed trades projected to USD using each trade's index_price", + "date_basis": "UTC daily aggregation retains actual last-trade timestamp and underlying; next UTC day availability assumed", + "synchronization": "different legs may have asynchronous last trades; portfolio preflight rejects unsynchronized legs", + "allowed_modes": ["analysis"], + }, + ) + + +def current_snapshot(resources, names, *, tick, now=None): + started = datetime.now(timezone.utc) if now is None else now + metadata = {name: instrument(resources, name, history=False) for name in names} + coin = next(iter(metadata.values()))["base_currency"] + summaries = resources.rpc( + "option-summary.json", + CURRENT + "get_book_summary_by_currency?" + urlencode({"currency": coin, "kind": "option"}), + ) + index = resources.rpc( + "index-price.json", + CURRENT + "get_index_price?" + urlencode({"index_name": coin.lower() + "_usd"}), + ) + fixing = decimal(index["index_price"]) + if fixing <= 0: + raise ValueError("Deribit conversion fixing must be positive") + finished = datetime.now(timezone.utc) if now is None else now + if (finished - started).total_seconds() > 120: + raise ValueError("snapshot collection exceeded the two-minute freshness bound") + selected = [r for r in summaries if r.get("instrument_name") in names] + if len(selected) != len(names) or len({r["instrument_name"] for r in selected}) != len(names): + raise ValueError("Deribit option snapshot is incomplete or duplicated") + contracts, quotes, native_rows = [], [], [] + for row in selected: + name = row["instrument_name"] + source_time = timestamp(row["creation_timestamp"]) + if not -5 <= (finished - source_time).total_seconds() <= 120: + raise ValueError("stale/future Deribit summary; refresh the snapshot") + c = contract(metadata[name], tick) + if finished >= c.expiry: + raise ValueError("expired instrument in current snapshot") + premium = decimal(row["mark_price"]) * fixing + if premium <= 0: + raise ValueError("nonpositive Deribit mark price") + # A calculation snapshot, not an OHLC bar or synchronized executable quote. + quotes.append( + Quote( + c.instrument_id, + finished, + finished, + finished.date().isoformat(), + premium, + premium, + premium, + premium, + Decimal(0), + open_interest=decimal(row["open_interest"]) / c.multiplier, + underlying_price=fixing, + ) + ) + contracts.append(c) + native_rows.append( + { + k: row.get(k) + for k in ( + "instrument_name", + "creation_timestamp", + "mark_price", + "mark_iv", + "bid_price", + "ask_price", + "underlying_price", + "underlying_index", + "volume", + "open_interest", + ) + } + ) + return ( + contracts, + quotes, + { + "native_instruments": metadata, + "native_observations": native_rows, + "conversion_index_price": str(fixing), + "capture_started": started.isoformat(), + "capture_finished": finished.isoformat(), + "price_basis": "vendor mark price projected to USD using a shared current index fixing; not executed trades", + "synchronization": "bounded sequential capture; common calculation time/fixing, not an atomic market snapshot", + "allowed_modes": ["analysis"], + }, + ) diff --git a/src/quant_data_kit/derivatives/free_sources.py b/src/quant_data_kit/derivatives/free_sources.py new file mode 100644 index 0000000..5d83596 --- /dev/null +++ b/src/quant_data_kit/derivatives/free_sources.py @@ -0,0 +1,93 @@ +"""Three public derivative sources with explicit research-use restrictions.""" + +from .bundle import write_bundle +from .free_common import Resources, margin_values, public_request, validate_request +from .free_daily import cboe_vix, nse_fo +from .free_deribit import current_snapshot, historical_trades +from .models import decimal + + +def collect_free( + output, + *, + provider, + kind, + selection, + start, + end, + rights_note, + source_mode="daily", + tick="0.05", + initial_margin=None, + maintenance_margin=None, + files_directory=None, + fetch=public_request, + now=None, +): + selected, first, last = validate_request(provider, kind, selection, start, end, source_mode) + if not isinstance(rights_note, str) or not 1 <= len(rights_note.strip()) <= 1000: + raise ValueError("declare personal research rights/use scope without credentials") + tick = decimal(tick) + if not 0 < tick <= 10000: + raise ValueError("positive analytical/research tick within 10000 required") + margins = margin_values(initial_margin, maintenance_margin) + resources = Resources(fetch=fetch, directory=files_directory) + if provider == "cboe-vix": + contracts, quotes, quality = cboe_vix( + resources, selected, first, last, tick=tick, margins=margins + ) + elif provider == "nse-fo": + contracts, quotes, quality = nse_fo( + resources, selected, first, last, kind=kind, tick=tick, margins=margins + ) + else: + if margins != (None, None): + raise ValueError("coin-settled Deribit projections cannot enable account replay") + if source_mode == "snapshot": + contracts, quotes, quality = current_snapshot(resources, selected, tick=tick, now=now) + else: + contracts, quotes, quality = historical_trades( + resources, selected, first, last, tick=tick + ) + allowed = quality.pop( + "allowed_modes", ["analysis", "replay"] if margins[0] is not None else ["analysis"] + ) + quotes.sort(key=lambda q: (q.at, q.instrument_id)) + contracts.sort(key=lambda c: c.instrument_id) + suggestion = {"mode": "analysis"} + if kind == "option": + suggestion["legs"] = contracts[0].instrument_id + "=1" + else: + suggestion["product"] = contracts[0].product + source = { + "schema": "qdk.free-derivatives/v1", + "provider": provider, + "source_mode": source_mode, + "selection": selected, + "requested_start": start, + "requested_end": end, + "allowed_modes": allowed, + "suggested_config": suggestion, + "resources": resources.receipts, + "quality": quality, + "tick_assumption": str(tick), + "margin_assumptions": [str(m) if m is not None else None for m in margins], + } + limits = "Public source does not imply redistribution rights. Retrospective research only; actual historical publication/contract listing times are not certified. Zero-volume daily rows are excluded and missing source dates disclosed; no complete calendar coverage claim. Tick/margin values are explicit research assumptions. " + if provider == "deribit": + limits += "ANALYSIS ONLY: BTC/ETH premium times observed USD index is a valuation projection. The native coin settlement, variable tick, collateral and inverse payoff are not implemented in this USD account; no replay or executable bid/ask claim. " + else: + limits += "Index contracts only; sample ends before expiry; final fixing and historical exchange margin are not inferred. Replay requires explicit per-contract margin assumptions. " + limits += "Price basis: " + quality.get( + "price_basis", "source daily traded OHLC plus explicitly reported settlement" + ) + return write_bundle( + output, + contracts, + quotes, + provider=provider, + evidence_kind="retrospective", + rights_note=rights_note, + limits=limits, + source=source, + ) diff --git a/tests/test_free_derivatives.py b/tests/test_free_derivatives.py new file mode 100644 index 0000000..3a660a7 --- /dev/null +++ b/tests/test_free_derivatives.py @@ -0,0 +1,376 @@ +"""Official-shaped synthetic responses; no downloaded market files in Git.""" + +import csv +import io +import json +import zipfile +from datetime import datetime, timezone +from decimal import Decimal +from urllib.parse import parse_qs, urlparse + +import pytest + +from quant_data_kit.derivatives import load_bundle +from quant_data_kit.derivatives.free_common import PublicSourceError, Resources, validate_request +from quant_data_kit.derivatives.free_daily import nse_csv +from quant_data_kit.derivatives.free_deribit import current_snapshot +from quant_data_kit.derivatives.free_sources import collect_free + + +def csv_bytes(rows): + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + return stream.getvalue().encode() + + +def cboe(**changes): + return dict( + { + "Trade Date": "2025-01-08", + "Futures": "G (Feb 2025)", + "Open": "18", + "High": "20", + "Low": "17", + "Close": "19", + "Settle": "19.1", + "Total Volume": "100", + "Open Interest": "200", + }, + **changes, + ) + + +def collect(tmp_path, provider="cboe-vix", kind="future", selection="2025-02-19", **kwargs): + return collect_free( + tmp_path / "bundle", + provider=provider, + kind=kind, + selection=selection, + start="2025-01-08", + end="2025-01-08", + rights_note="synthetic test responses", + **kwargs, + ) + + +def test_cboe_units_causal_availability_and_mode_guard(tmp_path): + b = collect(tmp_path, fetch=lambda _: csv_bytes([cboe()])) + assert b.contracts[0].multiplier == 1000 + assert b.quotes[0].settlement == Decimal("19.1") + assert b.asof(b.quotes[0].at) == {} + assert b.allowed_modes == ["analysis"] + with pytest.raises(ValueError, match="permits"): + b.require_mode("replay") + source = b.manifest["source"] + assert source["resources"][0]["sha256"] + assert source["suggested_config"] == {"mode": "analysis", "product": "VX"} + + +def test_cboe_official_file_import_and_explicit_margin(tmp_path): + (tmp_path / "VX_2025-02-19.csv").write_bytes(csv_bytes([cboe()])) + b = collect( + tmp_path, + files_directory=tmp_path, + initial_margin="10000", + maintenance_margin="8000", + fetch=lambda _: pytest.fail("offline request reached network"), + ) + b.require_mode("replay") + assert b.contracts[0].initial_margin == 10000 + + +@pytest.mark.parametrize( + "changes,match", + [ + ({"High": "1"}, "OHLC"), + ({"Futures": "H (Mar 2025)"}, "month"), + ({"Open": "0"}, "nonpositive"), + ({"Total Volume": "0"}, "no traded"), + ({"Total Volume": "-1"}, "negative"), + ], +) +def test_cboe_rejects_invalid_traded_rows(tmp_path, changes, match): + with pytest.raises(ValueError, match=match): + collect(tmp_path, fetch=lambda _: csv_bytes([cboe(**changes)])) + assert not (tmp_path / "bundle").exists() + + +def test_zero_volume_bad_ohlc_is_disclosed_not_fabricated(tmp_path): + rows = [ + cboe(), + cboe( + **{ + "Trade Date": "2025-01-09", + "Open": "0", + "High": "0", + "Low": "30", + "Close": "0", + "Total Volume": "0", + } + ), + ] + b = collect_free( + tmp_path / "b", + provider="cboe-vix", + kind="future", + selection="2025-02-19", + start="2025-01-08", + end="2025-01-09", + rights_note="test", + fetch=lambda _: csv_bytes(rows), + ) + assert len(b.quotes) == 1 + assert len(b.manifest["source"]["quality"]["excluded_rows"]) == 1 + + +def nse(**changes): + return dict( + { + "TradDt": "2025-01-08", + "Src": "NSE", + "Sgmt": "FO", + "FinInstrmTp": "IDO", + "FinInstrmId": "100", + "FinInstrmNm": "NIFTY25JAN23500PE", + "TckrSymb": "NIFTY", + "FininstrmActlXpryDt": "2025-01-30", + "NewBrdLotQty": "25", + "OpnPric": "225", + "HghPric": "288", + "LwPric": "194", + "ClsPric": "208", + "SttlmPric": "208", + "TtlTradgVol": "100", + "OpnIntrst": "2500", + "UndrlygPric": "23688", + "StrkPric": "23500", + "OptnTp": "PE", + }, + **changes, + ) + + +def zipped(rows, name="official.csv"): + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w", zipfile.ZIP_DEFLATED) as z: + z.writestr(name, csv_bytes(rows)) + return stream.getvalue() + + +def test_nse_historical_lot_and_open_interest_units(tmp_path): + b = collect(tmp_path, "nse-fo", "option", "NIFTY25JAN23500PE", fetch=lambda _: zipped([nse()])) + assert b.contracts[0].multiplier == 25 + assert b.contracts[0].currency == "INR" + assert b.quotes[0].volume == b.quotes[0].open_interest == 100 + assert b.quotes[0].underlying_price == 23688 + + +@pytest.mark.parametrize( + "changes,match", + [ + ({"TradDt": "2025-01-07"}, "date"), + ({"FinInstrmTp": "STO"}, "index"), + ({"NewBrdLotQty": "0"}, "lot"), + ({"UndrlygPric": ""}, "underlying"), + ({"OptnTp": "BAD"}, "CE or PE"), + ], +) +def test_nse_rejects_unsupported_or_incomplete_rules(tmp_path, changes, match): + with pytest.raises(ValueError, match=match): + collect( + tmp_path, + "nse-fo", + "option", + "NIFTY25JAN23500PE", + fetch=lambda _: zipped([nse(**changes)]), + ) + + +def test_nse_lot_change_requires_split(tmp_path): + def fetch(url): + return ( + zipped([nse()]) + if "20250108" in url + else zipped([nse(TradDt="2025-01-09", NewBrdLotQty="75")]) + ) + + with pytest.raises(ValueError, match="rules changed"): + collect_free( + tmp_path / "b", + provider="nse-fo", + kind="option", + selection="NIFTY25JAN23500PE", + start="2025-01-08", + end="2025-01-09", + rights_note="test", + fetch=fetch, + ) + + +def test_nse_discloses_404_but_never_bypasses_403(tmp_path): + def fetch(url): + if "20250108" in url: + return zipped([nse()]) + raise PublicSourceError(404) + + b = collect_free( + tmp_path / "b", + provider="nse-fo", + kind="option", + selection="NIFTY25JAN23500PE", + start="2025-01-08", + end="2025-01-09", + rights_note="test", + fetch=fetch, + ) + assert b.manifest["source"]["quality"]["unavailable_days"] == ["2025-01-09"] + calls = [] + + def blocked(url): + calls.append(url) + raise PublicSourceError(403) + + with pytest.raises(PublicSourceError): + collect(tmp_path, "nse-fo", "option", "NIFTY25JAN23500PE", fetch=blocked) + assert len(calls) == 1 + + +@pytest.mark.parametrize("raw", [b"blocked", zipped([nse()], "../escape.csv")]) +def test_unsafe_nse_archive(raw): + with pytest.raises(ValueError): + nse_csv(raw) + + +NAME = "BTC-31JAN25-100000-C" +INFO = { + "instrument_name": NAME, + "kind": "option", + "base_currency": "BTC", + "quote_currency": "BTC", + "settlement_currency": "BTC", + "counter_currency": "USD", + "option_type": "call", + "contract_size": 1, + "strike": 100000, + "creation_timestamp": 1729756811000, + "expiration_timestamp": 1738310400000, +} + + +def trade(seq=1, **changes): + return dict( + { + "instrument_name": NAME, + "trade_seq": seq, + "timestamp": 1736298000000 + seq, + "index_price": 100000, + "price": 0.04, + "amount": 0.5, + }, + **changes, + ) + + +def rpc(result): + return json.dumps({"result": result}).encode() + + +def test_deribit_pagination_and_usd_projection_never_replay(tmp_path): + calls = [] + + def fetch(url): + calls.append(url) + if "get_instrument?" in url: + return rpc(INFO) + if "start_seq" not in url: + return rpc({"trades": [trade()], "has_more": True}) + assert parse_qs(urlparse(url).query)["start_seq"] == ["2"] + return rpc({"trades": [trade(2, price=0.05)], "has_more": False}) + + b = collect(tmp_path, "deribit", "option", NAME, source_mode="trades", fetch=fetch) + assert len(calls) == 3 + assert b.quotes[0].close == 5000 and b.quotes[0].volume == 1 + assert b.allowed_modes == ["analysis"] and b.contracts[0].currency == "USD" + path = b.root / "manifest.json" + m = json.loads(path.read_text()) + m["source"]["allowed_modes"] = ["analysis", "replay"] + path.write_text(json.dumps(m)) + with pytest.raises(ValueError, match="analysis"): + load_bundle(b.root) + + +def test_same_millisecond_trade_order_is_normalized(tmp_path): + rows = [trade(2, timestamp=1736298000000), trade(1, timestamp=1736298000000)] + + def fetch(url): + return rpc(INFO if "get_instrument?" in url else {"trades": rows, "has_more": False}) + + b = collect(tmp_path, "deribit", "option", NAME, source_mode="trades", fetch=fetch) + assert b.quotes[0].volume == 1 + + +@pytest.mark.parametrize( + "rows", + [ + [trade(), trade(3)], + [trade(), trade()], + [], + [trade(index_price=0)], + [trade(instrument_name="ETH-bad")], + ], +) +def test_deribit_rejects_gaps_empty_and_bad_units(tmp_path, rows): + def fetch(url): + return rpc(INFO if "get_instrument?" in url else {"trades": rows, "has_more": False}) + + with pytest.raises(ValueError): + collect(tmp_path, "deribit", "option", NAME, source_mode="trades", fetch=fetch) + assert not (tmp_path / "bundle").exists() + + +@pytest.mark.parametrize("age,accepted", [(0, True), (121, False), (-6, False)]) +def test_snapshot_is_labeled_and_fresh(age, accepted): + now = datetime(2025, 1, 8, 12, tzinfo=timezone.utc) + + def fetch(url): + if "get_instrument?" in url: + return rpc(INFO) + if "get_index_price?" in url: + return rpc({"index_price": 100000}) + return rpc( + [ + { + "instrument_name": NAME, + "creation_timestamp": int(now.timestamp() * 1000) - age * 1000, + "mark_price": 0.04, + "open_interest": 50, + } + ] + ) + + if not accepted: + with pytest.raises(ValueError, match="stale/future"): + current_snapshot(Resources(fetch=fetch), [NAME], tick=Decimal(".05"), now=now) + else: + _, quotes, quality = current_snapshot( + Resources(fetch=fetch), [NAME], tick=Decimal(".05"), now=now + ) + assert quotes[0].close == 4000 and quotes[0].volume == 0 + assert quality["allowed_modes"] == ["analysis"] + assert "not executed trades" in quality["price_basis"] + + +@pytest.mark.parametrize( + "provider,kind,selection,mode", + [ + ("cboe-vix", "option", "2025-02-19", "daily"), + ("deribit", "option", NAME, "daily"), + ("deribit", "option", "https://evil.test", "trades"), + ("nse-fo", "future", "x", "snapshot"), + ], +) +def test_request_boundaries(provider, kind, selection, mode): + with pytest.raises(ValueError): + validate_request(provider, kind, selection, "2025-01-08", "2025-01-08", mode)