From 55be2d29e11e01860b5299f781e453314cb7df2d Mon Sep 17 00:00:00 2001 From: OlegOpyakin <145427708+OlegOpyakin@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:48:14 +0000 Subject: [PATCH 1/5] dev container impl --- .devcontainer/Dockerfile | 22 ++ .devcontainer/devcontainer.json | 28 ++ .devcontainer/postCreateCommand.sh | 23 ++ .gitignore | 20 ++ README.md | 8 + clickhouse/init/create_schema.sql | 300 ++++++++++++++++++ data/README.md | 49 +++ docker-compose.yml | 73 +++++ grafana/provisioning/dashboards/dashboard.yml | 11 + .../dashboards/options_analytics.json | 145 +++++++++ .../provisioning/datasources/datasource.yml | 17 + 11 files changed, 696 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/postCreateCommand.sh create mode 100644 .gitignore create mode 100644 README.md create mode 100644 clickhouse/init/create_schema.sql create mode 100644 data/README.md create mode 100644 docker-compose.yml create mode 100644 grafana/provisioning/dashboards/dashboard.yml create mode 100644 grafana/provisioning/dashboards/options_analytics.json create mode 100644 grafana/provisioning/datasources/datasource.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..5aac4f1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,22 @@ +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-22.04 + +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + apt-transport-https ca-certificates dirmngr wget gnupg2 \ + libssl-dev cmake ninja-build gdb python3 python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 \ + && echo "deb https://packages.clickhouse.com/deb stable main" | tee /etc/apt/sources.list.d/clickhouse.list \ + && apt-get update \ + && apt-get install -y clickhouse-client \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir \ + clickhouse-connect \ + clickhouse-driver \ + pandas \ + numpy \ + scipy \ + orjson \ + pytest \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..d4d5d6a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,28 @@ +{ + "name": "Options Pricing Engine", + "dockerComposeFile": "../docker-compose.yml", + "service": "workspace", + "workspaceFolder": "/workspace", + "forwardPorts": [8123, 9000, 3000], + "portsAttributes": { + "8123": { "label": "ClickHouse HTTP" }, + "9000": { "label": "ClickHouse TCP" }, + "3000": { "label": "Grafana" } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "twxs.cmake", + "ms-python.python", + "grafana.grafana-vscode" + ], + "settings": { + "terminal.integrated.defaultProfile.linux": "bash" + } + } + }, + "postCreateCommand": "bash .devcontainer/postCreateCommand.sh", + "remoteUser": "vscode" +} diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100644 index 0000000..8493a2f --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +echo "Waiting for ClickHouse to init..." +until clickhouse-client --host "${CLICKHOUSE_HOST:-clickhouse}" --query "SELECT 1" > /dev/null 2>&1; do + sleep 2 +done +echo "ClickHouse ready." + +echo "Tables in options_pricing:" +clickhouse-client --host "${CLICKHOUSE_HOST:-clickhouse}" \ + --query "SHOW TABLES FROM options_pricing" || true + +cat <<'EOF' + +------------------------------------------------------------------ + Dev environment ready. + ClickHouse HTTP : http://clickhouse:8123 (host: localhost:8123) + ClickHouse TCP : clickhouse:9000 (host: localhost:9000) + Grafana : http://localhost:3000 (admin / admin) + Raw data : ./data/CME/... +------------------------------------------------------------------ +EOF diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b25965 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +/data/** +!/data/ +!/data/README.md +!/data/.gitkeep + +.DS_Store + +__pycache__/ +*.py[cod] +.venv/ +venv/ +.env +*.egg-info/ +.pytest_cache/ +.ipynb_checkpoints/ + +/clickhouse/data/ +/clickhouse/logs/ +build/ +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..7fe1415 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# Option pricing engine + +## HOWTO + +Project should be opened via dev container: +- install docker and Dev container extension for VS code. +- press `f1` and type `reopen in dev container` +- all works! diff --git a/clickhouse/init/create_schema.sql b/clickhouse/init/create_schema.sql new file mode 100644 index 0000000..def4534 --- /dev/null +++ b/clickhouse/init/create_schema.sql @@ -0,0 +1,300 @@ +-- ============================================================================ +-- Options Pricing Engine — ClickHouse schema +-- ---------------------------------------------------------------------------- +-- Source: CME `ivcurve` JSON snapshots (see data/README.md). +-- +-- Design goals (each downstream task is called out where it is supported): +-- * Data-quality assessment -> Nullable quote fields + MATERIALIZED +-- validity flags + v_data_quality +-- * Enumerating option property -> normalized dimensions + v_option_universe +-- combinations +-- * Pricing model implementation -> v_pricing_inputs exposes every input a +-- model needs (S, K, r, T, style, quotes) +-- * Implied dividend rate -> v_put_call_pairs aligns the call & put at +-- each (underlying, expiration, strike) so +-- put-call parity can be solved downstream +-- +-- Modelling notes: +-- * The dataset is a point-in-time SNAPSHOT (`now` is identical across a +-- batch), but everything is keyed by `snapshot_time` so repeated snapshots +-- form a natural time series without schema changes. +-- * Grain choices: +-- underlyings : one row per underlying (slowly-changing dim) +-- option_chains : one row per (chain, snapshot) +-- underlying_quotes : one row per (underlying, snapshot) [fact] +-- option_quotes : one row per (option, snapshot) [fact] +-- * Prices are Nullable: a missing quote (`null` in JSON) is semantically +-- different from a 0 price and that difference is what the data-quality +-- task measures. +-- ============================================================================ + +CREATE DATABASE IF NOT EXISTS options_pricing; + +-- ============================================================================ +-- 1. DIMENSION: underlyings +-- One row per underlying futures contract (folder = CME-). +-- ============================================================================ +CREATE TABLE IF NOT EXISTS options_pricing.underlyings +( + underlying_id String, -- e.g. "CME:6AM2026" + market LowCardinality(String), -- e.g. "CME" + symbol String, -- e.g. "6AM2026" + underlying_type LowCardinality(String) -- e.g. "futures" +) +ENGINE = ReplacingMergeTree() +ORDER BY underlying_id; + +-- ============================================================================ +-- 2. DIMENSION: option_chains +-- One row per (underlying, expiration) per snapshot. Holds the contract-level +-- attributes that pricing/parity need (rate, exercise style, expiration). +-- ============================================================================ +CREATE TABLE IF NOT EXISTS options_pricing.option_chains +( + chain_id String, -- "CME:6AM2026;5AD;20260529" + underlying_id String, -- "CME:6AM2026" + product_code LowCardinality(String), -- "5AD" + expiration_date Date, -- parsed from expirationDate + expiration_date_raw UInt32, -- 20260529 (as delivered) + exercise_style LowCardinality(String), -- european | american + underlying_type LowCardinality(String), -- futures + interest_rate Float64, -- ivcurve.interestRate + is_paying_dividends UInt8, -- ivcurve.isPayingDividends + snapshot_time DateTime64(9, 'UTC'), -- ivcurve.now + source_file String +) +ENGINE = ReplacingMergeTree(snapshot_time) +ORDER BY (chain_id, snapshot_time); + +-- ============================================================================ +-- 3. FACT: underlying_quotes +-- The underlying (futures) bid/ask/last at each snapshot. This is the spot/ +-- forward input S for pricing and for put-call parity. +-- ============================================================================ +CREATE TABLE IF NOT EXISTS options_pricing.underlying_quotes +( + snapshot_time DateTime64(9, 'UTC'), + underlying_id LowCardinality(String), + mode LowCardinality(String), -- realtime | snapshot + quote_time Nullable(DateTime64(9, 'UTC')), -- marketData.underlying.time + + bid_price Nullable(Float64), + bid_size Nullable(UInt32), + bid_time Nullable(DateTime64(9, 'UTC')), + + ask_price Nullable(Float64), + ask_size Nullable(UInt32), + ask_time Nullable(DateTime64(9, 'UTC')), + + last_price Nullable(Float64), + last_size Nullable(UInt32), + last_time Nullable(DateTime64(9, 'UTC')), + + -- Derived convenience columns (Nullable propagates when a side is missing). + mid_price Nullable(Float64) MATERIALIZED (bid_price + ask_price) / 2, + spread Nullable(Float64) MATERIALIZED ask_price - bid_price +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(snapshot_time) +ORDER BY (underlying_id, snapshot_time); + +-- ============================================================================ +-- 4. FACT: option_quotes (core analytical table) +-- One row per option (put/call at a strike) per snapshot. A few chain-level +-- attributes are denormalized here (exercise_style, interest_rate, +-- expiration_date) so the common pricing/quality queries need no joins. +-- +-- ORDER BY groups every strike of an expiration together and keeps calls & +-- puts adjacent -> cheap self-join for put-call parity and cheap scans for +-- per-chain quality metrics and combination enumeration. +-- ============================================================================ +CREATE TABLE IF NOT EXISTS options_pricing.option_quotes +( + snapshot_time DateTime64(9, 'UTC'), + chain_id LowCardinality(String), + underlying_id LowCardinality(String), + product_code LowCardinality(String), + expiration_date Date, + strike Float64, + option_type Enum8('put' = 0, 'call' = 1), + option_id String, -- "CME:5AD260529P0.65" + + -- denormalized chain attributes (pricing inputs, avoid a join on hot path) + exercise_style LowCardinality(String), + interest_rate Float64, + + mode LowCardinality(String), -- realtime | snapshot + quote_time Nullable(DateTime64(9, 'UTC')), + + bid_price Nullable(Float64), + bid_size Nullable(UInt32), + bid_time Nullable(DateTime64(9, 'UTC')), + + ask_price Nullable(Float64), + ask_size Nullable(UInt32), + ask_time Nullable(DateTime64(9, 'UTC')), + + last_price Nullable(Float64), + last_size Nullable(UInt32), + last_time Nullable(DateTime64(9, 'UTC')), + + source_file String, + + -- ---- derived columns (data-quality + pricing convenience) -------------- + mid_price Nullable(Float64) MATERIALIZED (bid_price + ask_price) / 2, + spread Nullable(Float64) MATERIALIZED ask_price - bid_price, + spread_bps Nullable(Float64) MATERIALIZED + if((bid_price + ask_price) / 2 > 0, + (ask_price - bid_price) / ((bid_price + ask_price) / 2) * 10000, + NULL), + + -- Quote health flags used by the data-quality task. + has_bid UInt8 MATERIALIZED bid_price IS NOT NULL, + has_ask UInt8 MATERIALIZED ask_price IS NOT NULL, + is_two_sided UInt8 MATERIALIZED bid_price IS NOT NULL AND ask_price IS NOT NULL, + is_crossed UInt8 MATERIALIZED bid_price IS NOT NULL AND ask_price IS NOT NULL + AND bid_price > ask_price, + is_valid_quote UInt8 MATERIALIZED + bid_price IS NOT NULL AND ask_price IS NOT NULL + AND bid_price > 0 AND ask_price > 0 + AND ask_price >= bid_price + AND bid_size > 0 AND ask_size > 0, + seconds_since_last_trade Nullable(Int64) MATERIALIZED + if(last_time IS NULL, NULL, dateDiff('second', last_time, snapshot_time)) +) +ENGINE = MergeTree() +PARTITION BY toYYYYMM(expiration_date) +ORDER BY (underlying_id, expiration_date, option_type, strike, snapshot_time); + +-- ============================================================================ +-- 5. VIEWS +-- Thin, task-oriented projections. They contain NO pricing/greek/parity math +-- (that is the downstream tasks' job) — they only align and expose the +-- inputs those tasks need. +-- ============================================================================ + +-- 5a. Option universe / property-combination enumeration --------------------- +-- Supports: "Enumerating all combinations of options properties". +CREATE VIEW IF NOT EXISTS options_pricing.v_option_universe AS +SELECT + underlying_id, + product_code, + expiration_date, + exercise_style, + option_type, + strike, + count() AS snapshots_seen, + max(snapshot_time) AS last_snapshot +FROM options_pricing.option_quotes +GROUP BY + underlying_id, product_code, expiration_date, + exercise_style, option_type, strike; + +-- 5b. Pricing inputs --------------------------------------------------------- +-- Supports: "Implementation of the pricing models". +-- Joins each option to its chain and to the underlying quote of the same +-- snapshot, and derives time-to-expiry. No option value is computed here. +CREATE VIEW IF NOT EXISTS options_pricing.v_pricing_inputs AS +SELECT + oq.snapshot_time AS snapshot_time, + oq.chain_id AS chain_id, + oq.underlying_id AS underlying_id, + oq.option_id AS option_id, + oq.option_type AS option_type, + oq.strike AS strike, + oq.exercise_style AS exercise_style, + oq.interest_rate AS interest_rate, + oq.expiration_date AS expiration_date, + -- time to expiry in years (ACT/365), computed from the snapshot instant + dateDiff('second', oq.snapshot_time, toDateTime64(oq.expiration_date, 9, 'UTC')) + / (365.0 * 24 * 3600) AS time_to_expiry_years, + oq.bid_price AS option_bid, + oq.ask_price AS option_ask, + oq.mid_price AS option_mid, + oq.last_price AS option_last, + uq.bid_price AS underlying_bid, + uq.ask_price AS underlying_ask, + uq.mid_price AS underlying_mid, + uq.last_price AS underlying_last, + oq.is_valid_quote AS is_valid_quote +FROM options_pricing.option_quotes AS oq +LEFT JOIN options_pricing.underlying_quotes AS uq + ON oq.underlying_id = uq.underlying_id + AND oq.snapshot_time = uq.snapshot_time; + +-- 5c. Put-call pairs --------------------------------------------------------- +-- Supports: "Implementing logic for computing the implied dividend rate". +-- Aligns the call and put at the same (underlying, expiration, strike, +-- snapshot) with the underlying price, the rate and time-to-expiry — i.e. +-- every input of put-call parity C - P = S*e^{-qT} - K*e^{-rT}. +-- The implied dividend q is intentionally NOT solved here. +CREATE VIEW IF NOT EXISTS options_pricing.v_put_call_pairs AS +SELECT + c.snapshot_time AS snapshot_time, + c.underlying_id AS underlying_id, + c.chain_id AS chain_id, + c.expiration_date AS expiration_date, + c.strike AS strike, + c.exercise_style AS exercise_style, + c.interest_rate AS interest_rate, + dateDiff('second', c.snapshot_time, toDateTime64(c.expiration_date, 9, 'UTC')) + / (365.0 * 24 * 3600) AS time_to_expiry_years, + c.mid_price AS call_mid, + p.mid_price AS put_mid, + c.bid_price AS call_bid, + c.ask_price AS call_ask, + p.bid_price AS put_bid, + p.ask_price AS put_ask, + uq.mid_price AS underlying_mid, + uq.last_price AS underlying_last +FROM options_pricing.option_quotes AS c +INNER JOIN options_pricing.option_quotes AS p + ON c.underlying_id = p.underlying_id + AND c.expiration_date = p.expiration_date + AND c.strike = p.strike + AND c.snapshot_time = p.snapshot_time + AND c.option_type = 'call' + AND p.option_type = 'put' +LEFT JOIN options_pricing.underlying_quotes AS uq + ON c.underlying_id = uq.underlying_id + AND c.snapshot_time = uq.snapshot_time; + +-- 5d. Per-chain data-quality metrics ----------------------------------------- +-- Supports: "Assessing input data quality" and the Grafana dashboard. +CREATE VIEW IF NOT EXISTS options_pricing.chain_quality_metrics AS +SELECT + chain_id, + underlying_id, + expiration_date, + exercise_style, + snapshot_time AS timestamp, + count() AS total_quotes, + countIf(option_type = 'call') AS call_count, + countIf(option_type = 'put') AS put_count, + countIf(is_valid_quote) AS valid_quotes_count, + countIf(NOT is_valid_quote) AS invalid_quotes_count, + countIf(is_two_sided) AS two_sided_count, + countIf(is_crossed) AS crossed_count, + countIf(bid_price IS NULL AND ask_price IS NULL) AS empty_quote_count, + round(avgIf(spread_bps, is_valid_quote), 2) AS avg_spread_bps, + round(medianIf(spread_bps, is_valid_quote), 2) AS median_spread_bps, + min(strike) AS min_strike, + max(strike) AS max_strike +FROM options_pricing.option_quotes +GROUP BY + chain_id, underlying_id, expiration_date, exercise_style, snapshot_time; + +-- 5e. Snapshot-level data-quality overview ----------------------------------- +CREATE VIEW IF NOT EXISTS options_pricing.v_data_quality AS +SELECT + snapshot_time AS timestamp, + uniqExact(underlying_id) AS underlyings, + uniqExact(chain_id) AS chains, + count() AS total_quotes, + countIf(is_valid_quote) AS valid_quotes, + round(100.0 * countIf(is_valid_quote) / count(), 2) AS valid_pct, + countIf(is_crossed) AS crossed_quotes, + countIf(bid_price IS NULL AND ask_price IS NULL) AS empty_quotes, + round(avgIf(spread_bps, is_valid_quote), 2) AS avg_spread_bps +FROM options_pricing.option_quotes +GROUP BY snapshot_time; diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..86444e2 --- /dev/null +++ b/data/README.md @@ -0,0 +1,49 @@ +# `data/` — raw market data (git-ignored) + +This directory holds raw option-chain snapshots consumed by the parser. Its +contents are **ignored by git** (see the repo `.gitignore`) because the files are +large and reproducible from the data provider. + +## Layout + +``` +data/ +└── CME/ + └── CME-/ # one folder per underlying futures contract + └── _.json # one file per option chain (expiration) +``` + +Example: `data/CME/CME-6AM2026/5AD_20260529.json` + +## File format (`ivcurve` snapshot) + +Each JSON file is a single snapshot of one option chain: + +```jsonc +{ + "ivcurve": { + "id": "CME:6AM2026;5AD;20260529", // market:underlying;product;expiration + "now": "2026-05-25T14:59:01.800Z", // snapshot time (same across a batch) + "expirationDate": 20260529, // YYYYMMDD as an integer + "exerciseStyle": "european", // european | american + "interestRate": 0.01, + "underlyingType": "futures", + "isPayingDividends": false, + "marketData": { + "underlying": { "id": "CME:6AM2026", "mode": "realtime", + "bid": {"price": .., "size": .., "time": ".."}, + "ask": {...}, "last": {...} }, + "puts": { "": { "id": "..", "mode": "..", + "bid": {...}|null, "ask": {...}|null, "last": {...}|null } }, + "calls": { "": { ... } } + } + } +} +``` + +A `bid`/`ask`/`last` quote is either `null` (no quote) or `{price, size, time}`. +This nullability is preserved in the ClickHouse schema so it can be measured by +the data-quality task. + +The parser (implemented separately) reads these files and loads them into the +ClickHouse tables defined in `clickhouse/init/create_schema.sql`. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1755b43 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,73 @@ +name: options-pricing-engine + +services: + clickhouse: + image: clickhouse/clickhouse-server:24.8 + container_name: ope-clickhouse + hostname: clickhouse + environment: + CLICKHOUSE_DB: options_pricing + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + ports: + - "8123:8123" + - "9000:9000" + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - clickhouse-data:/var/lib/clickhouse + - ./clickhouse/init:/docker-entrypoint-initdb.d:ro + + - ./data:/data:ro + healthcheck: + test: ["CMD", "clickhouse-client", "--query", "SELECT 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + + grafana: + image: grafana/grafana:11.2.0 + container_name: ope-grafana + ports: + - "3000:3000" + environment: + GF_INSTALL_PLUGINS: grafana-clickhouse-datasource + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_USERS_DEFAULT_THEME: dark + volumes: + - grafana-data:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:ro + depends_on: + clickhouse: + condition: service_healthy + restart: unless-stopped + + workspace: + build: + context: . + dockerfile: .devcontainer/Dockerfile + container_name: ope-workspace + command: sleep infinity + volumes: + - .:/workspace:cached + working_dir: /workspace + environment: + CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_PORT: "9000" + CLICKHOUSE_HTTP_PORT: "8123" + CLICKHOUSE_DB: options_pricing + depends_on: + clickhouse: + condition: service_healthy + +volumes: + clickhouse-data: + grafana-data: diff --git a/grafana/provisioning/dashboards/dashboard.yml b/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..9ca6877 --- /dev/null +++ b/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/grafana/provisioning/dashboards/options_analytics.json b/grafana/provisioning/dashboards/options_analytics.json new file mode 100644 index 0000000..6ca9706 --- /dev/null +++ b/grafana/provisioning/dashboards/options_analytics.json @@ -0,0 +1,145 @@ +{ + "uid": "ope-source-data", + "title": "Options Source Data & Quality", + "tags": ["options", "clickhouse", "source-data"], + "style": "dark", + "editable": true, + "schemaVersion": 39, + "time": { "from": "now-6h", "to": "now" }, + "templating": { + "list": [ + { + "name": "chain_id", + "label": "Option chain", + "type": "query", + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "query": "SELECT DISTINCT chain_id FROM options_pricing.option_quotes ORDER BY chain_id", + "refresh": 2, + "includeAll": false, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "Underlyings", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT uniqExact(underlying_id) AS value FROM options_pricing.option_quotes" } + ], + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^value$/" }, "colorMode": "value" } + }, + { + "type": "stat", + "title": "Option chains", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT uniqExact(chain_id) AS value FROM options_pricing.option_quotes" } + ], + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^value$/" }, "colorMode": "value" } + }, + { + "type": "stat", + "title": "Total option quotes", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT count() AS value FROM options_pricing.option_quotes" } + ], + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^value$/" }, "colorMode": "value" } + }, + { + "type": "gauge", + "title": "Valid quotes %", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT round(100.0 * countIf(is_valid_quote) / count(), 2) AS value FROM options_pricing.option_quotes" } + ], + "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "thresholds": { "steps": [ { "color": "red", "value": 0 }, { "color": "orange", "value": 50 }, { "color": "green", "value": 80 } ] } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^value$/" } } + }, + { + "type": "stat", + "title": "Crossed markets", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT countIf(is_crossed) AS value FROM options_pricing.option_quotes" } + ], + "fieldConfig": { "defaults": { "thresholds": { "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 1 } ] } } }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "/^value$/" }, "colorMode": "background" } + }, + { + "type": "piechart", + "title": "Quotes by exercise style", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 4 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT exercise_style, count() AS quotes FROM options_pricing.option_quotes GROUP BY exercise_style ORDER BY quotes DESC" } + ], + "options": { "legend": { "displayMode": "table", "placement": "right", "values": ["value", "percent"] } } + }, + { + "type": "barchart", + "title": "Quote health breakdown", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 4 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT 'valid' AS category, countIf(is_valid_quote) AS n FROM options_pricing.option_quotes UNION ALL SELECT 'two-sided (not valid)', countIf(is_two_sided AND NOT is_valid_quote) FROM options_pricing.option_quotes UNION ALL SELECT 'one-sided', countIf((has_bid AND NOT has_ask) OR (has_ask AND NOT has_bid)) FROM options_pricing.option_quotes UNION ALL SELECT 'empty', countIf(bid_price IS NULL AND ask_price IS NULL) FROM options_pricing.option_quotes UNION ALL SELECT 'crossed', countIf(is_crossed) FROM options_pricing.option_quotes" } + ], + "options": { "xField": "category", "orientation": "horizontal", "legend": { "showLegend": false } } + }, + { + "type": "piechart", + "title": "Quotes by mode", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT mode, count() AS quotes FROM options_pricing.option_quotes GROUP BY mode ORDER BY quotes DESC" } + ], + "options": { "legend": { "displayMode": "table", "placement": "right", "values": ["value", "percent"] } } + }, + { + "type": "barchart", + "title": "Top 20 underlyings by option count", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT underlying_id, count() AS options FROM options_pricing.option_quotes GROUP BY underlying_id ORDER BY options DESC LIMIT 20" } + ], + "options": { "xField": "underlying_id", "orientation": "horizontal", "legend": { "showLegend": false } } + }, + { + "type": "table", + "title": "Data quality by chain", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT chain_id, exercise_style, total_quotes, valid_quotes_count, crossed_count, empty_quote_count, avg_spread_bps FROM options_pricing.chain_quality_metrics ORDER BY total_quotes DESC LIMIT 200" } + ] + }, + { + "type": "barchart", + "title": "Mid price by strike — $chain_id", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 21 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT toString(strike) AS strike, anyIf(mid_price, option_type = 'call') AS call_mid, anyIf(mid_price, option_type = 'put') AS put_mid FROM options_pricing.option_quotes WHERE chain_id = '$chain_id' GROUP BY strike ORDER BY toFloat64(strike)" } + ], + "options": { "xField": "strike", "legend": { "showLegend": true, "placement": "bottom" } } + }, + { + "type": "table", + "title": "Raw quotes — $chain_id", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 21 }, + "datasource": { "type": "grafana-clickhouse-datasource", "uid": "clickhouse" }, + "targets": [ + { "refId": "A", "rawSql": "SELECT option_type, strike, bid_price, ask_price, mid_price, spread_bps, is_valid_quote, mode FROM options_pricing.option_quotes WHERE chain_id = '$chain_id' ORDER BY option_type, strike" } + ] + } + ] +} diff --git a/grafana/provisioning/datasources/datasource.yml b/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..6721dd0 --- /dev/null +++ b/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,17 @@ +apiVersion: 1 + +datasources: + - name: ClickHouse + uid: clickhouse + type: grafana-clickhouse-datasource + access: proxy + isDefault: true + editable: true + jsonData: + host: clickhouse + port: 9000 + protocol: native + defaultDatabase: options_pricing + username: default + secureJsonData: + password: "" From c5f90ad7317c694138e919b2aea11045e7b52608 Mon Sep 17 00:00:00 2001 From: OlegOpyakin <145427708+OlegOpyakin@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:34:56 +0300 Subject: [PATCH 2/5] Add data parser --- CMakeLists.txt | 52 ++++ README.md | 46 +++ clickhouse/init/create_schema.sql | 26 +- include/ope/blocking_queue.hpp | 61 ++++ include/ope/ch_config.hpp | 44 +++ include/ope/inserter.hpp | 33 ++ include/ope/ivcurve_parser.hpp | 17 ++ include/ope/parsed_rows.hpp | 81 +++++ include/ope/time_util.hpp | 23 ++ src/inserter.cpp | 489 ++++++++++++++++++++++++++++++ src/ivcurve_parser.cpp | 176 +++++++++++ src/main.cpp | 78 +++++ src/time_util.cpp | 61 ++++ 13 files changed, 1176 insertions(+), 11 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 include/ope/blocking_queue.hpp create mode 100644 include/ope/ch_config.hpp create mode 100644 include/ope/inserter.hpp create mode 100644 include/ope/ivcurve_parser.hpp create mode 100644 include/ope/parsed_rows.hpp create mode 100644 include/ope/time_util.hpp create mode 100644 src/inserter.cpp create mode 100644 src/ivcurve_parser.cpp create mode 100644 src/main.cpp create mode 100644 src/time_util.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..5a6753e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,52 @@ +cmake_minimum_required(VERSION 3.20) +project(options_pricing_engine LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +include(FetchContent) + +# --- nlohmann/json (header-only JSON parser) -------------------------------- +FetchContent_Declare( + nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz +) + +# --- clickhouse-cpp (native TCP client, block/batch inserts) ---------------- +FetchContent_Declare( + clickhouse_cpp + GIT_REPOSITORY https://github.com/ClickHouse/clickhouse-cpp.git + GIT_TAG v2.5.1 + GIT_SHALLOW TRUE +) + +FetchContent_MakeAvailable(nlohmann_json clickhouse_cpp) + +find_package(Threads REQUIRED) + +add_executable(ope + src/main.cpp + src/time_util.cpp + src/ivcurve_parser.cpp + src/inserter.cpp +) + +target_include_directories(ope + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + # clickhouse-cpp public headers + its bundled contrib (abseil) headers + ${clickhouse_cpp_SOURCE_DIR} + ${clickhouse_cpp_SOURCE_DIR}/contrib/absl +) + +target_link_libraries(ope + PRIVATE + clickhouse-cpp-lib + nlohmann_json::nlohmann_json + Threads::Threads +) diff --git a/README.md b/README.md index 7fe1415..0f620d4 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,49 @@ Project should be opened via dev container: - install docker and Dev container extension for VS code. - press `f1` and type `reopen in dev container` - all works! + +The dev container brings up three services (see `docker-compose.yml`): +`clickhouse` (analytical store, schema auto-applied from `clickhouse/init/`), +`grafana` (http://localhost:3000, admin/admin) and the `workspace` you code in. + +## Data + +Raw CME `ivcurve` JSON snapshots live under `data/` (git-ignored). See +`data/README.md` for the folder layout and file format. + +## Building the parser + +From inside the dev container: + +```bash +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build -j +``` + +Dependencies (`nlohmann/json`, `clickhouse-cpp`) are fetched automatically by +CMake, so the first configure needs network access. + +## Running the importer + +The `ope` binary reads commands from an interactive event loop; `insert` imports +a file or a directory (walked recursively) into ClickHouse: + +```bash +./build/ope # interactive console +> insert data/CME # import everything +> insert data/CME/CME-6AM2026/5AD_20260529.json # import one file +> quit + +./build/ope insert data/CME # non-interactive one-shot (same command) +``` + +Files are distributed to a pool of worker threads through a blocking queue; each +worker parses JSON and streams rows into ClickHouse in batches. Connection and +tuning are read from the environment (defaults in parentheses): + +| Variable | Purpose | +| --- | --- | +| `CLICKHOUSE_HOST` (`localhost`), `CLICKHOUSE_PORT` (`9000`) | server address (native TCP) | +| `CLICKHOUSE_DB` (`options_pricing`), `CLICKHOUSE_USER` (`default`), `CLICKHOUSE_PASSWORD` (empty) | credentials | +| `OPE_WORKERS` (CPU count) | number of parser/inserter threads | +| `OPE_BATCH_SIZE` (`50000`) | rows accumulated before a batch `INSERT` | diff --git a/clickhouse/init/create_schema.sql b/clickhouse/init/create_schema.sql index def4534..878c01c 100644 --- a/clickhouse/init/create_schema.sql +++ b/clickhouse/init/create_schema.sql @@ -68,12 +68,17 @@ ORDER BY (chain_id, snapshot_time); -- ============================================================================ -- 3. FACT: underlying_quotes --- The underlying (futures) bid/ask/last at each snapshot. This is the spot/ --- forward input S for pricing and for put-call parity. +-- The underlying (futures) bid/ask/last as captured in each ivcurve file. +-- Grain = one row per (chain, snapshot): every ivcurve file carries its own +-- underlying snapshot, and those captures differ slightly across a +-- underlying's files (same instant, independently sampled). Keeping chain_id +-- lets pricing/parity use the underlying price from the SAME file as the +-- option, instead of an arbitrary one — see the joins in the views below. -- ============================================================================ CREATE TABLE IF NOT EXISTS options_pricing.underlying_quotes ( snapshot_time DateTime64(9, 'UTC'), + chain_id LowCardinality(String), -- file this capture came from underlying_id LowCardinality(String), mode LowCardinality(String), -- realtime | snapshot quote_time Nullable(DateTime64(9, 'UTC')), -- marketData.underlying.time @@ -96,7 +101,7 @@ CREATE TABLE IF NOT EXISTS options_pricing.underlying_quotes ) ENGINE = MergeTree() PARTITION BY toYYYYMM(snapshot_time) -ORDER BY (underlying_id, snapshot_time); +ORDER BY (underlying_id, chain_id, snapshot_time); -- ============================================================================ -- 4. FACT: option_quotes (core analytical table) @@ -219,7 +224,7 @@ SELECT oq.is_valid_quote AS is_valid_quote FROM options_pricing.option_quotes AS oq LEFT JOIN options_pricing.underlying_quotes AS uq - ON oq.underlying_id = uq.underlying_id + ON oq.chain_id = uq.chain_id AND oq.snapshot_time = uq.snapshot_time; -- 5c. Put-call pairs --------------------------------------------------------- @@ -249,14 +254,13 @@ SELECT uq.last_price AS underlying_last FROM options_pricing.option_quotes AS c INNER JOIN options_pricing.option_quotes AS p - ON c.underlying_id = p.underlying_id - AND c.expiration_date = p.expiration_date - AND c.strike = p.strike - AND c.snapshot_time = p.snapshot_time - AND c.option_type = 'call' - AND p.option_type = 'put' + ON c.chain_id = p.chain_id + AND c.strike = p.strike + AND c.snapshot_time = p.snapshot_time + AND c.option_type = 'call' + AND p.option_type = 'put' LEFT JOIN options_pricing.underlying_quotes AS uq - ON c.underlying_id = uq.underlying_id + ON c.chain_id = uq.chain_id AND c.snapshot_time = uq.snapshot_time; -- 5d. Per-chain data-quality metrics ----------------------------------------- diff --git a/include/ope/blocking_queue.hpp b/include/ope/blocking_queue.hpp new file mode 100644 index 0000000..ded9f45 --- /dev/null +++ b/include/ope/blocking_queue.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace ope { + +// A simple multi-producer / multi-consumer blocking queue. +// +// Producers push work items; consumers call pop() which blocks until an item is +// available or the queue is closed and drained. Once close() is called, pop() +// keeps returning the remaining buffered items and only then reports "empty". +template +class BlockingQueue { + public: + void push(T value) { + { + std::lock_guard lock(mutex_); + queue_.push(std::move(value)); + } + cv_.notify_one(); + } + + // Blocks until an item can be returned. Returns std::nullopt only when the + // queue is closed AND empty, which is the signal for a worker to stop. + std::optional pop() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return closed_ || !queue_.empty(); }); + if (queue_.empty()) { + return std::nullopt; // closed and drained + } + T value = std::move(queue_.front()); + queue_.pop(); + return value; + } + + // Signals that no more items will be pushed. Wakes all waiting consumers. + void close() { + { + std::lock_guard lock(mutex_); + closed_ = true; + } + cv_.notify_all(); + } + + std::size_t size() const { + std::lock_guard lock(mutex_); + return queue_.size(); + } + + private: + mutable std::mutex mutex_; + std::condition_variable cv_; + std::queue queue_; + bool closed_ = false; +}; + +} // namespace ope diff --git a/include/ope/ch_config.hpp b/include/ope/ch_config.hpp new file mode 100644 index 0000000..248884b --- /dev/null +++ b/include/ope/ch_config.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include + +namespace ope { + +// ClickHouse connection settings + parser tuning, sourced from environment +// variables so the same binary works inside the dev container (host +// "clickhouse") and against a locally forwarded port (host "localhost"). +struct ChConfig { + std::string host = "localhost"; + uint16_t port = 9000; + std::string database = "options_pricing"; + std::string user = "default"; + std::string password = ""; + + // How many rows to accumulate before flushing a batch INSERT. + std::size_t batch_size = 50000; + // Number of parser/inserter worker threads. + std::size_t workers = 0; // 0 -> auto (hardware_concurrency) + + static ChConfig from_env() { + ChConfig cfg; + if (const char* v = std::getenv("CLICKHOUSE_HOST")) cfg.host = v; + if (const char* v = std::getenv("CLICKHOUSE_PORT")) cfg.port = static_cast(std::atoi(v)); + if (const char* v = std::getenv("CLICKHOUSE_DB")) cfg.database = v; + if (const char* v = std::getenv("CLICKHOUSE_USER")) cfg.user = v; + if (const char* v = std::getenv("CLICKHOUSE_PASSWORD")) cfg.password = v; + if (const char* v = std::getenv("OPE_BATCH_SIZE")) cfg.batch_size = static_cast(std::strtoull(v, nullptr, 10)); + if (const char* v = std::getenv("OPE_WORKERS")) cfg.workers = static_cast(std::strtoull(v, nullptr, 10)); + + if (cfg.workers == 0) { + unsigned hw = std::thread::hardware_concurrency(); + cfg.workers = hw == 0 ? 4 : hw; + } + if (cfg.batch_size == 0) cfg.batch_size = 50000; + return cfg; + } +}; + +} // namespace ope diff --git a/include/ope/inserter.hpp b/include/ope/inserter.hpp new file mode 100644 index 0000000..d0e9e09 --- /dev/null +++ b/include/ope/inserter.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "ope/ch_config.hpp" + +namespace ope { + +// Imports CME ivcurve JSON data into ClickHouse. +// +// Usage from the main event loop: +// Inserter inserter(ChConfig::from_env()); +// bool ok = inserter.insert("data/CME"); // directory (recursive) +// bool ok = inserter.insert("data/CME/.../x.json"); // single file +// +// The path may point to a single file or a directory; directories are walked +// recursively and every *.json file is imported. Files are distributed to a +// pool of worker threads through a blocking queue; each worker parses files and +// streams rows into ClickHouse in batches (see ChConfig::batch_size). +// +// Returns true only if every discovered file was parsed and inserted without +// error. A human-readable summary is printed to stdout/stderr regardless. +class Inserter { + public: + explicit Inserter(ChConfig config); + + bool insert(const std::string& path); + + private: + ChConfig config_; +}; + +} // namespace ope diff --git a/include/ope/ivcurve_parser.hpp b/include/ope/ivcurve_parser.hpp new file mode 100644 index 0000000..36586d6 --- /dev/null +++ b/include/ope/ivcurve_parser.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include "ope/parsed_rows.hpp" + +namespace ope { + +// Parses a single CME ivcurve JSON file into row structs ready for insertion. +// `source_file` is stored verbatim on every emitted row for lineage. +// Returns std::nullopt (and leaves `error` populated) if the file cannot be +// read or does not match the expected ivcurve shape. +std::optional parse_ivcurve_file(const std::string& path, + std::string& error); + +} // namespace ope diff --git a/include/ope/parsed_rows.hpp b/include/ope/parsed_rows.hpp new file mode 100644 index 0000000..b6065c1 --- /dev/null +++ b/include/ope/parsed_rows.hpp @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include + +namespace ope { + +// One side of a quote (bid / ask / last). All fields are optional because the +// source JSON delivers `null` when there is no quote on that side. +struct QuoteSide { + std::optional price; + std::optional size; + std::optional time_ns; // nanoseconds since epoch +}; + +// -> options_pricing.underlyings (dimension) +struct UnderlyingRow { + std::string underlying_id; // "CME:6AM2026" + std::string market; // "CME" + std::string symbol; // "6AM2026" + std::string underlying_type; // "futures" +}; + +// -> options_pricing.option_chains (dimension, one per file) +struct ChainRow { + std::string chain_id; // "CME:6AM2026;5AD;20260529" + std::string underlying_id; + std::string product_code; // "5AD" + int32_t expiration_days = 0; // days since epoch (Date) + uint32_t expiration_raw = 0; // 20260529 + std::string exercise_style; // european | american + std::string underlying_type; + double interest_rate = 0.0; + uint8_t is_paying_dividends = 0; + int64_t snapshot_ns = 0; + std::string source_file; +}; + +// -> options_pricing.underlying_quotes (fact, one per file) +struct UnderlyingQuoteRow { + int64_t snapshot_ns = 0; + std::string chain_id; + std::string underlying_id; + std::string mode; // realtime | snapshot + std::optional quote_time_ns; + QuoteSide bid; + QuoteSide ask; + QuoteSide last; +}; + +// -> options_pricing.option_quotes (fact, many per file) +struct OptionRow { + int64_t snapshot_ns = 0; + std::string chain_id; + std::string underlying_id; + std::string product_code; + int32_t expiration_days = 0; + double strike = 0.0; + int8_t option_type = 0; // 0 = put, 1 = call (matches Enum8 in schema) + std::string option_id; + std::string exercise_style; + double interest_rate = 0.0; + std::string mode; + std::optional quote_time_ns; + QuoteSide bid; + QuoteSide ask; + QuoteSide last; + std::string source_file; +}; + +// The full parsed content of a single ivcurve JSON file. +struct ParsedFile { + UnderlyingRow underlying; + ChainRow chain; + UnderlyingQuoteRow underlying_quote; + std::vector options; +}; + +} // namespace ope diff --git a/include/ope/time_util.hpp b/include/ope/time_util.hpp new file mode 100644 index 0000000..386a6c0 --- /dev/null +++ b/include/ope/time_util.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +namespace ope::timeutil { + +// Parse an ISO-8601 UTC timestamp into nanoseconds since the Unix epoch. +// Accepts forms like: +// "2026-05-25T14:59:01.800326283Z" (up to 9 fractional digits) +// "2026-05-25T14:58:59Z" (no fraction) +// "1970-01-01T00:00:00Z" (epoch) +// Returns 0 for the epoch string and for anything it fails to parse. +int64_t parse_iso8601_ns(std::string_view s); + +// Convert a CME expirationDate integer (YYYYMMDD, e.g. 20260529) to the number +// of days since the Unix epoch (what ClickHouse's Date column stores) and to +// midnight-UTC seconds (handy for ColumnDate::Append which takes seconds). +int32_t yyyymmdd_to_days(uint32_t yyyymmdd); +std::time_t yyyymmdd_to_seconds(uint32_t yyyymmdd); + +} // namespace ope::timeutil diff --git a/src/inserter.cpp b/src/inserter.cpp new file mode 100644 index 0000000..470bf0a --- /dev/null +++ b/src/inserter.cpp @@ -0,0 +1,489 @@ +#include "ope/inserter.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ope/blocking_queue.hpp" +#include "ope/ivcurve_parser.hpp" + +namespace ope { +namespace { + +using namespace clickhouse; +namespace fs = std::filesystem; + +using LCString = ColumnLowCardinalityT; + +constexpr int64_t kSecondsPerDay = 86400; + +// --- Nullable column builders ---------------------------------------------- +// clickhouse-cpp represents Nullable(T) as a nested value column + a UInt8 null +// mask. These helpers keep the two in lock-step and hand back a wrapped column +// at flush time. +struct NullableFloat64 { + std::shared_ptr data = std::make_shared(); + std::shared_ptr nulls = std::make_shared(); + void append(const std::optional& v) { + data->Append(v.value_or(0.0)); + nulls->Append(v.has_value() ? 0 : 1); + } + ColumnRef column() { return std::make_shared(data, nulls); } + void reset() { + data = std::make_shared(); + nulls = std::make_shared(); + } +}; + +struct NullableUInt32 { + std::shared_ptr data = std::make_shared(); + std::shared_ptr nulls = std::make_shared(); + void append(const std::optional& v) { + data->Append(v.value_or(0)); + nulls->Append(v.has_value() ? 0 : 1); + } + ColumnRef column() { return std::make_shared(data, nulls); } + void reset() { + data = std::make_shared(); + nulls = std::make_shared(); + } +}; + +struct NullableDateTime64 { + std::shared_ptr data = std::make_shared(9); + std::shared_ptr nulls = std::make_shared(); + void append(const std::optional& v) { + data->Append(v.value_or(0)); + nulls->Append(v.has_value() ? 0 : 1); + } + ColumnRef column() { return std::make_shared(data, nulls); } + void reset() { + data = std::make_shared(9); + nulls = std::make_shared(); + } +}; + +// --- Per-table batch builders ---------------------------------------------- + +struct UnderlyingsBatch { + std::shared_ptr underlying_id = std::make_shared(); + std::shared_ptr market = std::make_shared(); + std::shared_ptr symbol = std::make_shared(); + std::shared_ptr underlying_type = std::make_shared(); + std::size_t count = 0; + + void append(const UnderlyingRow& r) { + underlying_id->Append(r.underlying_id); + market->Append(r.market); + symbol->Append(r.symbol); + underlying_type->Append(r.underlying_type); + ++count; + } + + void flush(Client& client) { + if (count == 0) return; + Block block; + block.AppendColumn("underlying_id", underlying_id); + block.AppendColumn("market", market); + block.AppendColumn("symbol", symbol); + block.AppendColumn("underlying_type", underlying_type); + client.Insert("options_pricing.underlyings", block); + underlying_id = std::make_shared(); + market = std::make_shared(); + symbol = std::make_shared(); + underlying_type = std::make_shared(); + count = 0; + } +}; + +struct ChainsBatch { + std::shared_ptr chain_id = std::make_shared(); + std::shared_ptr underlying_id = std::make_shared(); + std::shared_ptr product_code = std::make_shared(); + std::shared_ptr expiration_date = std::make_shared(); + std::shared_ptr expiration_date_raw = std::make_shared(); + std::shared_ptr exercise_style = std::make_shared(); + std::shared_ptr underlying_type = std::make_shared(); + std::shared_ptr interest_rate = std::make_shared(); + std::shared_ptr is_paying_dividends = std::make_shared(); + std::shared_ptr snapshot_time = std::make_shared(9); + std::shared_ptr source_file = std::make_shared(); + std::size_t count = 0; + + void append(const ChainRow& r) { + chain_id->Append(r.chain_id); + underlying_id->Append(r.underlying_id); + product_code->Append(r.product_code); + expiration_date->Append(static_cast(r.expiration_days) * kSecondsPerDay); + expiration_date_raw->Append(r.expiration_raw); + exercise_style->Append(r.exercise_style); + underlying_type->Append(r.underlying_type); + interest_rate->Append(r.interest_rate); + is_paying_dividends->Append(r.is_paying_dividends); + snapshot_time->Append(r.snapshot_ns); + source_file->Append(r.source_file); + ++count; + } + + void flush(Client& client) { + if (count == 0) return; + Block block; + block.AppendColumn("chain_id", chain_id); + block.AppendColumn("underlying_id", underlying_id); + block.AppendColumn("product_code", product_code); + block.AppendColumn("expiration_date", expiration_date); + block.AppendColumn("expiration_date_raw", expiration_date_raw); + block.AppendColumn("exercise_style", exercise_style); + block.AppendColumn("underlying_type", underlying_type); + block.AppendColumn("interest_rate", interest_rate); + block.AppendColumn("is_paying_dividends", is_paying_dividends); + block.AppendColumn("snapshot_time", snapshot_time); + block.AppendColumn("source_file", source_file); + client.Insert("options_pricing.option_chains", block); + chain_id = std::make_shared(); + underlying_id = std::make_shared(); + product_code = std::make_shared(); + expiration_date = std::make_shared(); + expiration_date_raw = std::make_shared(); + exercise_style = std::make_shared(); + underlying_type = std::make_shared(); + interest_rate = std::make_shared(); + is_paying_dividends = std::make_shared(); + snapshot_time = std::make_shared(9); + source_file = std::make_shared(); + count = 0; + } +}; + +struct UnderlyingQuotesBatch { + std::shared_ptr snapshot_time = std::make_shared(9); + std::shared_ptr chain_id = std::make_shared(); + std::shared_ptr underlying_id = std::make_shared(); + std::shared_ptr mode = std::make_shared(); + NullableDateTime64 quote_time; + NullableFloat64 bid_price; + NullableUInt32 bid_size; + NullableDateTime64 bid_time; + NullableFloat64 ask_price; + NullableUInt32 ask_size; + NullableDateTime64 ask_time; + NullableFloat64 last_price; + NullableUInt32 last_size; + NullableDateTime64 last_time; + std::size_t count = 0; + + void append(const UnderlyingQuoteRow& r) { + snapshot_time->Append(r.snapshot_ns); + chain_id->Append(r.chain_id); + underlying_id->Append(r.underlying_id); + mode->Append(r.mode); + quote_time.append(r.quote_time_ns); + bid_price.append(r.bid.price); + bid_size.append(r.bid.size); + bid_time.append(r.bid.time_ns); + ask_price.append(r.ask.price); + ask_size.append(r.ask.size); + ask_time.append(r.ask.time_ns); + last_price.append(r.last.price); + last_size.append(r.last.size); + last_time.append(r.last.time_ns); + ++count; + } + + void flush(Client& client) { + if (count == 0) return; + Block block; + block.AppendColumn("snapshot_time", snapshot_time); + block.AppendColumn("chain_id", chain_id); + block.AppendColumn("underlying_id", underlying_id); + block.AppendColumn("mode", mode); + block.AppendColumn("quote_time", quote_time.column()); + block.AppendColumn("bid_price", bid_price.column()); + block.AppendColumn("bid_size", bid_size.column()); + block.AppendColumn("bid_time", bid_time.column()); + block.AppendColumn("ask_price", ask_price.column()); + block.AppendColumn("ask_size", ask_size.column()); + block.AppendColumn("ask_time", ask_time.column()); + block.AppendColumn("last_price", last_price.column()); + block.AppendColumn("last_size", last_size.column()); + block.AppendColumn("last_time", last_time.column()); + client.Insert("options_pricing.underlying_quotes", block); + + snapshot_time = std::make_shared(9); + chain_id = std::make_shared(); + underlying_id = std::make_shared(); + mode = std::make_shared(); + quote_time.reset(); + bid_price.reset(); bid_size.reset(); bid_time.reset(); + ask_price.reset(); ask_size.reset(); ask_time.reset(); + last_price.reset(); last_size.reset(); last_time.reset(); + count = 0; + } +}; + +struct OptionQuotesBatch { + TypeRef option_type_enum = Type::CreateEnum8({{"put", 0}, {"call", 1}}); + + std::shared_ptr snapshot_time = std::make_shared(9); + std::shared_ptr chain_id = std::make_shared(); + std::shared_ptr underlying_id = std::make_shared(); + std::shared_ptr product_code = std::make_shared(); + std::shared_ptr expiration_date = std::make_shared(); + std::shared_ptr strike = std::make_shared(); + std::shared_ptr option_type = std::make_shared(option_type_enum); + std::shared_ptr option_id = std::make_shared(); + std::shared_ptr exercise_style = std::make_shared(); + std::shared_ptr interest_rate = std::make_shared(); + std::shared_ptr mode = std::make_shared(); + NullableDateTime64 quote_time; + NullableFloat64 bid_price; + NullableUInt32 bid_size; + NullableDateTime64 bid_time; + NullableFloat64 ask_price; + NullableUInt32 ask_size; + NullableDateTime64 ask_time; + NullableFloat64 last_price; + NullableUInt32 last_size; + NullableDateTime64 last_time; + std::shared_ptr source_file = std::make_shared(); + std::size_t count = 0; + + void append(const OptionRow& r) { + snapshot_time->Append(r.snapshot_ns); + chain_id->Append(r.chain_id); + underlying_id->Append(r.underlying_id); + product_code->Append(r.product_code); + expiration_date->Append(static_cast(r.expiration_days) * kSecondsPerDay); + strike->Append(r.strike); + option_type->Append(r.option_type); + option_id->Append(r.option_id); + exercise_style->Append(r.exercise_style); + interest_rate->Append(r.interest_rate); + mode->Append(r.mode); + quote_time.append(r.quote_time_ns); + bid_price.append(r.bid.price); + bid_size.append(r.bid.size); + bid_time.append(r.bid.time_ns); + ask_price.append(r.ask.price); + ask_size.append(r.ask.size); + ask_time.append(r.ask.time_ns); + last_price.append(r.last.price); + last_size.append(r.last.size); + last_time.append(r.last.time_ns); + source_file->Append(r.source_file); + ++count; + } + + void flush(Client& client) { + if (count == 0) return; + Block block; + block.AppendColumn("snapshot_time", snapshot_time); + block.AppendColumn("chain_id", chain_id); + block.AppendColumn("underlying_id", underlying_id); + block.AppendColumn("product_code", product_code); + block.AppendColumn("expiration_date", expiration_date); + block.AppendColumn("strike", strike); + block.AppendColumn("option_type", option_type); + block.AppendColumn("option_id", option_id); + block.AppendColumn("exercise_style", exercise_style); + block.AppendColumn("interest_rate", interest_rate); + block.AppendColumn("mode", mode); + block.AppendColumn("quote_time", quote_time.column()); + block.AppendColumn("bid_price", bid_price.column()); + block.AppendColumn("bid_size", bid_size.column()); + block.AppendColumn("bid_time", bid_time.column()); + block.AppendColumn("ask_price", ask_price.column()); + block.AppendColumn("ask_size", ask_size.column()); + block.AppendColumn("ask_time", ask_time.column()); + block.AppendColumn("last_price", last_price.column()); + block.AppendColumn("last_size", last_size.column()); + block.AppendColumn("last_time", last_time.column()); + block.AppendColumn("source_file", source_file); + client.Insert("options_pricing.option_quotes", block); + + snapshot_time = std::make_shared(9); + chain_id = std::make_shared(); + underlying_id = std::make_shared(); + product_code = std::make_shared(); + expiration_date = std::make_shared(); + strike = std::make_shared(); + option_type = std::make_shared(option_type_enum); + option_id = std::make_shared(); + exercise_style = std::make_shared(); + interest_rate = std::make_shared(); + mode = std::make_shared(); + quote_time.reset(); + bid_price.reset(); bid_size.reset(); bid_time.reset(); + ask_price.reset(); ask_size.reset(); ask_time.reset(); + last_price.reset(); last_size.reset(); last_time.reset(); + source_file = std::make_shared(); + count = 0; + } +}; + +// All four table buffers owned by a single worker. +struct WorkerBatches { + UnderlyingsBatch underlyings; + ChainsBatch chains; + UnderlyingQuotesBatch underlying_quotes; + OptionQuotesBatch option_quotes; + + void flush_full(Client& client, std::size_t batch_size) { + if (option_quotes.count >= batch_size) option_quotes.flush(client); + if (underlyings.count >= batch_size) underlyings.flush(client); + if (chains.count >= batch_size) chains.flush(client); + if (underlying_quotes.count >= batch_size) underlying_quotes.flush(client); + } + + void flush_all(Client& client) { + option_quotes.flush(client); + underlyings.flush(client); + chains.flush(client); + underlying_quotes.flush(client); + } +}; + +ClientOptions make_options(const ChConfig& cfg) { + return ClientOptions() + .SetHost(cfg.host) + .SetPort(cfg.port) + .SetUser(cfg.user) + .SetPassword(cfg.password) + .SetDefaultDatabase(cfg.database); +} + +// Collect the *.json files to import. A single file is taken as-is; a directory +// is walked recursively. +std::vector collect_files(const std::string& path, std::string& error) { + std::vector files; + std::error_code ec; + const fs::path p(path); + + const auto status = fs::status(p, ec); + if (ec) { + error = "cannot stat path: " + ec.message(); + return files; + } + + if (fs::is_regular_file(status)) { + files.push_back(p.string()); + return files; + } + if (fs::is_directory(status)) { + for (auto it = fs::recursive_directory_iterator(p, ec); + !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (it->is_regular_file() && it->path().extension() == ".json") { + files.push_back(it->path().string()); + } + } + if (ec) error = "error while walking directory: " + ec.message(); + return files; + } + + error = "path is neither a file nor a directory"; + return files; +} + +} // namespace + +Inserter::Inserter(ChConfig config) : config_(std::move(config)) {} + +bool Inserter::insert(const std::string& path) { + using clock = std::chrono::steady_clock; + const auto started = clock::now(); + + std::string error; + std::vector files = collect_files(path, error); + if (!error.empty()) { + std::cerr << "[insert] " << error << "\n"; + return false; + } + if (files.empty()) { + std::cerr << "[insert] no .json files found at '" << path << "'\n"; + return false; + } + + // Fail fast if ClickHouse is unreachable, with a clear message. + try { + Client probe(make_options(config_)); + probe.Execute("SELECT 1"); + } catch (const std::exception& e) { + std::cerr << "[insert] cannot connect to ClickHouse at " << config_.host << ":" + << config_.port << " (" << e.what() << ")\n"; + return false; + } + + const std::size_t worker_count = std::min(config_.workers, files.size()); + std::cout << "[insert] " << files.size() << " file(s), " << worker_count + << " worker(s), batch=" << config_.batch_size << "\n"; + + BlockingQueue queue; + for (auto& f : files) queue.push(std::move(f)); + queue.close(); + + std::atomic files_ok{0}; + std::atomic files_failed{0}; + std::atomic option_rows{0}; + std::atomic fatal{false}; + std::mutex log_mutex; + + auto worker = [&]() { + Client client(make_options(config_)); + WorkerBatches batches; + try { + while (auto file = queue.pop()) { + std::string perr; + auto parsed = parse_ivcurve_file(*file, perr); + if (!parsed) { + files_failed.fetch_add(1); + std::lock_guard lk(log_mutex); + std::cerr << "[insert] skip " << *file << ": " << perr << "\n"; + continue; + } + batches.underlyings.append(parsed->underlying); + batches.chains.append(parsed->chain); + batches.underlying_quotes.append(parsed->underlying_quote); + for (const auto& o : parsed->options) batches.option_quotes.append(o); + option_rows.fetch_add(parsed->options.size()); + files_ok.fetch_add(1); + batches.flush_full(client, config_.batch_size); + } + batches.flush_all(client); + } catch (const std::exception& e) { + fatal.store(true); + std::lock_guard lk(log_mutex); + std::cerr << "[insert] worker aborted: " << e.what() << "\n"; + } + }; + + std::vector pool; + pool.reserve(worker_count); + for (std::size_t i = 0; i < worker_count; ++i) pool.emplace_back(worker); + for (auto& t : pool) t.join(); + + const auto elapsed = std::chrono::duration(clock::now() - started).count(); + std::cout << "[insert] done in " << elapsed << "s: " + << files_ok.load() << " ok, " << files_failed.load() << " failed, " + << option_rows.load() << " option rows\n"; + + return !fatal.load() && files_failed.load() == 0; +} + +} // namespace ope diff --git a/src/ivcurve_parser.cpp b/src/ivcurve_parser.cpp new file mode 100644 index 0000000..9c62ef5 --- /dev/null +++ b/src/ivcurve_parser.cpp @@ -0,0 +1,176 @@ +#include "ope/ivcurve_parser.hpp" + +#include + +#include +#include +#include + +#include "ope/time_util.hpp" + +namespace ope { + +namespace { + +using nlohmann::json; + +// Split "CME:6AM2026;5AD;20260529" on ';'. +std::vector split(const std::string& s, char delim) { + std::vector out; + std::string cur; + for (char c : s) { + if (c == delim) { + out.push_back(cur); + cur.clear(); + } else { + cur.push_back(c); + } + } + out.push_back(cur); + return out; +} + +// Read one bid/ask/last node, which is either JSON null or {price,size,time}. +QuoteSide parse_side(const json& node) { + QuoteSide side; + if (node.is_null()) return side; + if (auto it = node.find("price"); it != node.end() && !it->is_null()) + side.price = it->get(); + if (auto it = node.find("size"); it != node.end() && !it->is_null()) + side.size = it->get(); + if (auto it = node.find("time"); it != node.end() && !it->is_null()) + side.time_ns = timeutil::parse_iso8601_ns(it->get()); + return side; +} + +std::optional parse_optional_time(const json& obj, const char* key) { + if (auto it = obj.find(key); it != obj.end() && !it->is_null()) + return timeutil::parse_iso8601_ns(it->get()); + return std::nullopt; +} + +// Parse the puts/calls map into OptionRows sharing the given chain context. +void parse_option_map(const json& node, int8_t option_type, const ChainRow& chain, + const std::string& source_file, std::vector& out) { + if (!node.is_object()) return; + for (auto it = node.begin(); it != node.end(); ++it) { + const json& entry = it.value(); + OptionRow row; + row.snapshot_ns = chain.snapshot_ns; + row.chain_id = chain.chain_id; + row.underlying_id = chain.underlying_id; + row.product_code = chain.product_code; + row.expiration_days = chain.expiration_days; + row.strike = std::stod(it.key()); // map key is the strike, e.g. "0.65" + row.option_type = option_type; + if (auto f = entry.find("id"); f != entry.end() && !f->is_null()) + row.option_id = f->get(); + row.exercise_style = chain.exercise_style; + row.interest_rate = chain.interest_rate; + if (auto f = entry.find("mode"); f != entry.end() && !f->is_null()) + row.mode = f->get(); + row.quote_time_ns = parse_optional_time(entry, "time"); + if (auto f = entry.find("bid"); f != entry.end()) row.bid = parse_side(*f); + if (auto f = entry.find("ask"); f != entry.end()) row.ask = parse_side(*f); + if (auto f = entry.find("last"); f != entry.end()) row.last = parse_side(*f); + row.source_file = source_file; + out.push_back(std::move(row)); + } +} + +} // namespace + +std::optional parse_ivcurve_file(const std::string& path, + std::string& error) { + std::ifstream in(path, std::ios::binary); + if (!in) { + error = "cannot open file"; + return std::nullopt; + } + + json root; + try { + in >> root; + } catch (const std::exception& e) { + error = std::string("json parse error: ") + e.what(); + return std::nullopt; + } + + const auto iv_it = root.find("ivcurve"); + if (iv_it == root.end() || !iv_it->is_object()) { + error = "missing 'ivcurve' object"; + return std::nullopt; + } + const json& iv = *iv_it; + + try { + ParsedFile pf; + + const std::string chain_id = iv.at("id").get(); + const int64_t snapshot_ns = timeutil::parse_iso8601_ns(iv.at("now").get()); + const uint32_t expiration_raw = iv.at("expirationDate").get(); + const std::string exercise_style = iv.value("exerciseStyle", std::string{}); + const std::string underlying_type = iv.value("underlyingType", std::string{}); + const double interest_rate = iv.value("interestRate", 0.0); + const bool is_paying_dividends = iv.value("isPayingDividends", false); + + // chain_id = ";;" + const auto parts = split(chain_id, ';'); + const std::string underlying_id = parts.size() > 0 ? parts[0] : std::string{}; + const std::string product_code = parts.size() > 1 ? parts[1] : std::string{}; + + // underlying_id = ":" + std::string market, symbol; + if (const auto colon = underlying_id.find(':'); colon != std::string::npos) { + market = underlying_id.substr(0, colon); + symbol = underlying_id.substr(colon + 1); + } else { + symbol = underlying_id; + } + + // --- underlying dimension --- + pf.underlying.underlying_id = underlying_id; + pf.underlying.market = market; + pf.underlying.symbol = symbol; + pf.underlying.underlying_type = underlying_type; + + // --- chain dimension --- + pf.chain.chain_id = chain_id; + pf.chain.underlying_id = underlying_id; + pf.chain.product_code = product_code; + pf.chain.expiration_days = timeutil::yyyymmdd_to_days(expiration_raw); + pf.chain.expiration_raw = expiration_raw; + pf.chain.exercise_style = exercise_style; + pf.chain.underlying_type = underlying_type; + pf.chain.interest_rate = interest_rate; + pf.chain.is_paying_dividends = is_paying_dividends ? 1 : 0; + pf.chain.snapshot_ns = snapshot_ns; + pf.chain.source_file = path; + + const json& md = iv.at("marketData"); + + // --- underlying quote fact --- + const json& u = md.at("underlying"); + pf.underlying_quote.snapshot_ns = snapshot_ns; + pf.underlying_quote.chain_id = chain_id; + pf.underlying_quote.underlying_id = underlying_id; + pf.underlying_quote.mode = u.value("mode", std::string{}); + pf.underlying_quote.quote_time_ns = parse_optional_time(u, "time"); + if (auto f = u.find("bid"); f != u.end()) pf.underlying_quote.bid = parse_side(*f); + if (auto f = u.find("ask"); f != u.end()) pf.underlying_quote.ask = parse_side(*f); + if (auto f = u.find("last"); f != u.end()) pf.underlying_quote.last = parse_side(*f); + + // --- option quote facts --- + if (auto f = md.find("puts"); f != md.end()) + parse_option_map(*f, /*put*/ 0, pf.chain, path, pf.options); + if (auto f = md.find("calls"); f != md.end()) + parse_option_map(*f, /*call*/ 1, pf.chain, path, pf.options); + + return pf; + } catch (const std::exception& e) { + error = std::string("schema error: ") + e.what(); + return std::nullopt; + } +} + +} // namespace ope diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 0000000..eacdba1 --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include + +#include "ope/ch_config.hpp" +#include "ope/inserter.hpp" + +namespace { + +void print_help() { + std::cout << + "Commands:\n" + " insert Import a JSON file or a directory (recursively) into ClickHouse\n" + " help Show this help\n" + " quit | exit Leave the program\n"; +} + +std::vector tokenize(const std::string& line) { + std::vector tokens; + std::istringstream iss(line); + std::string tok; + while (iss >> tok) tokens.push_back(tok); + return tokens; +} + +// Dispatch a single command. Returns false when the loop should stop. +bool dispatch(const std::vector& args, ope::Inserter& inserter) { + if (args.empty()) return true; + const std::string& cmd = args[0]; + + if (cmd == "quit" || cmd == "exit") return false; + if (cmd == "help") { + print_help(); + return true; + } + if (cmd == "insert") { + if (args.size() < 2) { + std::cerr << "usage: insert \n"; + return true; + } + const bool ok = inserter.insert(args[1]); + std::cout << (ok ? "insert: OK" : "insert: FAILED") << "\n"; + return true; + } + + std::cerr << "unknown command: " << cmd << " (type 'help')\n"; + return true; +} + +} // namespace + +int main(int argc, char** argv) { + const ope::ChConfig config = ope::ChConfig::from_env(); + ope::Inserter inserter(config); + + // Non-interactive mode: treat argv as a single command, run it, and exit. + // e.g. ./ope insert data/CME + if (argc > 1) { + std::vector args(argv + 1, argv + argc); + const bool cont = dispatch(args, inserter); + (void)cont; + return 0; + } + + // Interactive event loop. `insert` is one of several commands; more data + // interaction commands can be added to dispatch() later. + std::cout << "Options Pricing Engine — data console\n"; + print_help(); + + std::string line; + while (true) { + std::cout << "> " << std::flush; + if (!std::getline(std::cin, line)) break; // EOF (Ctrl-D) + if (!dispatch(tokenize(line), inserter)) break; + } + return 0; +} diff --git a/src/time_util.cpp b/src/time_util.cpp new file mode 100644 index 0000000..48b9e8b --- /dev/null +++ b/src/time_util.cpp @@ -0,0 +1,61 @@ +#include "ope/time_util.hpp" + +#include +#include + +namespace ope::timeutil { + +int64_t parse_iso8601_ns(std::string_view s) { + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + // The leading date-time part is fixed width; sscanf is simple and fast here. + if (std::sscanf(s.data(), "%4d-%2d-%2dT%2d:%2d:%2d", &year, &month, &day, &hour, + &minute, &second) != 6) { + return 0; + } + + std::tm tm{}; + tm.tm_year = year - 1900; + tm.tm_mon = month - 1; + tm.tm_mday = day; + tm.tm_hour = hour; + tm.tm_min = minute; + tm.tm_sec = second; + // timegm interprets the struct as UTC (unlike mktime which uses local time). + const std::time_t secs = timegm(&tm); + int64_t ns = static_cast(secs) * 1'000'000'000LL; + + // Optional fractional seconds: "." up to nanosecond precision. + const auto dot = s.find('.'); + if (dot != std::string_view::npos) { + int64_t frac = 0; + int digits = 0; + for (std::size_t i = dot + 1; i < s.size() && std::isdigit(static_cast(s[i])); ++i) { + if (digits < 9) { + frac = frac * 10 + (s[i] - '0'); + ++digits; + } + } + // Scale to nanoseconds (pad missing low-order digits). + for (; digits < 9; ++digits) frac *= 10; + ns += frac; + } + return ns; +} + +std::time_t yyyymmdd_to_seconds(uint32_t yyyymmdd) { + const int year = static_cast(yyyymmdd / 10000); + const int month = static_cast((yyyymmdd / 100) % 100); + const int day = static_cast(yyyymmdd % 100); + + std::tm tm{}; + tm.tm_year = year - 1900; + tm.tm_mon = month - 1; + tm.tm_mday = day; + return timegm(&tm); +} + +int32_t yyyymmdd_to_days(uint32_t yyyymmdd) { + return static_cast(yyyymmdd_to_seconds(yyyymmdd) / 86400); +} + +} // namespace ope::timeutil From 3cc884274d2a08d5e06dc62b9fd5cf998c83481c Mon Sep 17 00:00:00 2001 From: Benedichuk Margarita Date: Sat, 18 Jul 2026 14:16:32 +0300 Subject: [PATCH 3/5] bugfix --- .gitignore | 4 + CMakeLists.txt | 38 +++- README.md | 62 ++++++ data/README.md | 11 +- include/ope/parsed_rows.hpp | 16 ++ include/ope/time_util.hpp | 9 +- src/inserter.cpp | 43 +++- src/ivcurve_parser.cpp | 129 +++++++++-- src/time_util.cpp | 32 ++- tests/ivcurve_parser_test.cpp | 406 ++++++++++++++++++++++++++++++++++ tests/time_util_test.cpp | 63 ++++++ 11 files changed, 779 insertions(+), 34 deletions(-) create mode 100644 tests/ivcurve_parser_test.cpp create mode 100644 tests/time_util_test.cpp diff --git a/.gitignore b/.gitignore index 0b25965..4796b0f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ venv/ /clickhouse/logs/ build/ *.log + +# Claude Code project files — personal working notes, not shared project docs. +/CLAUDE.md +/.claude/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a6753e..3d7fc2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,15 @@ FetchContent_Declare( GIT_SHALLOW TRUE ) -FetchContent_MakeAvailable(nlohmann_json clickhouse_cpp) +# --- Catch2 (unit tests) ----------------------------------------------------- +FetchContent_Declare( + catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.6.0 + GIT_SHALLOW TRUE +) + +FetchContent_MakeAvailable(nlohmann_json clickhouse_cpp catch2) find_package(Threads REQUIRED) @@ -50,3 +58,31 @@ target_link_libraries(ope nlohmann_json::nlohmann_json Threads::Threads ) + +# --- Unit tests -------------------------------------------------------------- +# Links the parser/time-util sources directly (not the `ope` binary) so tests +# don't need a ClickHouse connection: parsing is pure and DB-agnostic per +# parser_specification.md §21 Non-Goals. +enable_testing() + +add_executable(ope_tests + src/time_util.cpp + src/ivcurve_parser.cpp + tests/time_util_test.cpp + tests/ivcurve_parser_test.cpp +) + +target_include_directories(ope_tests + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +target_link_libraries(ope_tests + PRIVATE + nlohmann_json::nlohmann_json + Catch2::Catch2WithMain +) + +list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras) +include(CTest) +include(Catch) +catch_discover_tests(ope_tests) diff --git a/README.md b/README.md index 0f620d4..34bcbfd 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,40 @@ The dev container brings up three services (see `docker-compose.yml`): Raw CME `ivcurve` JSON snapshots live under `data/` (git-ignored). See `data/README.md` for the folder layout and file format. +## Input format + +One file = one snapshot of one option series, JSON wrapped in a root +`"ivcurve"` key: + +```json +{ + "ivcurve": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "now": "2026-05-07T09:54:21.752997491Z", + "expirationDate": 20260610, + "exerciseStyle": "european", + "interestRate": 0.01, + "underlyingType": "futures", + "isPayingDividends": false, + "marketData": { + "underlying": { "id": "...", "mode": "...", "bid": {...}|null, "ask": {...}|null, "last": {...}|null }, + "puts": { "": { "id": "...", "mode": "...", "bid": ..., "ask": ..., "last": ... } }, + "calls": { "": { ... } } + } + } +} +``` + +Full field-by-field format, folder layout, and which exchanges use it: see +`data/README.md`. + +## Output + +Each successfully parsed file writes into four ClickHouse tables (schema: +`clickhouse/init/create_schema.sql`): one row in `underlyings`, one row in +`option_chains`, one row in `underlying_quotes`, and one row per surviving +put/call in `option_quotes`. + ## Building the parser From inside the dev container: @@ -52,3 +86,31 @@ tuning are read from the environment (defaults in parentheses): | `CLICKHOUSE_DB` (`options_pricing`), `CLICKHOUSE_USER` (`default`), `CLICKHOUSE_PASSWORD` (empty) | credentials | | `OPE_WORKERS` (CPU count) | number of parser/inserter threads | | `OPE_BATCH_SIZE` (`50000`) | rows accumulated before a batch `INSERT` | + +At the end of a run the tool prints a summary: + +```text +[insert] done in 24.0s: 30564 ok, 0 failed, 2067514 option rows, 15 option(s) skipped (invalid strike) +``` + +- **ok / failed** — whole files that parsed successfully vs. were rejected + outright (missing required field, malformed enum value, bad JSON, wrong + root shape, etc.). A failed file is skipped entirely; the reason is + printed as `[insert] skip : `. +- **skipped (invalid strike)** — individual put/call entries dropped from an + otherwise-valid file because their strike key was malformed/non-finite/ + non-positive (real data has a handful of `"0."` placeholder entries with + no live quote at all). Each one is logged as + `[insert] : skipped option ` — never silently discarded. + +## Tests + +```bash +cmake --build build --target ope_tests -j +./build/ope_tests +``` + +Unit tests cover `time_util` (timestamp parsing) and `ivcurve_parser` +(required-field/enum/strike validation, skip-not-fail behavior, real-shape +fixtures) — no ClickHouse connection needed, `ope_tests` only links the +parsing code. diff --git a/data/README.md b/data/README.md index 86444e2..eb41445 100644 --- a/data/README.md +++ b/data/README.md @@ -6,14 +6,17 @@ large and reproducible from the data provider. ## Layout +Test data spans multiple exchanges (CME, CME_MINI, CBOE, CBOT, CBOT_MINI, COMEX, +NASDAQ, NYMEX, NYSE), each delivered as one archive: + ``` data/ -└── CME/ - └── CME-/ # one folder per underlying futures contract - └── _.json # one file per option chain (expiration) +└── / + └── -/ # one folder per underlying (futures, stock, ETF, or index) + └── _.json # one file per option series (expiration) ``` -Example: `data/CME/CME-6AM2026/5AD_20260529.json` +Examples: `data/CME/CME-6AM2026/5AD_20260529.json` (futures), `data/NASDAQ/NASDAQ-AAL/AAL_20260529.json` (stock). ## File format (`ivcurve` snapshot) diff --git a/include/ope/parsed_rows.hpp b/include/ope/parsed_rows.hpp index b6065c1..946d4ac 100644 --- a/include/ope/parsed_rows.hpp +++ b/include/ope/parsed_rows.hpp @@ -76,6 +76,22 @@ struct ParsedFile { ChainRow chain; UnderlyingQuoteRow underlying_quote; std::vector options; + + // marketData.id / marketData.time: distinct from ivcurve.id / ivcurve.now + // (real samples show different values for the two). Not yet mapped to a + // ClickHouse column — see parser_specification.md §23.2, which leaves that + // storage decision open. Captured here so parsing itself loses no source + // information regardless of what the storage layer later does with it. + std::string market_data_id; + int64_t market_data_time_ns = 0; + + // Human-readable records of individual put/call entries that were skipped + // because their strike key was malformed/non-finite/non-positive (see + // parse_strike in ivcurve_parser.cpp). A single bad entry does not fail + // the whole file — real data has files where one degenerate placeholder + // strike sits alongside otherwise-valid quotes — but every skip must be + // documented here rather than silently dropped (parser_specification.md §25). + std::vector skipped_options; }; } // namespace ope diff --git a/include/ope/time_util.hpp b/include/ope/time_util.hpp index 386a6c0..2e2f616 100644 --- a/include/ope/time_util.hpp +++ b/include/ope/time_util.hpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace ope::timeutil { @@ -11,8 +12,12 @@ namespace ope::timeutil { // "2026-05-25T14:59:01.800326283Z" (up to 9 fractional digits) // "2026-05-25T14:58:59Z" (no fraction) // "1970-01-01T00:00:00Z" (epoch) -// Returns 0 for the epoch string and for anything it fails to parse. -int64_t parse_iso8601_ns(std::string_view s); +// The epoch string is a legitimate sentinel value in the source data (meaning +// "no quote yet") and parses to exactly 0. A malformed or unparseable string +// must not be confused with it, so failure is reported as std::nullopt rather +// than silently returned as 0 — callers decide whether a missing timestamp is +// fatal or tolerable, the parser must never guess. +std::optional parse_iso8601_ns(std::string_view s); // Convert a CME expirationDate integer (YYYYMMDD, e.g. 20260529) to the number // of days since the Unix epoch (what ClickHouse's Date column stores) and to diff --git a/src/inserter.cpp b/src/inserter.cpp index 470bf0a..6ee389f 100644 --- a/src/inserter.cpp +++ b/src/inserter.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -369,6 +370,14 @@ ClientOptions make_options(const ChConfig& cfg) { .SetDefaultDatabase(cfg.database); } +// Case-insensitive ".json" check — a stray ".JSON" export should still be +// picked up, not silently skipped with no diagnostic. +bool has_json_extension(const fs::path& p) { + std::string ext = p.extension().string(); + for (char& c : ext) c = static_cast(std::tolower(static_cast(c))); + return ext == ".json"; +} + // Collect the *.json files to import. A single file is taken as-is; a directory // is walked recursively. std::vector collect_files(const std::string& path, std::string& error) { @@ -389,7 +398,7 @@ std::vector collect_files(const std::string& path, std::string& err if (fs::is_directory(status)) { for (auto it = fs::recursive_directory_iterator(p, ec); !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { - if (it->is_regular_file() && it->path().extension() == ".json") { + if (it->is_regular_file() && has_json_extension(it->path())) { files.push_back(it->path().string()); } } @@ -441,6 +450,7 @@ bool Inserter::insert(const std::string& path) { std::atomic files_ok{0}; std::atomic files_failed{0}; std::atomic option_rows{0}; + std::atomic options_skipped{0}; std::atomic fatal{false}; std::mutex log_mutex; @@ -457,6 +467,13 @@ bool Inserter::insert(const std::string& path) { std::cerr << "[insert] skip " << *file << ": " << perr << "\n"; continue; } + if (!parsed->skipped_options.empty()) { + options_skipped.fetch_add(parsed->skipped_options.size()); + std::lock_guard lk(log_mutex); + for (const auto& diag : parsed->skipped_options) { + std::cerr << "[insert] " << *file << ": skipped option " << diag << "\n"; + } + } batches.underlyings.append(parsed->underlying); batches.chains.append(parsed->chain); batches.underlying_quotes.append(parsed->underlying_quote); @@ -468,8 +485,27 @@ bool Inserter::insert(const std::string& path) { batches.flush_all(client); } catch (const std::exception& e) { fatal.store(true); + // Each Batch::flush() only clears its columns *after* a successful + // Insert (see e.g. OptionQuotesBatch::flush above), so whatever hasn't + // been durably written yet is still sitting in `batches` right now. + // Silently dropping it here would lose up to (batch_size - 1) rows per + // table with no record it ever happened — retry the flush once so a + // transient fault (e.g. one bad Insert) doesn't cost data that was + // otherwise fine, and report exactly what's lost if it isn't. + const std::size_t pending = batches.option_quotes.count + batches.underlyings.count + + batches.chains.count + batches.underlying_quotes.count; + std::string outcome; + if (pending > 0) { + try { + batches.flush_all(client); + outcome = " (" + std::to_string(pending) + " pending row(s) recovered on retry)"; + } catch (const std::exception& flush_err) { + outcome = " (" + std::to_string(pending) + + " pending row(s) LOST — retry flush also failed: " + flush_err.what() + ")"; + } + } std::lock_guard lk(log_mutex); - std::cerr << "[insert] worker aborted: " << e.what() << "\n"; + std::cerr << "[insert] worker aborted: " << e.what() << outcome << "\n"; } }; @@ -481,7 +517,8 @@ bool Inserter::insert(const std::string& path) { const auto elapsed = std::chrono::duration(clock::now() - started).count(); std::cout << "[insert] done in " << elapsed << "s: " << files_ok.load() << " ok, " << files_failed.load() << " failed, " - << option_rows.load() << " option rows\n"; + << option_rows.load() << " option rows, " + << options_skipped.load() << " option(s) skipped (invalid strike)\n"; return !fatal.load() && files_failed.load() == 0; } diff --git a/src/ivcurve_parser.cpp b/src/ivcurve_parser.cpp index 9c62ef5..a7bb2d6 100644 --- a/src/ivcurve_parser.cpp +++ b/src/ivcurve_parser.cpp @@ -2,8 +2,11 @@ #include +#include +#include #include #include +#include #include #include "ope/time_util.hpp" @@ -30,6 +33,47 @@ std::vector split(const std::string& s, char delim) { return out; } +// A required field was missing, malformed, or held an unsupported value. +// Always fatal for the whole file — see parser_specification.md §9.1. +[[noreturn]] void fail(const std::string& message) { throw std::runtime_error(message); } + +void require_enum(const std::string& value, std::initializer_list allowed, + const char* field_name) { + for (const char* candidate : allowed) { + if (value == candidate) return; + } + fail(std::string(field_name) + ": unsupported value '" + value + "'"); +} + +// Strict strike-key parsing (parser_specification.md §7 / §24.2): the whole +// key must be consumed, the result must be finite, and a strike is a price so +// it must be strictly positive. std::stod alone enforces none of these — it +// accepts trailing garbage and "nan"/"inf" text silently. +double parse_strike(const std::string& key) { + if (key.empty()) fail("strike key is empty"); + char* end = nullptr; + const double value = std::strtod(key.c_str(), &end); + if (end != key.c_str() + key.size()) { + fail("strike key not fully consumed: '" + key + "'"); + } + if (!std::isfinite(value)) { + fail("strike key is not finite: '" + key + "'"); + } + if (value <= 0.0) { + fail("strike key must be > 0: '" + key + "'"); + } + return value; +} + +// Required top-level/marketData timestamp: malformed input is fatal, it must +// never silently collapse to the epoch sentinel (§14.3, §24.3). +int64_t require_time(const json& obj, const char* key) { + const std::string raw = obj.at(key).get(); + const auto parsed = timeutil::parse_iso8601_ns(raw); + if (!parsed) fail(std::string(key) + ": malformed timestamp '" + raw + "'"); + return *parsed; +} + // Read one bid/ask/last node, which is either JSON null or {price,size,time}. QuoteSide parse_side(const json& node) { QuoteSide side; @@ -38,37 +82,65 @@ QuoteSide parse_side(const json& node) { side.price = it->get(); if (auto it = node.find("size"); it != node.end() && !it->is_null()) side.size = it->get(); - if (auto it = node.find("time"); it != node.end() && !it->is_null()) - side.time_ns = timeutil::parse_iso8601_ns(it->get()); + if (auto it = node.find("time"); it != node.end() && !it->is_null()) { + const std::string raw = it->get(); + const auto parsed = timeutil::parse_iso8601_ns(raw); + if (!parsed) fail(std::string("quote time: malformed timestamp '") + raw + "'"); + side.time_ns = *parsed; + } return side; } +// Per-instrument "time" (present but null is legitimate; present-and-malformed +// is fatal per §9.1 — a format error, not a missing value). std::optional parse_optional_time(const json& obj, const char* key) { - if (auto it = obj.find(key); it != obj.end() && !it->is_null()) - return timeutil::parse_iso8601_ns(it->get()); - return std::nullopt; + auto it = obj.find(key); + if (it == obj.end() || it->is_null()) return std::nullopt; + const std::string raw = it->get(); + const auto parsed = timeutil::parse_iso8601_ns(raw); + if (!parsed) fail(std::string(key) + ": malformed timestamp '" + raw + "'"); + return parsed; } // Parse the puts/calls map into OptionRows sharing the given chain context. +// A malformed strike key skips just that one entry (recorded in `skipped`) +// rather than failing the whole file — see ParsedFile::skipped_options. void parse_option_map(const json& node, int8_t option_type, const ChainRow& chain, - const std::string& source_file, std::vector& out) { - if (!node.is_object()) return; + const std::string& source_file, std::vector& out, + std::vector& skipped) { + if (!node.is_object()) fail("puts/calls collection is not an object"); + const char* type_name = option_type == 1 ? "call" : "put"; for (auto it = node.begin(); it != node.end(); ++it) { const json& entry = it.value(); + + double strike = 0.0; + try { + strike = parse_strike(it.key()); // map key is the strike, e.g. "0.65" + } catch (const std::exception& e) { + // entry.value() throws if entry isn't a JSON object at all — a doubly + // malformed record (bad strike key AND a non-object value) must still + // only skip itself, not the whole file, so guard instead of calling + // .value() unconditionally. + const std::string option_id = + entry.is_object() ? entry.value("id", std::string{""}) : std::string{""}; + skipped.push_back(std::string(type_name) + " '" + it.key() + "' (" + option_id + + "): " + e.what()); + continue; + } + OptionRow row; row.snapshot_ns = chain.snapshot_ns; row.chain_id = chain.chain_id; row.underlying_id = chain.underlying_id; row.product_code = chain.product_code; row.expiration_days = chain.expiration_days; - row.strike = std::stod(it.key()); // map key is the strike, e.g. "0.65" + row.strike = strike; row.option_type = option_type; - if (auto f = entry.find("id"); f != entry.end() && !f->is_null()) - row.option_id = f->get(); + row.option_id = entry.at("id").get(); row.exercise_style = chain.exercise_style; row.interest_rate = chain.interest_rate; - if (auto f = entry.find("mode"); f != entry.end() && !f->is_null()) - row.mode = f->get(); + row.mode = entry.at("mode").get(); + require_enum(row.mode, {"realtime", "snapshot"}, "mode"); row.quote_time_ns = parse_optional_time(entry, "time"); if (auto f = entry.find("bid"); f != entry.end()) row.bid = parse_side(*f); if (auto f = entry.find("ask"); f != entry.end()) row.ask = parse_side(*f); @@ -107,12 +179,18 @@ std::optional parse_ivcurve_file(const std::string& path, ParsedFile pf; const std::string chain_id = iv.at("id").get(); - const int64_t snapshot_ns = timeutil::parse_iso8601_ns(iv.at("now").get()); + const int64_t snapshot_ns = require_time(iv, "now"); const uint32_t expiration_raw = iv.at("expirationDate").get(); - const std::string exercise_style = iv.value("exerciseStyle", std::string{}); - const std::string underlying_type = iv.value("underlyingType", std::string{}); - const double interest_rate = iv.value("interestRate", 0.0); - const bool is_paying_dividends = iv.value("isPayingDividends", false); + const std::string exercise_style = iv.at("exerciseStyle").get(); + require_enum(exercise_style, {"american", "european"}, "exerciseStyle"); + const std::string underlying_type = iv.at("underlyingType").get(); + // "fund" (not "etf") is how this data source labels ETFs; "dr" is a + // depositary receipt. Measured directly across the full real dataset — + // see parser_specification.md §25. + require_enum(underlying_type, {"stock", "futures", "index", "fund", "dr"}, + "underlyingType"); + const double interest_rate = iv.at("interestRate").get(); + const bool is_paying_dividends = iv.at("isPayingDividends").get(); // chain_id = ";;" const auto parts = split(chain_id, ';'); @@ -148,23 +226,30 @@ std::optional parse_ivcurve_file(const std::string& path, pf.chain.source_file = path; const json& md = iv.at("marketData"); + pf.market_data_id = md.at("id").get(); + pf.market_data_time_ns = require_time(md, "time"); // --- underlying quote fact --- const json& u = md.at("underlying"); + const std::string underlying_json_id = u.at("id").get(); + if (underlying_json_id != underlying_id) { + fail("marketData.underlying.id ('" + underlying_json_id + + "') does not match the underlying id derived from ivcurve.id ('" + underlying_id + + "')"); + } pf.underlying_quote.snapshot_ns = snapshot_ns; pf.underlying_quote.chain_id = chain_id; pf.underlying_quote.underlying_id = underlying_id; - pf.underlying_quote.mode = u.value("mode", std::string{}); + pf.underlying_quote.mode = u.at("mode").get(); + require_enum(pf.underlying_quote.mode, {"realtime", "snapshot"}, "mode"); pf.underlying_quote.quote_time_ns = parse_optional_time(u, "time"); if (auto f = u.find("bid"); f != u.end()) pf.underlying_quote.bid = parse_side(*f); if (auto f = u.find("ask"); f != u.end()) pf.underlying_quote.ask = parse_side(*f); if (auto f = u.find("last"); f != u.end()) pf.underlying_quote.last = parse_side(*f); // --- option quote facts --- - if (auto f = md.find("puts"); f != md.end()) - parse_option_map(*f, /*put*/ 0, pf.chain, path, pf.options); - if (auto f = md.find("calls"); f != md.end()) - parse_option_map(*f, /*call*/ 1, pf.chain, path, pf.options); + parse_option_map(md.at("puts"), /*put*/ 0, pf.chain, path, pf.options, pf.skipped_options); + parse_option_map(md.at("calls"), /*call*/ 1, pf.chain, path, pf.options, pf.skipped_options); return pf; } catch (const std::exception& e) { diff --git a/src/time_util.cpp b/src/time_util.cpp index 48b9e8b..2fe2406 100644 --- a/src/time_util.cpp +++ b/src/time_util.cpp @@ -5,12 +5,40 @@ namespace ope::timeutil { -int64_t parse_iso8601_ns(std::string_view s) { +namespace { + +bool is_leap_year(int year) { + return (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0); +} + +int days_in_month(int month, int year) { + static constexpr int kDaysPerMonth[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + if (month == 2 && is_leap_year(year)) return 29; + return kDaysPerMonth[month - 1]; +} + +} // namespace + +std::optional parse_iso8601_ns(std::string_view s) { int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; // The leading date-time part is fixed width; sscanf is simple and fast here. + // std::string_view::data() is not guaranteed NUL-terminated, but every + // caller passes a std::string's .c_str()-backed view (see ivcurve_parser.cpp), + // so this is safe here; sscanf also stops at %2d field widths regardless. if (std::sscanf(s.data(), "%4d-%2d-%2dT%2d:%2d:%2d", &year, &month, &day, &hour, &minute, &second) != 6) { - return 0; + return std::nullopt; + } + // sscanf only checks digit shape, not that the date/time is real (e.g. it + // happily accepts month 13 or hour 99) — reject those explicitly instead of + // letting timegm() silently normalize them into some other valid instant. + if (month < 1 || month > 12 || day < 1 || day > 31 || hour < 0 || hour > 23 || + minute < 0 || minute > 59 || second < 0 || second > 60 /* leap second */) { + return std::nullopt; + } + // month is already known valid here, so days_in_month's [1,12] indexing is safe. + if (day > days_in_month(month, year)) { + return std::nullopt; } std::tm tm{}; diff --git a/tests/ivcurve_parser_test.cpp b/tests/ivcurve_parser_test.cpp new file mode 100644 index 0000000..6594c5e --- /dev/null +++ b/tests/ivcurve_parser_test.cpp @@ -0,0 +1,406 @@ +#include + +#include +#include +#include +#include +#include + +#include "ope/ivcurve_parser.hpp" + +namespace { + +std::string write_fixture(const std::string& name, const std::string& content) { + const auto dir = std::filesystem::temp_directory_path() / "ope_parser_tests"; + std::filesystem::create_directories(dir); + const auto path = dir / name; + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out << content; + return path.string(); +} + +std::string render(std::string tmpl, + const std::vector>& subs) { + for (const auto& [key, value] : subs) { + const std::string token = "{{" + key + "}}"; + std::size_t pos; + while ((pos = tmpl.find(token)) != std::string::npos) { + tmpl.replace(pos, token.size(), value); + } + } + return tmpl; +} + +// Shaped after the real CME_MINI-ESM2026-E2C-20260610.json sample: same field +// names/nesting, {{TOKEN}} placeholders for the values each test varies. +constexpr const char* kTemplate = R"JSON({ + "ivcurve": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "now": "{{NOW}}", + "expirationDate": 20260610, + "exerciseStyle": "{{EXERCISE}}", + "interestRate": 0.01, + "underlyingType": "{{UTYPE}}", + "isPayingDividends": false, + "marketData": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "time": "2026-05-07T09:54:18Z", + "underlying": { + "id": "{{UID}}", + "mode": "realtime", + "time": "2026-05-07T09:54:18Z", + "bid": {"price": 7392.5, "size": 23, "time": "2026-05-07T09:54:18Z"}, + "ask": {"price": 7392.75, "size": 6, "time": "2026-05-07T09:54:18Z"}, + "last": {"price": 7392.75, "size": 5, "time": "2026-05-07T09:54:18Z"} + }, + "puts": { + "{{PUT_STRIKE}}": { + "id": "CME_MINI:E2C260610P4500", + "mode": "{{PUT_MODE}}", + "time": "2026-05-07T09:25:41Z", + "bid": {"price": 0.5, "size": 168, "time": "2026-05-07T09:25:41Z"}, + "ask": {"price": 0.85, "size": 168, "time": "2026-05-07T09:25:41Z"}, + "last": null + } + }, + "calls": { + "4600.": { + "id": "CME_MINI:E2C260610C4600", + "mode": "snapshot", + "time": "1970-01-01T00:00:00Z", + "bid": null, + "ask": null, + "last": null + } + } + } + } +})JSON"; + +std::string default_valid_json() { + return render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); +} + +} // namespace + +TEST_CASE("parses a valid file matching the real ivcurve shape", "[parser][valid]") { + const auto path = write_fixture("valid.json", default_valid_json()); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + CHECK(parsed->underlying.underlying_id == "CME_MINI:ESM2026"); + CHECK(parsed->chain.exercise_style == "european"); + CHECK(parsed->options.size() == 2); +} + +TEST_CASE("captures marketData.id and marketData.time distinctly from ivcurve.id/now", + "[parser][marketdata]") { + const auto path = write_fixture("marketdata.json", default_valid_json()); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + CHECK(parsed->market_data_id == "CME_MINI:ESM2026;E2C;20260610"); + // marketData.time (09:54:18Z) differs from ivcurve.now (09:54:21...Z) in + // the fixture, exactly as it does in the real sample file. + CHECK(parsed->market_data_time_ns != parsed->chain.snapshot_ns); + CHECK(parsed->market_data_time_ns != 0); +} + +TEST_CASE("rejects a file whose marketData.underlying.id disagrees with ivcurve.id", + "[parser][marketdata]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:SOMETHING_ELSE"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("mismatched_underlying_id.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE_FALSE(parsed.has_value()); + CHECK(error.find("underlying") != std::string::npos); +} + +// A single invalid strike must not discard the rest of an otherwise valid +// file — real data has files where one degenerate placeholder entry +// (e.g. strike "0.", no live quotes) sits alongside genuinely-quoted +// strikes. The bad entry is skipped and recorded in ParsedFile::skipped_options +// instead of failing the whole file. See parser_specification.md §25. +TEST_CASE("skips (not fails) a strike key with trailing garbage", "[parser][strike]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "170abc"}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("strike_garbage.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + CHECK(parsed->options.size() == 1); // only the valid call remains + REQUIRE(parsed->skipped_options.size() == 1); + CHECK(parsed->skipped_options[0].find("170abc") != std::string::npos); +} + +TEST_CASE("skips (not fails) a zero or negative strike key", "[parser][strike]") { + for (const std::string& bad_strike : {"0.", "-5."}) { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", bad_strike}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("strike_nonpositive.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + INFO("strike key: " << bad_strike); + + REQUIRE(parsed.has_value()); + CHECK(parsed->options.size() == 1); + REQUIRE(parsed->skipped_options.size() == 1); + CHECK(parsed->skipped_options[0].find(bad_strike) != std::string::npos); + } +} + +TEST_CASE("skips (not fails) a non-finite strike key", "[parser][strike]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "nan"}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("strike_nan.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + CHECK(parsed->options.size() == 1); + REQUIRE(parsed->skipped_options.size() == 1); + CHECK(parsed->skipped_options[0].find("nan") != std::string::npos); +} + +TEST_CASE("rejects an unsupported exercise style", "[parser][enum]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "bermudan"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("bad_exercise_style.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("rejects an unsupported underlying type", "[parser][enum]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "bond"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("bad_underlying_type.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("accepts every underlyingType value observed in the real full dataset", + "[parser][enum]") { + // stock/fund/futures/dr/index — measured across all 30564 real sample + // files (data/README.md exchanges). Note "etf" never appears literally; + // ETFs are represented as "fund" in this source. See + // parser_specification.md §24/§25 for the discrepancy with the original + // docx description. + for (const std::string& utype : {"stock", "fund", "futures", "dr", "index"}) { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", utype}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("utype_" + utype + ".json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + INFO("underlyingType: " << utype); + REQUIRE(parsed.has_value()); + } +} + +TEST_CASE("rejects an unsupported quote mode", "[parser][enum]") { + const auto json = render(kTemplate, { + {"NOW", "2026-05-07T09:54:21.752997491Z"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "delayed"}, + }); + const auto path = write_fixture("bad_mode.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("rejects a malformed ivcurve.now timestamp instead of defaulting to epoch", + "[parser][timestamp]") { + const auto json = render(kTemplate, { + {"NOW", "not-a-timestamp"}, + {"EXERCISE", "european"}, + {"UTYPE", "futures"}, + {"UID", "CME_MINI:ESM2026"}, + {"PUT_STRIKE", "4500."}, + {"PUT_MODE", "realtime"}, + }); + const auto path = write_fixture("bad_now.json", json); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("rejects a file missing a required top-level field", "[parser][required]") { + constexpr const char* kMissingExerciseStyle = R"JSON({ + "ivcurve": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "now": "2026-05-07T09:54:21.752997491Z", + "expirationDate": 20260610, + "interestRate": 0.01, + "underlyingType": "futures", + "isPayingDividends": false, + "marketData": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "time": "2026-05-07T09:54:18Z", + "underlying": { + "id": "CME_MINI:ESM2026", "mode": "realtime", + "time": "2026-05-07T09:54:18Z", + "bid": null, "ask": null, "last": null + }, + "puts": {}, + "calls": {} + } + } + })JSON"; + const auto path = write_fixture("missing_exercise_style.json", kMissingExerciseStyle); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("rejects a file missing the calls collection entirely", "[parser][required]") { + constexpr const char* kMissingCalls = R"JSON({ + "ivcurve": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "now": "2026-05-07T09:54:21.752997491Z", + "expirationDate": 20260610, + "exerciseStyle": "european", + "interestRate": 0.01, + "underlyingType": "futures", + "isPayingDividends": false, + "marketData": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "time": "2026-05-07T09:54:18Z", + "underlying": { + "id": "CME_MINI:ESM2026", "mode": "realtime", + "time": "2026-05-07T09:54:18Z", + "bid": null, "ask": null, "last": null + }, + "puts": {} + } + } + })JSON"; + const auto path = write_fixture("missing_calls.json", kMissingCalls); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + REQUIRE_FALSE(parsed.has_value()); +} + +TEST_CASE("skips a doubly-malformed entry (bad strike AND non-object value) instead of " + "failing the whole file", + "[parser][strike][regression]") { + // entry.value("id", ...) throws if the entry isn't a JSON object at all — + // guard against that so a doubly-broken record still only costs itself, + // not every valid strike elsewhere in the file (see §25.2's "skip not + // fail" guarantee). + constexpr const char* kDoublyBroken = R"JSON({ + "ivcurve": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "now": "2026-05-07T09:54:21.752997491Z", + "expirationDate": 20260610, + "exerciseStyle": "european", + "interestRate": 0.01, + "underlyingType": "futures", + "isPayingDividends": false, + "marketData": { + "id": "CME_MINI:ESM2026;E2C;20260610", + "time": "2026-05-07T09:54:18Z", + "underlying": { + "id": "CME_MINI:ESM2026", "mode": "realtime", + "time": "2026-05-07T09:54:18Z", + "bid": null, "ask": null, "last": null + }, + "puts": { + "0.": "not an object at all", + "4500.": { + "id": "CME_MINI:E2C260610P4500", "mode": "realtime", + "time": "2026-05-07T09:25:41Z", + "bid": {"price": 0.5, "size": 168, "time": "2026-05-07T09:25:41Z"}, + "ask": {"price": 0.85, "size": 168, "time": "2026-05-07T09:25:41Z"}, + "last": null + } + }, + "calls": {} + } + } + })JSON"; + const auto path = write_fixture("doubly_broken.json", kDoublyBroken); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + CHECK(parsed->options.size() == 1); + REQUIRE(parsed->skipped_options.size() == 1); + CHECK(parsed->skipped_options[0].find("0.") != std::string::npos); +} + +TEST_CASE("preserves null bid/ask/last and the epoch sentinel without correction", + "[parser][regression]") { + const auto path = write_fixture("epoch_and_nulls.json", default_valid_json()); + std::string error; + const auto parsed = ope::parse_ivcurve_file(path, error); + + REQUIRE(parsed.has_value()); + const auto& call = parsed->options[std::string_view(parsed->options[0].option_id).find("C4600") != + std::string_view::npos + ? 0 + : 1]; + CHECK_FALSE(call.bid.price.has_value()); + CHECK_FALSE(call.ask.price.has_value()); + CHECK_FALSE(call.last.price.has_value()); + REQUIRE(call.quote_time_ns.has_value()); + CHECK(*call.quote_time_ns == 0); // 1970-01-01T00:00:00Z sentinel, preserved verbatim +} diff --git a/tests/time_util_test.cpp b/tests/time_util_test.cpp new file mode 100644 index 0000000..bd77e02 --- /dev/null +++ b/tests/time_util_test.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +#include "ope/time_util.hpp" + +using namespace ope::timeutil; + +TEST_CASE("parses a timestamp with fractional seconds", "[time_util]") { + const auto base = parse_iso8601_ns("2026-05-25T14:59:01Z"); + const auto with_frac = parse_iso8601_ns("2026-05-25T14:59:01.800326283Z"); + REQUIRE(base.has_value()); + REQUIRE(with_frac.has_value()); + REQUIRE(*with_frac == *base + 800326283LL); +} + +TEST_CASE("parses a timestamp without a fraction", "[time_util]") { + const auto ns = parse_iso8601_ns("2026-05-25T14:58:59Z"); + REQUIRE(ns.has_value()); + REQUIRE(*ns % 1'000'000'000LL == 0); +} + +TEST_CASE("epoch sentinel parses to exactly 0, not an error", "[time_util]") { + const auto ns = parse_iso8601_ns("1970-01-01T00:00:00Z"); + REQUIRE(ns.has_value()); + REQUIRE(*ns == 0); +} + +TEST_CASE("malformed timestamp is a parse failure, not silently 0", "[time_util]") { + REQUIRE_FALSE(parse_iso8601_ns("not-a-timestamp").has_value()); + REQUIRE_FALSE(parse_iso8601_ns("").has_value()); + REQUIRE_FALSE(parse_iso8601_ns("2026-13-99T99:99:99Z").has_value()); +} + +TEST_CASE("rejects a day that doesn't exist in the given month", "[time_util]") { + // April, June, September, November only have 30 days. + REQUIRE_FALSE(parse_iso8601_ns("2026-04-31T00:00:00Z").has_value()); + // February 30th never exists. + REQUIRE_FALSE(parse_iso8601_ns("2026-02-30T00:00:00Z").has_value()); + // 2026 is not a leap year. + REQUIRE_FALSE(parse_iso8601_ns("2026-02-29T00:00:00Z").has_value()); +} + +TEST_CASE("accepts February 29th in a leap year", "[time_util]") { + const auto ns = parse_iso8601_ns("2024-02-29T00:00:00Z"); + REQUIRE(ns.has_value()); +} + +TEST_CASE("accepts the last real day of every month in a non-leap year", "[time_util]") { + for (const auto& [ymd, expect_ok] : std::vector>{ + {"2026-01-31T00:00:00Z", true}, {"2026-02-28T00:00:00Z", true}, + {"2026-03-31T00:00:00Z", true}, {"2026-04-30T00:00:00Z", true}, + {"2026-05-31T00:00:00Z", true}, {"2026-06-30T00:00:00Z", true}, + {"2026-07-31T00:00:00Z", true}, {"2026-08-31T00:00:00Z", true}, + {"2026-09-30T00:00:00Z", true}, {"2026-10-31T00:00:00Z", true}, + {"2026-11-30T00:00:00Z", true}, {"2026-12-31T00:00:00Z", true}, + }) { + INFO("timestamp: " << ymd); + CHECK(parse_iso8601_ns(ymd).has_value() == expect_ok); + } +} From 3ee9fe3e3946417a15530e08bfbbe6c31323012f Mon Sep 17 00:00:00 2001 From: Benedichuk Margarita Date: Sat, 18 Jul 2026 16:06:38 +0300 Subject: [PATCH 4/5] Add .gitattributes for shell script line endings --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf From 9aeb6172a2451f214d11c71ab8fe1b2abdf241af Mon Sep 17 00:00:00 2001 From: Benedichuk Margarita Date: Sat, 18 Jul 2026 16:06:55 +0300 Subject: [PATCH 5/5] Add zip import scripts and README usage section --- README.md | 31 ++++++++++++++++++------------- scripts/import_folder.sh | 36 ++++++++++++++++++++++++++++++++++++ scripts/import_zip.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 scripts/import_folder.sh create mode 100644 scripts/import_zip.sh diff --git a/README.md b/README.md index 34bcbfd..34d8b4b 100644 --- a/README.md +++ b/README.md @@ -76,19 +76,6 @@ a file or a directory (walked recursively) into ClickHouse: ./build/ope insert data/CME # non-interactive one-shot (same command) ``` -Files are distributed to a pool of worker threads through a blocking queue; each -worker parses JSON and streams rows into ClickHouse in batches. Connection and -tuning are read from the environment (defaults in parentheses): - -| Variable | Purpose | -| --- | --- | -| `CLICKHOUSE_HOST` (`localhost`), `CLICKHOUSE_PORT` (`9000`) | server address (native TCP) | -| `CLICKHOUSE_DB` (`options_pricing`), `CLICKHOUSE_USER` (`default`), `CLICKHOUSE_PASSWORD` (empty) | credentials | -| `OPE_WORKERS` (CPU count) | number of parser/inserter threads | -| `OPE_BATCH_SIZE` (`50000`) | rows accumulated before a batch `INSERT` | - -At the end of a run the tool prints a summary: - ```text [insert] done in 24.0s: 30564 ok, 0 failed, 2067514 option rows, 15 option(s) skipped (invalid strike) ``` @@ -103,6 +90,24 @@ At the end of a run the tool prints a summary: no live quote at all). Each one is logged as `[insert] : skipped option ` — never silently discarded. +## Importing from zip archives + +Data provider ships one zip per exchange (`CME.zip`, `NASDAQ.zip`, ...). Two +helper scripts in `scripts/` extract into `data/` and run `ope insert` for you +(run from inside the dev container): + +```bash +# one archive +bash scripts/import_zip.sh /path/to/CME.zip + +# whole download folder as-is (e.g. the Google Drive download folder, +# zips side by side, no extra structure) — extracts all of them, then +# runs one insert pass over data/ +bash scripts/import_folder.sh /path/to/drive-download-folder +``` + +Both accept an optional second argument to override the `data/` target dir. + ## Tests ```bash diff --git a/scripts/import_folder.sh b/scripts/import_folder.sh new file mode 100644 index 0000000..71cd720 --- /dev/null +++ b/scripts/import_folder.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Drop-in importer for a folder of per-exchange zip archives, shaped exactly +# like the Google Drive download folder (CME.zip, NASDAQ.zip, ... side by side, +# no extra structure). Extracts every *.zip into data// and runs one +# parser/inserter pass over the whole data dir. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [ $# -lt 1 ]; then + echo "Usage: $0 [data_dir]" >&2 + exit 1 +fi + +src_dir="$1" +data_dir="${2:-$repo_root/data}" + +shopt -s nullglob +zips=("$src_dir"/*.zip) +shopt -u nullglob + +if [ ${#zips[@]} -eq 0 ]; then + echo "error: no .zip files found in $src_dir" >&2 + exit 1 +fi + +for zip_path in "${zips[@]}"; do + exchange="$(basename "$zip_path" .zip)" + target="$data_dir/$exchange" + mkdir -p "$target" + echo "[import_folder] extracting $zip_path -> $target" + python3 -m zipfile -e "$zip_path" "$target" +done + +echo "[import_folder] running parser on $data_dir" +"$repo_root/build/ope" insert "$data_dir" diff --git a/scripts/import_zip.sh b/scripts/import_zip.sh new file mode 100644 index 0000000..af80d02 --- /dev/null +++ b/scripts/import_zip.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Extract one exchange zip archive (e.g. CME.zip) into data// +# and run the parser/inserter on it. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [ $# -lt 1 ]; then + echo "Usage: $0 [data_dir]" >&2 + exit 1 +fi + +zip_path="$1" +data_dir="${2:-$repo_root/data}" + +if [ ! -f "$zip_path" ]; then + echo "error: file not found: $zip_path" >&2 + exit 1 +fi + +exchange="$(basename "$zip_path" .zip)" +target="$data_dir/$exchange" + +mkdir -p "$target" +echo "[import_zip] extracting $zip_path -> $target" +python3 -m zipfile -e "$zip_path" "$target" + +echo "[import_zip] running parser on $target" +"$repo_root/build/ope" insert "$target"