Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-22.04

RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
&& apt-get -y install --no-install-recommends \
apt-transport-https ca-certificates dirmngr wget gnupg2 \
libssl-dev cmake ninja-build gdb python3 python3-pip \
&& rm -rf /var/lib/apt/lists/*

RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 \
&& echo "deb https://packages.clickhouse.com/deb stable main" | tee /etc/apt/sources.list.d/clickhouse.list \
&& apt-get update \
&& apt-get install -y clickhouse-client \
&& rm -rf /var/lib/apt/lists/*

RUN pip3 install --no-cache-dir \
clickhouse-connect \
clickhouse-driver \
pandas \
numpy \
scipy \
orjson \
pytest
28 changes: 28 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "Options Pricing Engine",
"dockerComposeFile": "../docker-compose.yml",
"service": "workspace",
"workspaceFolder": "/workspace",
"forwardPorts": [8123, 9000, 3000],
"portsAttributes": {
"8123": { "label": "ClickHouse HTTP" },
"9000": { "label": "ClickHouse TCP" },
"3000": { "label": "Grafana" }
},
"customizations": {
"vscode": {
"extensions": [
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"twxs.cmake",
"ms-python.python",
"grafana.grafana-vscode"
],
"settings": {
"terminal.integrated.defaultProfile.linux": "bash"
}
}
},
"postCreateCommand": "bash .devcontainer/postCreateCommand.sh",
"remoteUser": "vscode"
}
23 changes: 23 additions & 0 deletions .devcontainer/postCreateCommand.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash
set -e

echo "Waiting for ClickHouse to init..."
until clickhouse-client --host "${CLICKHOUSE_HOST:-clickhouse}" --query "SELECT 1" > /dev/null 2>&1; do
sleep 2
done
echo "ClickHouse ready."

echo "Tables in options_pricing:"
clickhouse-client --host "${CLICKHOUSE_HOST:-clickhouse}" \
--query "SHOW TABLES FROM options_pricing" || true

cat <<'EOF'

------------------------------------------------------------------
Dev environment ready.
ClickHouse HTTP : http://clickhouse:8123 (host: localhost:8123)
ClickHouse TCP : clickhouse:9000 (host: localhost:9000)
Grafana : http://localhost:3000 (admin / admin)
Raw data : ./data/CME/...
------------------------------------------------------------------
EOF
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.sh text eol=lf
24 changes: 24 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/data/**
!/data/
!/data/README.md
!/data/.gitkeep

.DS_Store

__pycache__/
*.py[cod]
.venv/
venv/
.env
*.egg-info/
.pytest_cache/
.ipynb_checkpoints/

/clickhouse/data/
/clickhouse/logs/
build/
*.log

# Claude Code project files — personal working notes, not shared project docs.
/CLAUDE.md
/.claude/
88 changes: 88 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
cmake_minimum_required(VERSION 3.20)
project(options_pricing_engine LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif()

include(FetchContent)

# --- nlohmann/json (header-only JSON parser) --------------------------------
FetchContent_Declare(
nlohmann_json
URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
)

# --- clickhouse-cpp (native TCP client, block/batch inserts) ----------------
FetchContent_Declare(
clickhouse_cpp
GIT_REPOSITORY https://github.com/ClickHouse/clickhouse-cpp.git
GIT_TAG v2.5.1
GIT_SHALLOW TRUE
)

# --- Catch2 (unit tests) -----------------------------------------------------
FetchContent_Declare(
catch2
GIT_REPOSITORY https://github.com/catchorg/Catch2.git
GIT_TAG v3.6.0
GIT_SHALLOW TRUE
)

FetchContent_MakeAvailable(nlohmann_json clickhouse_cpp catch2)

find_package(Threads REQUIRED)

add_executable(ope
src/main.cpp
src/time_util.cpp
src/ivcurve_parser.cpp
src/inserter.cpp
)

target_include_directories(ope
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include
# clickhouse-cpp public headers + its bundled contrib (abseil) headers
${clickhouse_cpp_SOURCE_DIR}
${clickhouse_cpp_SOURCE_DIR}/contrib/absl
)

target_link_libraries(ope
PRIVATE
clickhouse-cpp-lib
nlohmann_json::nlohmann_json
Threads::Threads
)

# --- Unit tests --------------------------------------------------------------
# Links the parser/time-util sources directly (not the `ope` binary) so tests
# don't need a ClickHouse connection: parsing is pure and DB-agnostic per
# parser_specification.md §21 Non-Goals.
enable_testing()

add_executable(ope_tests
src/time_util.cpp
src/ivcurve_parser.cpp
tests/time_util_test.cpp
tests/ivcurve_parser_test.cpp
)

target_include_directories(ope_tests
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
)

target_link_libraries(ope_tests
PRIVATE
nlohmann_json::nlohmann_json
Catch2::Catch2WithMain
)

list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
include(CTest)
include(Catch)
catch_discover_tests(ope_tests)
121 changes: 121 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# Option pricing engine

## HOWTO

Project should be opened via dev container:
- install docker and Dev container extension for VS code.
- press `f1` and type `reopen in dev container`
- all works!

The dev container brings up three services (see `docker-compose.yml`):
`clickhouse` (analytical store, schema auto-applied from `clickhouse/init/`),
`grafana` (http://localhost:3000, admin/admin) and the `workspace` you code in.

## Data

Raw CME `ivcurve` JSON snapshots live under `data/` (git-ignored). See
`data/README.md` for the folder layout and file format.

## Input format

One file = one snapshot of one option series, JSON wrapped in a root
`"ivcurve"` key:

```json
{
"ivcurve": {
"id": "CME_MINI:ESM2026;E2C;20260610",
"now": "2026-05-07T09:54:21.752997491Z",
"expirationDate": 20260610,
"exerciseStyle": "european",
"interestRate": 0.01,
"underlyingType": "futures",
"isPayingDividends": false,
"marketData": {
"underlying": { "id": "...", "mode": "...", "bid": {...}|null, "ask": {...}|null, "last": {...}|null },
"puts": { "<strike>": { "id": "...", "mode": "...", "bid": ..., "ask": ..., "last": ... } },
"calls": { "<strike>": { ... } }
}
}
}
```

Full field-by-field format, folder layout, and which exchanges use it: see
`data/README.md`.

## Output

Each successfully parsed file writes into four ClickHouse tables (schema:
`clickhouse/init/create_schema.sql`): one row in `underlyings`, one row in
`option_chains`, one row in `underlying_quotes`, and one row per surviving
put/call in `option_quotes`.

## Building the parser

From inside the dev container:

```bash
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release
cmake --build build -j
```

Dependencies (`nlohmann/json`, `clickhouse-cpp`) are fetched automatically by
CMake, so the first configure needs network access.

## Running the importer

The `ope` binary reads commands from an interactive event loop; `insert` imports
a file or a directory (walked recursively) into ClickHouse:

```bash
./build/ope # interactive console
> insert data/CME # import everything
> insert data/CME/CME-6AM2026/5AD_20260529.json # import one file
> quit

./build/ope insert data/CME # non-interactive one-shot (same command)
```

```text
[insert] done in 24.0s: 30564 ok, 0 failed, 2067514 option rows, 15 option(s) skipped (invalid strike)
```

- **ok / failed** — whole files that parsed successfully vs. were rejected
outright (missing required field, malformed enum value, bad JSON, wrong
root shape, etc.). A failed file is skipped entirely; the reason is
printed as `[insert] skip <file>: <error>`.
- **skipped (invalid strike)** — individual put/call entries dropped from an
otherwise-valid file because their strike key was malformed/non-finite/
non-positive (real data has a handful of `"0."` placeholder entries with
no live quote at all). Each one is logged as
`[insert] <file>: skipped option <detail>` — never silently discarded.

## Importing from zip archives

Data provider ships one zip per exchange (`CME.zip`, `NASDAQ.zip`, ...). Two
helper scripts in `scripts/` extract into `data/` and run `ope insert` for you
(run from inside the dev container):

```bash
# one archive
bash scripts/import_zip.sh /path/to/CME.zip

# whole download folder as-is (e.g. the Google Drive download folder,
# zips side by side, no extra structure) — extracts all of them, then
# runs one insert pass over data/
bash scripts/import_folder.sh /path/to/drive-download-folder
```

Both accept an optional second argument to override the `data/` target dir.

## Tests

```bash
cmake --build build --target ope_tests -j
./build/ope_tests
```

Unit tests cover `time_util` (timestamp parsing) and `ivcurve_parser`
(required-field/enum/strike validation, skip-not-fail behavior, real-shape
fixtures) — no ClickHouse connection needed, `ope_tests` only links the
parsing code.
Loading