From 890aea6cd686a47da4e538ab4661832b06b584a3 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 05:38:17 -0400 Subject: [PATCH 1/6] docs: add User Guide entry point and executable home example The published guide lacked a direct overview, and the homepage convenience example referenced undefined names. Link the existing task pages and verify a complete example against the documentation checker. --- docs/book-guide/index.md | 3 +- docs/index.md | 22 ++++++++---- docs/user-guide/index.md | 41 ++++++++++++++++++++++ mkdocs.yml | 1 + validation/check_documentation_examples.py | 1 + 5 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 docs/user-guide/index.md diff --git a/docs/book-guide/index.md b/docs/book-guide/index.md index 015c10da..65b74d3e 100644 --- a/docs/book-guide/index.md +++ b/docs/book-guide/index.md @@ -2,7 +2,8 @@ This map connects selected *Machine Learning for Trading, Third Edition* companion notebooks to the library workflows that implement the corresponding -backtest operations. The [tutorials](../getting-started/quickstart.md) work with +backtest operations. For the library task guide, start at the +[User Guide](../user-guide/index.md). The [tutorials](../getting-started/quickstart.md) work with bundled synthetic data, so the book and its datasets are optional. All book links below point to one checked companion revision. diff --git a/docs/index.md b/docs/index.md index 030587b4..149b8e99 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,11 +13,11 @@ how cash is reserved, and how results change when you match another framework's Define a strategy, pick a config profile, get results in 10 lines. [:octicons-arrow-right-24: Quickstart](getting-started/quickstart.md) -- :material-tune:{ .lg .middle } __40+ Configurable Knobs__ +- :material-tune:{ .lg .middle } __User Guide__ --- - Fill ordering, stop modes, cash policy, settlement, and account behavior - are explicit named parameters. - [:octicons-arrow-right-24: Configuration](user-guide/configuration.md) + Find the workflow for data, strategies, orders, account rules, costs, + risk, and result analysis. + [:octicons-arrow-right-24: User Guide](user-guide/index.md) - :material-check-all:{ .lg .middle } __Validated Against 4 Frameworks__ --- @@ -76,12 +76,22 @@ fills=2 final=$100300.00 Each `Engine` instance is single-use. Create a new instance for every independent run. -Or use the convenience function: +The convenience function accepts the same price panel and strategy directly: + ```python +import polars as pl from ml4t.backtest import run_backtest +from ml4t.backtest.example_data import ExampleRoundTrip, load_example_prices + +prices = load_example_prices("equity").filter(pl.col("asset") == "AAPL") +result = run_backtest(prices, ExampleRoundTrip("AAPL", 100), config="backtrader") +print(f"fills={len(result.fills)} final=${result.metrics['final_value']:.2f}") +``` -result = run_backtest(prices, BuyAndHold(), config="backtrader") + +```text +fills=2 final=$100300.00 ``` ## Why ML4T Backtest? diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md new file mode 100644 index 00000000..4ebe6f52 --- /dev/null +++ b/docs/user-guide/index.md @@ -0,0 +1,41 @@ +# User Guide + +Use this guide to build a backtest from prepared market data and strategy decisions, then inspect what the engine actually executed. The [quickstart](../getting-started/quickstart.md) gives you a complete first run. The topic pages below explain the choices you make when adapting that run to your own data and strategy. + +The code in the [Getting Started tutorials](../getting-started/quickstart.md) runs from the installed package against bundled synthetic data. The topic pages describe individual interfaces and settings. When a topic page shows only a method call, use its linked tutorial for a complete program and expected output. The [API reference](../api/index.md) gives the signatures and source docstrings for the released package. + +## Find the task you need + +| Task | Start here | Complete example | +|---|---|---| +| Prepare bars, signals, context, and timestamps | [Data Feed](data-feed.md) | [Bundled example data](../tutorials/data.md) | +| Write decisions and carry state between callbacks | [Strategies](strategies.md) and [Stateful Strategies](stateful-strategies.md) | [First backtest](../getting-started/quickstart.md) and [Risk and State](../tutorials/risk-and-state.md) | +| Choose order types and determine when fills can occur | [Order Types](orders.md) and [Execution Semantics](execution-semantics.md) | [Orders and Timing](../tutorials/orders-and-timing.md) | +| Allocate across assets and deal with different daily close times | [Rebalancing](rebalancing.md) | [Multi-asset Rebalancing](../tutorials/multiasset-rebalancing.md) | +| Set cash, margin, contract, and sizing rules | [Account Policies](accounts.md) and [Configuration](configuration.md) | [Accounts and Constraints](../tutorials/accounts-and-constraints.md) | +| Model commission, slippage, impact, and funding | [Market Impact and Execution Costs](market-impact.md) | [Costs and Funding](../tutorials/costs-and-funding.md) | +| Add position exits and portfolio limits | [Risk Management](risk-management.md) | [Risk and State](../tutorials/risk-and-state.md) | +| Compare explicit execution settings | [Profiles](profiles.md) | [Profiles and Parity](../tutorials/profiles-and-parity.md) | +| Export fills, trades, equity, and portfolio state | [Results and Analysis](results.md) | [Results and Analysis](../tutorials/results-and-analysis.md) | + +## Choose an asset example + +All five bundled panels are synthetic. They test the library workflow offline; they do not estimate a strategy's live performance or reproduce a book case study. The [data page](../tutorials/data.md) runs a round trip for each panel and states its price and volume assumptions. + +| Asset setting | Example | Setting to check before using your own data | +|---|---|---| +| Equity | [Equity round trip](../tutorials/data.md#equities) | Share precision, trading calendar, and corporate-action handling in the input | +| ETF | [ETF round trip](../tutorials/data.md#etfs) | Weight targets, rebalance timing, and cash available for the next fill | +| Futures | [Futures round trip](../tutorials/data.md#futures) | Contract multiplier, margin, and the timestamp of each market close | +| FX | [FX round trip](../tutorials/data.md#fx) | Pair quotation and the account currency used for valuation | +| Crypto perpetual | [Perpetual round trip](../tutorials/data.md#crypto-perpetuals) | Funding timestamps, rate sign, and the position held before each event | + +The examples use declared settings rather than implied market defaults. Start with the [configuration guide](configuration.md) when you replace a bundled panel with another instrument or venue. + +## From a book notebook to a library run + +The [Book Guide](../book-guide/index.md) maps checked companion notebooks to the matching library workflows. Begin with the library quickstart, then compare the notebook's data, decision time, fill time, account rules, and costs with the settings in your run. Keep the original notebook's research question separate from the engine configuration. The book links point to a pinned companion revision, and each linked tutorial states what its notebook adds. + +## Check a result + +Inspect [fills](results.md#fills), [trades](results.md#trades-dataframe), and [equity](results.md#equity-dataframe) before interpreting summary metrics. A missing fill can mean that an order was rejected, an order type did not trigger, the next eligible bar was absent, or buying power was insufficient. The [result tutorial](../tutorials/results-and-analysis.md) exports these records and joins event timestamps back to the input data. The [diagnostic handoff](../tutorials/diagnostic-handoff.md) shows the optional analysis package used for downstream metrics. diff --git a/mkdocs.yml b/mkdocs.yml index ccf8c73d..8d947cea 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -118,6 +118,7 @@ nav: - Concepts: - How It Works: concepts/how-it-works.md - User Guide: + - Overview: user-guide/index.md - Strategies: user-guide/strategies.md - Order Types: user-guide/orders.md - Execution Semantics: user-guide/execution-semantics.md diff --git a/validation/check_documentation_examples.py b/validation/check_documentation_examples.py index 064f7ec2..4f9f3ead 100644 --- a/validation/check_documentation_examples.py +++ b/validation/check_documentation_examples.py @@ -35,6 +35,7 @@ "preopen-mixed-rules", "readme-quickstart", "home-example", + "home-convenience", "quickstart-minimal", "smoke-equity", "smoke-etf", From 441164843fd70a1036f23811edc32c1d6e0fafe3 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 05:42:50 -0400 Subject: [PATCH 2/6] docs: verify diagnostic tearsheet example against optional viz extra The old result guide referenced an undefined analysis variable and omitted the visualization dependency. Run a complete HTML tearsheet example against the installed wheel in CI and release verification. --- .github/workflows/ci.yml | 3 ++- .github/workflows/release.yml | 3 ++- docs/user-guide/results.md | 43 ++++++++++++++++++++++++----------- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06102bf3..9744d368 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -368,8 +368,9 @@ jobs: - name: Exercise the optional diagnostic handoff against the wheel run: | - uv pip install --python "$RUNNER_TEMP/candidate-venv/bin/python" "ml4t-diagnostic==0.1.4" + uv pip install --python "$RUNNER_TEMP/candidate-venv/bin/python" "ml4t-diagnostic[viz]==0.1.4" "$RUNNER_TEMP/candidate-venv/bin/python" validation/check_documentation_examples.py docs/tutorials/diagnostic-handoff.md + "$RUNNER_TEMP/candidate-venv/bin/python" validation/check_documentation_examples.py docs/user-guide/results.md - name: Verify source provenance run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9634ea9f..0ee75dd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,8 +269,9 @@ jobs: "$RUNNER_TEMP/published-venv/bin/python" -I -c "import ml4t.backtest as package; assert package.__version__ == '${{ needs.preflight.outputs.version }}'" "$RUNNER_TEMP/published-venv/bin/python" validation/check_documentation_examples.py - uv pip install --python "$RUNNER_TEMP/published-venv/bin/python" "ml4t-diagnostic==0.1.4" + uv pip install --python "$RUNNER_TEMP/published-venv/bin/python" "ml4t-diagnostic[viz]==0.1.4" "$RUNNER_TEMP/published-venv/bin/python" validation/check_documentation_examples.py docs/tutorials/diagnostic-handoff.md + "$RUNNER_TEMP/published-venv/bin/python" validation/check_documentation_examples.py docs/user-guide/results.md - name: Download and verify GitHub release assets env: diff --git a/docs/user-guide/results.md b/docs/user-guide/results.md index 91b9aaee..bdfeed92 100644 --- a/docs/user-guide/results.md +++ b/docs/user-guide/results.md @@ -495,8 +495,10 @@ analysis_net = portfolio_analysis_from_result(results_net, calendar="crypto") ``` !!! note "Requires ml4t-diagnostic" - Install with `pip install ml4t-diagnostic`. The import is deferred so ml4t-backtest - works standalone without ml4t-diagnostic installed. + Add `ml4t-diagnostic==0.1.4` for portfolio analysis. The full HTML + tearsheet also requires the visualization extra: + `uv add 'ml4t-diagnostic[viz]==0.1.4'`. Backtest imports and runs + without this optional package. ### Trade Records @@ -515,20 +517,35 @@ The bridge exports all cost decomposition fields (`gross_pnl`, `net_return`, `to ### Full Tearsheet -Pass all result data for the richest tearsheet (up to 24 sections): +Use the diagnostic integration function to build the HTML report from a +complete Backtest result. This example uses the bundled synthetic equity panel +and writes `tearsheet.html` in the current directory. + ```python -from ml4t.diagnostic.visualization.backtest import generate_backtest_tearsheet - -html = generate_backtest_tearsheet( - trades=result.to_trades_dataframe(), - returns=analysis.returns, - equity_curve=result.to_equity_dataframe(), - metrics=result.metrics, - template="full", - title="My Strategy — Full Report", - output_path="tearsheet.html", +from pathlib import Path + +import polars as pl +from ml4t.backtest import BacktestConfig, DataFeed, Engine +from ml4t.backtest.example_data import ExampleRoundTrip, load_example_prices +from ml4t.diagnostic.integration import generate_tearsheet_from_result + +prices = load_example_prices("equity").filter(pl.col("asset") == "AAPL") +result = Engine( + DataFeed(prices_df=prices), + ExampleRoundTrip("AAPL", 100), + BacktestConfig(initial_cash=100_000), +).run() +html = generate_tearsheet_from_result( + result, title="AAPL example", template="full", output_path="tearsheet.html" ) +assert " +```text +html_saved=True fills=2 ``` #### Metrics Keys That Enable Tearsheet Sections From b2d2702df6b338d7bd24443a36f3f24bc950e136 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 05:46:39 -0400 Subject: [PATCH 3/6] docs: state User Guide snippet context and narrow unsupported claims Explain the prerequisites for reference snippets on each topic page. Remove absolute claims about framework equivalence, vectorization, and an incorrect test count. --- docs/user-guide/accounts.md | 2 ++ docs/user-guide/configuration.md | 4 +++- docs/user-guide/data-feed.md | 2 ++ docs/user-guide/execution-semantics.md | 4 +++- docs/user-guide/market-impact.md | 2 ++ docs/user-guide/orders.md | 2 ++ docs/user-guide/profiles.md | 11 ++++++----- docs/user-guide/rebalancing.md | 2 ++ docs/user-guide/results.md | 2 ++ docs/user-guide/risk-management.md | 2 ++ docs/user-guide/stateful-strategies.md | 10 ++++++---- docs/user-guide/strategies.md | 2 ++ 12 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/user-guide/accounts.md b/docs/user-guide/accounts.md index 82f8d254..062d7488 100644 --- a/docs/user-guide/accounts.md +++ b/docs/user-guide/accounts.md @@ -12,6 +12,8 @@ The configuration is intentionally simple: instead of switching between account "types", you set the policy flags directly and let the broker enforce the resulting buying-power rules. +The short `Engine` and `Broker` snippets below assume a prepared feed and strategy. Run the linked accounts tutorial for complete inputs, orders, and result records. + ## Quick Example ```python diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index b9eb4bc1..ee997264 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -1,6 +1,6 @@ # Configuration -`BacktestConfig` is the single source of truth for all backtest behavior. Every behavioral difference between frameworks is a named parameter -- no subclassing or monkey-patching required. +`BacktestConfig` records the execution and account settings used for a run. Named profiles choose settings for the supported comparison scenarios; they do not reproduce every behavior of another framework. It is also the canonical serializable backtest preset: @@ -10,6 +10,8 @@ It is also the canonical serializable backtest preset: This keeps the input simple while still giving you an exact replayable record of what ran. +Configuration snippets isolate individual settings and assume the named classes and enums have been imported. The [profiles tutorial](../tutorials/profiles-and-parity.md) runs a complete comparison with declared inputs. + ## Creating a Config ```python diff --git a/docs/user-guide/data-feed.md b/docs/user-guide/data-feed.md index 4fb58088..0cc43413 100644 --- a/docs/user-guide/data-feed.md +++ b/docs/user-guide/data-feed.md @@ -4,6 +4,8 @@ The [multi-asset rebalancing tutorial](../tutorials/multiasset-rebalancing.md) r `DataFeed` converts a Polars DataFrame into per-bar data for the engine. It handles partitioning by timestamp, multi-asset iteration, optional signals/context data, and additive quote caches for execution-aware workloads. +Schema snippets using `[...]` stand for user-supplied columns; they are not standalone datasets. Run the [bundled data examples](../tutorials/data.md) for complete panels and checked output. + ## Required Columns With the default `FeedSpec`, the prices DataFrame must include: diff --git a/docs/user-guide/execution-semantics.md b/docs/user-guide/execution-semantics.md index 42601d78..9ac9a7a8 100644 --- a/docs/user-guide/execution-semantics.md +++ b/docs/user-guide/execution-semantics.md @@ -1,9 +1,11 @@ # Execution Semantics -This is the reference for how orders execute in ml4t-backtest. Every behavioral detail described here corresponds to a named config parameter, so you can tune or override it. +This page describes when orders are eligible to fill and how the engine processes them. Configuration controls the listed execution choices; the lifecycle and causal ordering rules remain part of the engine contract. Run the [order and timing tutorial](../tutorials/orders-and-timing.md) to compare fills on one fixed price panel. +The configuration fragments below change one execution choice at a time and assume their enum imports. Run the linked orders tutorial to inspect the resulting fills. + ## Execution Timing ### NEXT_BAR (Default) diff --git a/docs/user-guide/market-impact.md b/docs/user-guide/market-impact.md index 9090c5d0..147861a3 100644 --- a/docs/user-guide/market-impact.md +++ b/docs/user-guide/market-impact.md @@ -4,6 +4,8 @@ The [costs and funding tutorial](../tutorials/costs-and-funding.md) reconciles e Realistic backtesting requires modeling the costs of executing trades. ml4t-backtest provides three layers of cost modeling: commission, slippage, and market impact. +Individual cost-model snippets assume a configured backtest and, where shown, an existing `result`. The linked costs tutorial runs complete gross-to-net comparisons. + ## Cost Layers | Layer | What It Models | Config | diff --git a/docs/user-guide/orders.md b/docs/user-guide/orders.md index 32db9d74..21c861a6 100644 --- a/docs/user-guide/orders.md +++ b/docs/user-guide/orders.md @@ -4,6 +4,8 @@ Orders are submitted from a strategy via `broker.submit_order()`. The default or Run the [order and timing tutorial](../tutorials/orders-and-timing.md) to compare fills on one fixed price panel. +The order calls below assume `broker` is the argument passed to a strategy callback. Run the linked orders tutorial for a complete strategy, feed, and fill records. + ## Market Orders Execute at the configured execution price (open or close, depending on profile): diff --git a/docs/user-guide/profiles.md b/docs/user-guide/profiles.md index cd3b665d..ee5ce013 100644 --- a/docs/user-guide/profiles.md +++ b/docs/user-guide/profiles.md @@ -4,8 +4,9 @@ The [profiles and parity tutorial](../tutorials/profiles-and-parity.md) runs a one-setting comparison and explains the supported evidence boundaries. Profiles are pre-configured `BacktestConfig` settings for framework-specific execution semantics. -The retained evidence in this page states which pinned framework scenarios currently match -exactly and which remain release-blocking. +The retained evidence on this page identifies supported framework comparisons, their input settings, and their stated tolerances. + +Examples using `prices` and `strategy` assume those inputs already exist. The linked profiles tutorial supplies both and compares one setting at a time. ## Available Profiles @@ -14,9 +15,9 @@ exactly and which remain release-blocking. | Profile | Description | |---------|-------------| | `default` | Sensible defaults for general use with integer-share execution | -| `fast` | Zero-cost, integer-share execution -- fastest possible execution | -| `backtrader` | Match Backtrader's default behavior | -| `vectorbt` | Match VectorBT's default behavior (including fractional shares) | +| `fast` | Zero-cost, integer-share settings for iteration; performance depends on the workload | +| `backtrader` | Settings used for the documented Backtrader comparisons | +| `vectorbt` | Settings used for the documented VectorBT comparisons, including fractional shares | | `zipline` | Match the documented Zipline comparison protocol | | `lean` | Match the frozen LEAN daily US-equity comparison protocol | | `realistic` | Conservative settings for production | diff --git a/docs/user-guide/rebalancing.md b/docs/user-guide/rebalancing.md index 91f2d41d..15a399d9 100644 --- a/docs/user-guide/rebalancing.md +++ b/docs/user-guide/rebalancing.md @@ -4,6 +4,8 @@ The [multi-asset rebalancing tutorial](../tutorials/multiasset-rebalancing.md) r For multi-asset strategies that target portfolio weights, the broker provides `rebalance_to_weights()` and the execution module provides a `TargetWeightExecutor` for advanced control. +The broker calls below belong inside a strategy callback. The linked multi-asset tutorial supplies the feed, strategy, and result checks. + ## Simple Rebalancing ```python diff --git a/docs/user-guide/results.md b/docs/user-guide/results.md index bdfeed92..53330a82 100644 --- a/docs/user-guide/results.md +++ b/docs/user-guide/results.md @@ -28,6 +28,8 @@ For reproducibility, `BacktestResult` also exposes: - `result.to_parquet(...)`, which writes `config.yaml`, `spec.yaml`, and `predictions.parquet` when available +Snippets using `result` assume an existing `BacktestResult` from `Engine.run()`. The linked export tutorial supplies the complete run and checks each exported table. + ## Metrics ```python diff --git a/docs/user-guide/risk-management.md b/docs/user-guide/risk-management.md index bc4dc927..312aaac6 100644 --- a/docs/user-guide/risk-management.md +++ b/docs/user-guide/risk-management.md @@ -5,6 +5,8 @@ portfolio reductions, and the recovery between breaches from one declared price ml4t-backtest has two levels of risk management: **position rules** (per-position exits) and **portfolio limits** (portfolio-wide constraints). Position rules are the primary tool -- they automatically evaluate on every bar and generate exit orders when triggered. +Rule snippets below assume a strategy or broker with current positions and prices. The linked risk tutorial runs the full event sequence and inspects reduction fills. + ## Position Rules ### StopLoss diff --git a/docs/user-guide/stateful-strategies.md b/docs/user-guide/stateful-strategies.md index d7e70bba..7de9c75e 100644 --- a/docs/user-guide/stateful-strategies.md +++ b/docs/user-guide/stateful-strategies.md @@ -1,19 +1,21 @@ # Stateful Strategies -Stateful strategies are the reason event-driven backtesting exists. In a vectorized framework, every signal is computed in advance from historical data alone. In an event-driven engine, each trading decision can depend on the **entire history of prior decisions** — fills, P&L, position state, equity path. This feedback loop is impossible to vectorize. +Use a stateful strategy when a later decision depends on earlier order outcomes, cash, positions, or equity. A signal array computed from price history alone does not capture those execution outcomes. The event loop passes updated broker state into each decision. + +The strategy classes below illustrate individual state patterns. Run [the complete strategy examples](https://github.com/ml4t/backtest/blob/main/examples/stateful_strategies.py) or the [risk and state tutorial](../tutorials/risk-and-state.md) for a complete feed and result. ## When You Need Event-Driven Use vectorized backtesting when your signal is a pure function of price history: ``` -signal[t] = f(prices[0:t]) # No feedback — vectorizable +signal[t] = f(prices[0:t]) # No execution feedback ``` Use event-driven backtesting when your trading decision depends on execution state: ``` -action[t] = g(prices[0:t], fills[0:t], equity[0:t]) # Feedback — requires event loop +action[t] = g(prices[0:t], fills[0:t], equity[0:t]) # Update execution state in order ``` Five categories of stateful patterns: @@ -401,7 +403,7 @@ def test_pyramiding_respects_max_levels(): assert max(strategy.pyramid_levels.values()) <= 3 ``` -See `examples/test_stateful_strategies.py` for 45+ test methods covering all five patterns. +See `examples/stateful_strategies.py` for complete implementations and `examples/test_stateful_strategies.py` for behavioral checks. ## In the book diff --git a/docs/user-guide/strategies.md b/docs/user-guide/strategies.md index ea6449eb..4fc52c7e 100644 --- a/docs/user-guide/strategies.md +++ b/docs/user-guide/strategies.md @@ -2,6 +2,8 @@ Every strategy subclasses `Strategy` and implements `on_data`. The broker is your interface to the market -- use it to submit orders, query positions, and set risk rules. +The `MyStrategy` class sketches the callback interface. Later `broker` calls belong inside a callback; the [quickstart](../getting-started/quickstart.md) supplies a complete, checked run. + ## Strategy Interface ```python From 1026d7441d8438e46d7979d56e9b28b71159b37c Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 06:10:25 -0400 Subject: [PATCH 4/6] docs: verify rendered guide links in CI --- .github/workflows/ci.yml | 3 + .github/workflows/docs.yml | 3 + docs/index.md | 20 +-- docs/user-guide/profiles.md | 20 +-- tests/contracts/test_documentation_links.py | 73 +++++++++++ validation/check_documentation_links.py | 137 ++++++++++++++++++++ validation/release_checks.txt | 1 + 7 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 tests/contracts/test_documentation_links.py create mode 100644 validation/check_documentation_links.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9744d368..6c6ba5e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,6 +295,9 @@ jobs: - name: Build documentation with strict validation run: uv run mkdocs build --strict + - name: Check rendered guide links + run: uv run python validation/check_documentation_links.py --site site + - name: Verify rendered documentation identity run: >- uv run python validation/check_documentation_identity.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ef87a5ab..2d96b393 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -43,6 +43,9 @@ jobs: - name: Build docs run: uv run mkdocs build --strict + - name: Check rendered guide links + run: uv run python validation/check_documentation_links.py --site site + - name: Verify rendered documentation identity run: >- uv run python validation/check_documentation_identity.py diff --git a/docs/index.md b/docs/index.md index 149b8e99..cd717e90 100644 --- a/docs/index.md +++ b/docs/index.md @@ -132,19 +132,19 @@ The parity protocol disables transaction costs and position rules on both sides. | Real strategy | Pinned framework | Current result | Evidence | |---|---|---|---| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000015); terminal within $0.01 (raw gap $0.00000015) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,426 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001910); terminal within $0.01 (raw gap $0.00001870) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00000170); terminal within $0.01 (raw gap $0.00000160) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 4,027 valuations within $0.01 (max raw gap $0.00000190); terminal within $0.01 (raw gap $0.00000030) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | @@ -156,19 +156,19 @@ The table reports engine-call wall time for all 17 correctness-passing pairs. Th | Real strategy | Pinned framework | Framework median (95% CI), s | ML4T median (95% CI), s | Framework / ML4T median | |---|---|---:|---:|---:| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.192 (0.176-0.493) | 0.414 (0.413-0.417) | 0.462x | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 9.631 (9.428-9.710) | 0.439 (0.434-0.444) | 21.954x | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 3.888 (3.878-3.959) | 0.621 (0.618-0.626) | 6.256x | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.638 (2.516-2.707) | 0.750 (0.743-0.757) | 3.520x | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 6.639 (5.831-8.332) | 0.428 (0.422-1.235) | 15.504x | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.828 (2.737-2.952) | 1.327 (0.657-2.183) | 2.131x | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.144 (0.142-0.146) | 0.145 (0.145-0.146) | 0.988x | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 0.434 (0.430-0.440) | 0.147 (0.146-0.149) | 2.947x | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | 0.953 (0.936-1.164) | 0.158 (0.155-0.164) | 6.033x | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 17.116 (17.069-17.633) | 25.738 (24.983-27.163) | 0.665x | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 668.611 (642.993-718.202) | 26.887 (26.406-29.406) | 24.868x | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 117.445 (116.308-118.391) | 26.821 (25.473-27.644) | 4.379x | @@ -182,7 +182,7 @@ The scenario matrix contains synthetic conformance tests. "Exact" means terminal | Profile | Pinned framework | Required scenarios | Evidence | |---|---|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | @@ -191,7 +191,7 @@ The synthetic stress workload contains 250 assets and 5,040 daily sessions (1,26 | Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence | |---|---|---:|---:|---:|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 417,941 | 211,322 | 1,345,348.850000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.560000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | diff --git a/docs/user-guide/profiles.md b/docs/user-guide/profiles.md index ee5ce013..547f1575 100644 --- a/docs/user-guide/profiles.md +++ b/docs/user-guide/profiles.md @@ -174,19 +174,19 @@ The parity protocol disables transaction costs and position rules on both sides. | Real strategy | Pinned framework | Current result | Evidence | |---|---|---|---| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000015); terminal within $0.01 (raw gap $0.00000015) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,426 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001910); terminal within $0.01 (raw gap $0.00001870) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00000170); terminal within $0.01 (raw gap $0.00000160) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 4,027 valuations within $0.01 (max raw gap $0.00000190); terminal within $0.01 (raw gap $0.00000030) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | @@ -198,19 +198,19 @@ The table reports engine-call wall time for all 17 correctness-passing pairs. Th | Real strategy | Pinned framework | Framework median (95% CI), s | ML4T median (95% CI), s | Framework / ML4T median | |---|---|---:|---:|---:| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.192 (0.176-0.493) | 0.414 (0.413-0.417) | 0.462x | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 9.631 (9.428-9.710) | 0.439 (0.434-0.444) | 21.954x | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 3.888 (3.878-3.959) | 0.621 (0.618-0.626) | 6.256x | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.638 (2.516-2.707) | 0.750 (0.743-0.757) | 3.520x | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 6.639 (5.831-8.332) | 0.428 (0.422-1.235) | 15.504x | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.828 (2.737-2.952) | 1.327 (0.657-2.183) | 2.131x | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.144 (0.142-0.146) | 0.145 (0.145-0.146) | 0.988x | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 0.434 (0.430-0.440) | 0.147 (0.146-0.149) | 2.947x | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | 0.953 (0.936-1.164) | 0.158 (0.155-0.164) | 6.033x | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 17.116 (17.069-17.633) | 25.738 (24.983-27.163) | 0.665x | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 668.611 (642.993-718.202) | 26.887 (26.406-29.406) | 24.868x | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 117.445 (116.308-118.391) | 26.821 (25.473-27.644) | 4.379x | @@ -224,7 +224,7 @@ The scenario matrix contains synthetic conformance tests. "Exact" means terminal | Profile | Pinned framework | Required scenarios | Evidence | |---|---|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | @@ -233,7 +233,7 @@ The synthetic stress workload contains 250 assets and 5,040 daily sessions (1,26 | Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence | |---|---|---:|---:|---:|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 417,941 | 211,322 | 1,345,348.850000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.560000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | diff --git a/tests/contracts/test_documentation_links.py b/tests/contracts/test_documentation_links.py new file mode 100644 index 00000000..d67ecb6d --- /dev/null +++ b/tests/contracts/test_documentation_links.py @@ -0,0 +1,73 @@ +"""Rendered guide links must resolve to pages, sections, and reachable URLs.""" + +from __future__ import annotations + +import importlib.util +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + +import pytest + +_ROOT = Path(__file__).parents[2] + + +def _load_checker(): + path = _ROOT / "validation" / "check_documentation_links.py" + spec = importlib.util.spec_from_file_location("ml4t_documentation_links", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _write_site(site: Path, href: str) -> None: + page = site / "user-guide" / "index.html" + page.parent.mkdir(parents=True, exist_ok=True) + page.write_text(f'') + destination = site / "tutorials" / "index.html" + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text('

Working

') + + +def test_rendered_page_and_fragment_links(tmp_path: Path) -> None: + checker = _load_checker() + _write_site(tmp_path, "../tutorials/#working-section") + assert checker.check_links(tmp_path) == (1, 0) + + _write_site(tmp_path, "../tutorials/#missing-section") + with pytest.raises(ValueError, match="missing anchor"): + checker.check_links(tmp_path) + + _write_site(tmp_path, "../absent/") + with pytest.raises(ValueError, match="missing target"): + checker.check_links(tmp_path) + + +def test_external_url_rejects_http_404(tmp_path: Path) -> None: + checker = _load_checker() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200 if self.path == "/present" else 404) + self.end_headers() + + def log_message(self, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever) + thread.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + _write_site(tmp_path, base + "/present") + assert checker.check_links(tmp_path) == (1, 1) + _write_site(tmp_path, base + "/absent") + with pytest.raises(ValueError, match="HTTP Error 404"): + checker.check_links(tmp_path) + finally: + server.shutdown() + thread.join() + server.server_close() diff --git a/validation/check_documentation_links.py b/validation/check_documentation_links.py new file mode 100644 index 00000000..7bb9812d --- /dev/null +++ b/validation/check_documentation_links.py @@ -0,0 +1,137 @@ +"""Check links in rendered documentation content, including section anchors.""" + +from __future__ import annotations + +import argparse +import posixpath +import time +from concurrent.futures import ThreadPoolExecutor +from html.parser import HTMLParser +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urljoin, urlsplit +from urllib.request import Request, urlopen + +_SITE_ORIGIN = "https://www.ml4trading.io" +_SITE_PREFIX = "/docs/backtest/" +_USER_AGENT = "ml4t-backtest-documentation-link-check/1.0" +_GUIDE_SECTIONS = {"user-guide", "book-guide"} + + +class ContentParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.in_article = False + self.links: list[str] = [] + self.ids: set[str] = set() + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = dict(attrs) + if tag == "article": + self.in_article = True + if identifier := values.get("id"): + self.ids.add(identifier) + if self.in_article: + attribute = "src" if tag == "img" else "href" if tag == "a" else None + if attribute and (link := values.get(attribute)): + self.links.append(link) + + def handle_endtag(self, tag: str) -> None: + if tag == "article": + self.in_article = False + + +def _page_url(path: Path, site: Path) -> str: + relative = path.relative_to(site).as_posix() + if relative == "index.html": + return _SITE_ORIGIN + _SITE_PREFIX + if relative.endswith("/index.html"): + return _SITE_ORIGIN + _SITE_PREFIX + relative[: -len("index.html")] + return _SITE_ORIGIN + _SITE_PREFIX + relative + + +def _target_path(site: Path, path: str) -> Path: + relative = unquote(path.removeprefix(_SITE_PREFIX)) + normalized = posixpath.normpath(relative).lstrip("/") + if path.endswith("/") or normalized == ".": + normalized = posixpath.join(normalized, "index.html") + target = (site / normalized).resolve() + if not target.is_relative_to(site.resolve()): + raise ValueError("link escapes the documentation site") + return target + + +def _check_external(url: str) -> str | None: + for attempt in range(3): + try: + request = Request(url, headers={"User-Agent": _USER_AGENT}) + with urlopen(request, timeout=15) as response: + if response.status >= 400: + return f"HTTP {response.status}" + return None + except (HTTPError, URLError, TimeoutError) as error: + if attempt == 2 or (isinstance(error, HTTPError) and error.code < 500): + return str(error) + time.sleep(attempt + 1) + return "unreachable" + + +def check_links(site: Path) -> tuple[int, int]: + site = site.resolve() + pages: dict[Path, ContentParser] = {} + for page in sorted(site.rglob("*.html")): + parser = ContentParser() + parser.feed(page.read_text(encoding="utf-8")) + pages[page.resolve()] = parser + + failures: list[str] = [] + external: set[str] = set() + checked = 0 + for page, content in pages.items(): + if page.relative_to(site).parts[0] not in _GUIDE_SECTIONS: + continue + for link in content.links: + checked += 1 + destination = urlsplit(urljoin(_page_url(page, site), link)) + if destination.scheme not in {"http", "https"}: + failures.append(f"{page.relative_to(site)}: unsupported link {link}") + continue + if destination.netloc not in {"www.ml4trading.io", "ml4trading.io"}: + external.add(destination._replace(fragment="").geturl()) + continue + if not destination.path.startswith(_SITE_PREFIX): + external.add(destination._replace(fragment="").geturl()) + continue + try: + target = _target_path(site, destination.path) + except ValueError as error: + failures.append(f"{page.relative_to(site)}: {link}: {error}") + continue + if not target.is_file(): + failures.append(f"{page.relative_to(site)}: {link}: missing target") + continue + if destination.fragment and target.suffix == ".html": + anchor = unquote(destination.fragment) + if anchor not in pages[target].ids: + failures.append(f"{page.relative_to(site)}: {link}: missing anchor") + + with ThreadPoolExecutor(max_workers=8) as executor: + for url, error in zip(sorted(external), executor.map(_check_external, sorted(external))): + if error: + failures.append(f"{url}: {error}") + if failures: + raise ValueError("Documentation link failures:\n" + "\n".join(failures)) + return checked, len(external) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--site", type=Path, default=Path("site")) + args = parser.parse_args() + checked, external = check_links(args.site) + print(f"Checked {checked} rendered content links and {external} external destinations") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/validation/release_checks.txt b/validation/release_checks.txt index a42181f6..a1cb731a 100644 --- a/validation/release_checks.txt +++ b/validation/release_checks.txt @@ -2,6 +2,7 @@ validation/check_artifacts.py validation/check_book_links.py validation/check_coverage.py validation/check_documentation_examples.py +validation/check_documentation_links.py validation/check_documentation_identity.py validation/check_security.py validation/build_framework_env.py From 8569e80ffa2724820c284ef3308d76108157b43b Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 06:22:42 -0400 Subject: [PATCH 5/6] docs: publish reachable framework links from parity claims --- README.md | 20 +++++++++---------- .../contracts/test_parity_claim_generation.py | 2 +- validation/METHODOLOGY.md | 20 +++++++++---------- validation/README.md | 20 +++++++++---------- validation/generate_parity_claims.py | 17 ++++++++++++---- 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index cb3785df..94b5d131 100644 --- a/README.md +++ b/README.md @@ -316,19 +316,19 @@ The parity protocol disables transaction costs and position rules on both sides. | Real strategy | Pinned framework | Current result | Evidence | |---|---|---|---| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000015); terminal within $0.01 (raw gap $0.00000015) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,426 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001910); terminal within $0.01 (raw gap $0.00001870) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00000170); terminal within $0.01 (raw gap $0.00000160) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 4,027 valuations within $0.01 (max raw gap $0.00000190); terminal within $0.01 (raw gap $0.00000030) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | @@ -340,19 +340,19 @@ The table reports engine-call wall time for all 17 correctness-passing pairs. Th | Real strategy | Pinned framework | Framework median (95% CI), s | ML4T median (95% CI), s | Framework / ML4T median | |---|---|---:|---:|---:| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.192 (0.176-0.493) | 0.414 (0.413-0.417) | 0.462x | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 9.631 (9.428-9.710) | 0.439 (0.434-0.444) | 21.954x | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 3.888 (3.878-3.959) | 0.621 (0.618-0.626) | 6.256x | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.638 (2.516-2.707) | 0.750 (0.743-0.757) | 3.520x | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 6.639 (5.831-8.332) | 0.428 (0.422-1.235) | 15.504x | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.828 (2.737-2.952) | 1.327 (0.657-2.183) | 2.131x | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.144 (0.142-0.146) | 0.145 (0.145-0.146) | 0.988x | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 0.434 (0.430-0.440) | 0.147 (0.146-0.149) | 2.947x | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | 0.953 (0.936-1.164) | 0.158 (0.155-0.164) | 6.033x | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 17.116 (17.069-17.633) | 25.738 (24.983-27.163) | 0.665x | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 668.611 (642.993-718.202) | 26.887 (26.406-29.406) | 24.868x | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 117.445 (116.308-118.391) | 26.821 (25.473-27.644) | 4.379x | @@ -366,7 +366,7 @@ The scenario matrix contains synthetic conformance tests. "Exact" means terminal | Profile | Pinned framework | Required scenarios | Evidence | |---|---|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | @@ -375,7 +375,7 @@ The synthetic stress workload contains 250 assets and 5,040 daily sessions (1,26 | Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence | |---|---|---:|---:|---:|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 417,941 | 211,322 | 1,345,348.850000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.560000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | diff --git a/tests/contracts/test_parity_claim_generation.py b/tests/contracts/test_parity_claim_generation.py index b232c0fe..a787e180 100644 --- a/tests/contracts/test_parity_claim_generation.py +++ b/tests/contracts/test_parity_claim_generation.py @@ -146,7 +146,7 @@ def test_claims_pin_every_advertised_framework_and_expose_failures( for pin in correctness["frameworks"].values(): assert pin["version"] in claims - assert pin["source"] in claims + assert generate_parity_claims._documentation_source(pin["source"]) in claims assert f"`{pin['profile']}`" in claims assert "16/16 exact" in claims assert "synthetic stress workload contains 250 assets" in claims diff --git a/validation/METHODOLOGY.md b/validation/METHODOLOGY.md index 295c3128..6ec597cd 100644 --- a/validation/METHODOLOGY.md +++ b/validation/METHODOLOGY.md @@ -356,19 +356,19 @@ The parity protocol disables transaction costs and position rules on both sides. | Real strategy | Pinned framework | Current result | Evidence | |---|---|---|---| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000015); terminal within $0.01 (raw gap $0.00000015) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,426 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001910); terminal within $0.01 (raw gap $0.00001870) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00000170); terminal within $0.01 (raw gap $0.00000160) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 4,027 valuations within $0.01 (max raw gap $0.00000190); terminal within $0.01 (raw gap $0.00000030) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | @@ -380,19 +380,19 @@ The table reports engine-call wall time for all 17 correctness-passing pairs. Th | Real strategy | Pinned framework | Framework median (95% CI), s | ML4T median (95% CI), s | Framework / ML4T median | |---|---|---:|---:|---:| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.192 (0.176-0.493) | 0.414 (0.413-0.417) | 0.462x | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 9.631 (9.428-9.710) | 0.439 (0.434-0.444) | 21.954x | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 3.888 (3.878-3.959) | 0.621 (0.618-0.626) | 6.256x | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.638 (2.516-2.707) | 0.750 (0.743-0.757) | 3.520x | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 6.639 (5.831-8.332) | 0.428 (0.422-1.235) | 15.504x | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.828 (2.737-2.952) | 1.327 (0.657-2.183) | 2.131x | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.144 (0.142-0.146) | 0.145 (0.145-0.146) | 0.988x | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 0.434 (0.430-0.440) | 0.147 (0.146-0.149) | 2.947x | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | 0.953 (0.936-1.164) | 0.158 (0.155-0.164) | 6.033x | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 17.116 (17.069-17.633) | 25.738 (24.983-27.163) | 0.665x | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 668.611 (642.993-718.202) | 26.887 (26.406-29.406) | 24.868x | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 117.445 (116.308-118.391) | 26.821 (25.473-27.644) | 4.379x | @@ -406,7 +406,7 @@ The scenario matrix contains synthetic conformance tests. "Exact" means terminal | Profile | Pinned framework | Required scenarios | Evidence | |---|---|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | @@ -415,7 +415,7 @@ The synthetic stress workload contains 250 assets and 5,040 daily sessions (1,26 | Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence | |---|---|---:|---:|---:|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 417,941 | 211,322 | 1,345,348.850000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.560000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | diff --git a/validation/README.md b/validation/README.md index 10bd5829..3613c984 100644 --- a/validation/README.md +++ b/validation/README.md @@ -32,19 +32,19 @@ The parity protocol disables transaction costs and position rules on both sides. | Real strategy | Pinned framework | Current result | Evidence | |---|---|---|---| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 1,995 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000010); terminal within $0.01 (raw gap $0.00000007) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 1,595 valuations within $0.01 (max raw gap $0.00000015); terminal within $0.01 (raw gap $0.00000015) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,426 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | fills equal at declared field precision; 2,108 valuations and terminal exact at 1e-8 | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001950); terminal within $0.01 (raw gap $0.00001880) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00001910); terminal within $0.01 (raw gap $0.00001870) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | fills equal at declared field precision; 4,146 valuations within $0.01 (max raw gap $0.00000170); terminal within $0.01 (raw gap $0.00000160) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | fills equal at declared field precision; 4,027 valuations within $0.01 (max raw gap $0.00000190); terminal within $0.01 (raw gap $0.00000030) | [real-strategy evidence](https://github.com/ml4t/backtest/blob/main/validation/REAL_STRATEGY_RESULTS.json) | @@ -56,19 +56,19 @@ The table reports engine-call wall time for all 17 correctness-passing pairs. Th | Real strategy | Pinned framework | Framework median (95% CI), s | ML4T median (95% CI), s | Framework / ML4T median | |---|---|---:|---:|---:| -| ETF allocation | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | +| ETF allocation | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.293 (0.290-0.298) | 0.424 (0.421-0.434) | 0.691x | | ETF allocation | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.192 (0.176-0.493) | 0.414 (0.413-0.417) | 0.462x | | ETF allocation | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 9.631 (9.428-9.710) | 0.439 (0.434-0.444) | 21.954x | | ETF allocation | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 3.888 (3.878-3.959) | 0.621 (0.618-0.626) | 6.256x | | ETF allocation | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.638 (2.516-2.707) | 0.750 (0.743-0.757) | 3.520x | -| CME futures | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | +| CME futures | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.290 (0.289-0.294) | 0.410 (0.406-0.958) | 0.708x | | CME futures | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 6.639 (5.831-8.332) | 0.428 (0.422-1.235) | 15.504x | | Crypto perpetual funding | [LEAN 18001](https://github.com/QuantConnect/Lean) | 2.828 (2.737-2.952) | 1.327 (0.657-2.183) | 2.131x | -| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | +| FX allocation (USD-quoted pairs) | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 0.288 (0.286-0.294) | 0.148 (0.147-0.149) | 1.947x | | FX allocation (USD-quoted pairs) | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 0.144 (0.142-0.146) | 0.145 (0.145-0.146) | 0.988x | | FX allocation (USD-quoted pairs) | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 0.434 (0.430-0.440) | 0.147 (0.146-0.149) | 2.947x | | FX allocation (USD-quoted pairs) | [LEAN 18001](https://github.com/QuantConnect/Lean) | 0.953 (0.936-1.164) | 0.158 (0.155-0.164) | 6.033x | -| US equity panel | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | +| US equity panel | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 3.806 (3.336-4.341) | 23.306 (21.906-25.833) | 0.163x | | US equity panel | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 17.116 (17.069-17.633) | 25.738 (24.983-27.163) | 0.665x | | US equity panel | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 668.611 (642.993-718.202) | 26.887 (26.406-29.406) | 24.868x | | US equity panel | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 117.445 (116.308-118.391) | 26.821 (25.473-27.644) | 4.379x | @@ -82,7 +82,7 @@ The scenario matrix contains synthetic conformance tests. "Exact" means terminal | Profile | Pinned framework | Required scenarios | Evidence | |---|---|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) | @@ -91,7 +91,7 @@ The synthetic stress workload contains 250 assets and 5,040 daily sessions (1,26 | Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence | |---|---|---:|---:|---:|---:|---| -| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | +| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://vectorbt.pro/) | 427,790 | 423,313 | 222,751 | 1,285,886.320000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 417,941 | 211,322 | 1,345,348.850000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.560000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | | `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) | diff --git a/validation/generate_parity_claims.py b/validation/generate_parity_claims.py index d4273459..080ffcfe 100644 --- a/validation/generate_parity_claims.py +++ b/validation/generate_parity_claims.py @@ -32,6 +32,13 @@ END_MARKER = "" GITHUB_EVIDENCE_ROOT = "https://github.com/ml4t/backtest/blob/main/validation" SCENARIO_FRAMEWORKS = load_framework_manifest().scenario_framework_ids +PRIVATE_VECTORBT_SOURCE = "https://github.com/polakowo/vectorbt.pro" +PUBLIC_VECTORBT_SOURCE = "https://vectorbt.pro/" + + +def _documentation_source(source: str) -> str: + """Link readers to the public product page when evidence names a private repository.""" + return PUBLIC_VECTORBT_SOURCE if source == PRIVATE_VECTORBT_SOURCE else source def _load_json(path: Path) -> dict[str, Any]: @@ -118,7 +125,9 @@ def render_claims( result = f"{len(passed)}/{len(required)} pass; blocked by scenario {failed_ids}" else: result = f"{len(passed)}/{len(required)} exact" - pinned_framework = f"[{pin['display_name']} {pin['version']}]({pin['source']})" + pinned_framework = ( + f"[{pin['display_name']} {pin['version']}]({_documentation_source(pin['source'])})" + ) rows.append( f"| `{pin['profile']}` | {pinned_framework} | {result} | " f"[scenario evidence]({correctness_url}) |" @@ -132,7 +141,7 @@ def render_claims( fills = checks["fills"]["expected_count"] trades = checks["trades"]["expected_count"] terminal_value = checks["final_value"]["canonical_expected"] - pinned_framework = f"[{target['display_name']} {target['version']}]({target['source']})" + pinned_framework = f"[{target['display_name']} {target['version']}]({_documentation_source(target['source'])})" scale_rows.append( f"| `{target['profile']}` | {pinned_framework} | {intents:,} | {fills:,} | " f"{trades:,} | {terminal_value:,.6f} | " @@ -159,7 +168,7 @@ def render_claims( } for record in required_real: target = real_targets[record["framework"]] - pinned_framework = f"[{target['display_name']} {target['version']}]({target['source']})" + pinned_framework = f"[{target['display_name']} {target['version']}]({_documentation_source(target['source'])})" fills = record["surfaces"]["fills"] equity = record["surfaces"]["equity"] terminal = record["surfaces"]["terminal"] @@ -201,7 +210,7 @@ def render_claims( performance_rows = [] for record in real_performance["records"]: target = real_targets[record["framework"]] - pinned_framework = f"[{target['display_name']} {target['version']}]({target['source']})" + pinned_framework = f"[{target['display_name']} {target['version']}]({_documentation_source(target['source'])})" framework_result = record["framework_engine"] ml4t_result = record["ml4t_engine"] framework_ci = framework_result["ci_95_seconds"] From 7ae9662c0b58b13e2aa7a35c277e34b1cfc2bf6f Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Thu, 24 Sep 2026 06:28:10 -0400 Subject: [PATCH 6/6] test: close failed HTTP responses in link checker --- validation/check_documentation_links.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/validation/check_documentation_links.py b/validation/check_documentation_links.py index 7bb9812d..e03dc0e3 100644 --- a/validation/check_documentation_links.py +++ b/validation/check_documentation_links.py @@ -70,6 +70,8 @@ def _check_external(url: str) -> str | None: return f"HTTP {response.status}" return None except (HTTPError, URLError, TimeoutError) as error: + if isinstance(error, HTTPError): + error.close() if attempt == 2 or (isinstance(error, HTTPError) and error.code < 500): return str(error) time.sleep(attempt + 1)