diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..a4a94a5 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-06-25 - [Binary Search for Chronological Time Series Arrays] +**Learning:** Time-series market data arrays (like OHLCV bars) are chronologically sorted in this codebase. O(n) array filtering operations (e.g. `bars.filter(b => b.time <= asOf)`) on these arrays within hot paths like backtesting loops cause significant performance overhead. +**Action:** Replace `Array.prototype.filter` with an O(log n) binary search lookup (e.g., a `findLastBarIndex` utility) when truncating or querying the latest available bar up to a specific time. Use `.slice(0, lastIndex + 1)` if the truncated array is needed, or just access the index directly if only the latest bar is required. diff --git a/package.json b/package.json index dd2a5c4..7f29e49 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,12 @@ "pnpm": { "onlyBuiltDependencies": [ "better-sqlite3" - ] + ], + "overrides": { + "ws": ">=8.21.0", + "cheerio>undici": ">=7.28.0", + "undici": "^6.27.0" + } }, "packageManager": "pnpm@10.33.2+sha512.a90faf6feeab71ad6c6e57f94e0fe1a12f5dcc22cd754db40ae9593eb6a3e0b6b12e3540218bb37ae083404b1f2ce6db2a4121e979829b4aff94b99f49da1cf8" -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e3f135..89f6056 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,11 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + ws: '>=8.21.0' + cheerio>undici: '>=7.28.0' + undici: ^6.27.0 + importers: .: @@ -33,8 +38,8 @@ importers: specifier: ^3.1.0 version: 3.1.0 undici: - specifier: ^6.20.0 - version: 6.25.0 + specifier: ^6.27.0 + version: 6.27.0 yaml: specifier: ^2.6.0 version: 2.8.3 @@ -1516,13 +1521,13 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici@6.25.0: - resolution: {integrity: sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg==} + undici@6.27.0: + resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} engines: {node: '>=18.17'} - undici@7.25.0: - resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} - engines: {node: '>=20.18.1'} + undici@8.5.0: + resolution: {integrity: sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==} + engines: {node: '>=22.19.0'} universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} @@ -1622,8 +1627,8 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.20.0: - resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 @@ -2281,7 +2286,7 @@ snapshots: parse5: 7.3.0 parse5-htmlparser2-tree-adapter: 7.1.0 parse5-parser-stream: 7.1.2 - undici: 7.25.0 + undici: 8.5.0 whatwg-mimetype: 4.0.0 chownr@1.1.4: {} @@ -2590,7 +2595,7 @@ snapshots: type-fest: 4.41.0 widest-line: 5.0.0 wrap-ansi: 9.0.2 - ws: 8.20.0 + ws: 8.21.0 yoga-layout: 3.2.1 optionalDependencies: '@types/react': 18.3.28 @@ -2986,9 +2991,9 @@ snapshots: undici-types@6.21.0: {} - undici@6.25.0: {} + undici@6.27.0: {} - undici@7.25.0: {} + undici@8.5.0: {} universalify@0.1.2: {} @@ -3083,7 +3088,7 @@ snapshots: wrappy@1.0.2: {} - ws@8.20.0: {} + ws@8.21.0: {} yaml@2.8.3: {} diff --git a/src/agent/backtestRunner.ts b/src/agent/backtestRunner.ts index e91de2c..76abfb8 100644 --- a/src/agent/backtestRunner.ts +++ b/src/agent/backtestRunner.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { loadConfig } from "../config/loader.js"; import { getDb } from "../storage/db.js"; import { getBacktestBroker } from "../broker/index.js"; -import { getStockOhlcv, getIndexOhlcv, type Bar } from "../data/sources/dnsePublic.js"; +import { getStockOhlcv, getIndexOhlcv, findLastBarIndex, type Bar } from "../data/sources/dnsePublic.js"; import { DISCOVERY_UNIVERSE, discoverTickers } from "../tools/discover.js"; import { setActiveAsOf } from "./clock.js"; import { runTeamAnalysis } from "./team/index.js"; @@ -298,8 +298,9 @@ export async function runBacktestSession( ); const vnindexAt = (asOf: number): number | null => { - const series = vnindex.filter((b) => b.time <= asOf); - return series.length ? series[series.length - 1]!.close : null; + // ⚡ Bolt: Replace O(n) filtering with O(log n) binary search lookup + const lastIdx = findLastBarIndex(vnindex, asOf); + return lastIdx !== -1 ? vnindex[lastIdx]!.close : null; }; const vnindexBaseline = vnindexAt(intervalTurns[0]!); if (vnindexBaseline == null) throw new Error(`no VNINDEX data at first ${interval.label} turn`); @@ -312,8 +313,11 @@ export async function runBacktestSession( throwIfAborted(cb.signal); const dateIso = ictLabel(asOf); const priceOverride = (sym: string): number | null => { - const series = bars[sym]?.filter((b) => b.time <= asOf) ?? []; - return series.length ? series[series.length - 1]!.close : null; + // ⚡ Bolt: Replace O(n) filtering with O(log n) binary search lookup + const symBars = bars[sym]; + if (!symBars || symBars.length === 0) return null; + const lastIdx = findLastBarIndex(symBars, asOf); + return lastIdx !== -1 ? symBars[lastIdx]!.close : null; }; broker.setPriceOverride(priceOverride); cb.onTurnStart?.({ asOf, dateIso }); diff --git a/src/data/sources/dnsePublic.ts b/src/data/sources/dnsePublic.ts index 151733c..0c14516 100644 --- a/src/data/sources/dnsePublic.ts +++ b/src/data/sources/dnsePublic.ts @@ -45,6 +45,28 @@ async function fetchOhlcs( return (await body.json()) as OhlcvSeries; } +/** + * Reusable binary search utility to find the index of the last bar with a time <= targetTime. + * Returns -1 if no such bar exists. + */ +export function findLastBarIndex(bars: Bar[], targetTime: number): number { + let left = 0; + let right = bars.length - 1; + let result = -1; + + while (left <= right) { + const mid = Math.floor((left + right) / 2); + if (bars[mid].time <= targetTime) { + result = mid; + left = mid + 1; + } else { + right = mid - 1; + } + } + + return result; +} + export function seriesToBars(s: OhlcvSeries): Bar[] { const out: Bar[] = []; for (let i = 0; i < s.t.length; i++) { @@ -68,7 +90,10 @@ function clipBars(bars: Bar[]): Bar[] { asOfClock.getStore()?.asOfSec != null || isAsOfOverridden(); if (!hasOverride) return bars; const asOf = nowSec(); - return bars.filter((b) => b.time <= asOf); + // ⚡ Bolt: Replace O(n) filtering with O(log n) binary search lookup + const lastIndex = findLastBarIndex(bars, asOf); + if (lastIndex === -1) return []; + return bars.slice(0, lastIndex + 1); } export async function getStockOhlcv(