diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..21d90ca --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2023-10-24 - [O(N) to O(log N) for Chronological Lookups] +**Learning:** Found a major bottleneck in `src/agent/backtestRunner.ts` where large OHLCV market data arrays were being filtered iteratively with `array.filter((b) => b.time <= asOf)` to find the latest available bar. Given these arrays are natively sorted chronologically, doing an O(N) filter on thousands of bars in the inner loop (e.g. interval turns for all backtest symbols) scales poorly. +**Action:** Created and used `findLastBarIndex`, a reusable O(log N) binary search utility, when extracting subset views or current elements of OHLCV bars. Applied this specifically to `clipBars`, `vnindexAt`, and `priceOverride` to improve backtesting engine performance. diff --git a/package.json b/package.json index dd2a5c4..1b2068a 100644 --- a/package.json +++ b/package.json @@ -55,13 +55,13 @@ "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.1.0", "better-sqlite3": "^11.5.0", - "cheerio": "^1.0.0", - "ink": "^5.0.1", + "cheerio": "^1.2.0", + "ink": "^5.2.1", "ink-spinner": "^5.0.0", "ink-text-input": "^6.0.0", "react": "^18.3.1", "technicalindicators": "^3.1.0", - "undici": "^6.20.0", + "undici": "^7.28.0", "yaml": "^2.6.0", "zod": "^3.23.8" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e3f135..1d514ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,10 +15,10 @@ importers: specifier: ^11.5.0 version: 11.10.0 cheerio: - specifier: ^1.0.0 + specifier: ^1.2.0 version: 1.2.0 ink: - specifier: ^5.0.1 + specifier: ^5.2.1 version: 5.2.1(@types/react@18.3.28)(react@18.3.1) ink-spinner: specifier: ^5.0.0 @@ -33,8 +33,8 @@ importers: specifier: ^3.1.0 version: 3.1.0 undici: - specifier: ^6.20.0 - version: 6.25.0 + specifier: ^7.28.0 + version: 7.28.0 yaml: specifier: ^2.6.0 version: 2.8.3 @@ -1516,12 +1516,8 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici@6.25.0: - resolution: {integrity: sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg==} - engines: {node: '>=18.17'} - - undici@7.25.0: - resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==} + undici@7.28.0: + resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} universalify@0.1.2: @@ -1622,8 +1618,8 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - ws@8.20.0: - resolution: {integrity: sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} peerDependencies: bufferutil: ^4.0.1 @@ -2281,7 +2277,7 @@ snapshots: parse5: 7.3.0 parse5-htmlparser2-tree-adapter: 7.1.0 parse5-parser-stream: 7.1.2 - undici: 7.25.0 + undici: 7.28.0 whatwg-mimetype: 4.0.0 chownr@1.1.4: {} @@ -2590,7 +2586,7 @@ snapshots: type-fest: 4.41.0 widest-line: 5.0.0 wrap-ansi: 9.0.2 - ws: 8.20.0 + ws: 8.21.0 yoga-layout: 3.2.1 optionalDependencies: '@types/react': 18.3.28 @@ -2986,9 +2982,7 @@ snapshots: undici-types@6.21.0: {} - undici@6.25.0: {} - - undici@7.25.0: {} + undici@7.28.0: {} universalify@0.1.2: {} @@ -3083,7 +3077,7 @@ snapshots: wrappy@1.0.2: {} - ws@8.20.0: {} + ws@8.21.0: {} yaml@2.8.3: {} diff --git a/src/agent/backtestRunner.ts b/src/agent/backtestRunner.ts index e91de2c..b984a0a 100644 --- a/src/agent/backtestRunner.ts +++ b/src/agent/backtestRunner.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { loadConfig } from "../config/loader.js"; import { getDb } from "../storage/db.js"; import { getBacktestBroker } from "../broker/index.js"; -import { getStockOhlcv, getIndexOhlcv, type Bar } from "../data/sources/dnsePublic.js"; +import { getStockOhlcv, getIndexOhlcv, type Bar, findLastBarIndex } from "../data/sources/dnsePublic.js"; import { DISCOVERY_UNIVERSE, discoverTickers } from "../tools/discover.js"; import { setActiveAsOf } from "./clock.js"; import { runTeamAnalysis } from "./team/index.js"; @@ -298,8 +298,8 @@ export async function runBacktestSession( ); const vnindexAt = (asOf: number): number | null => { - const series = vnindex.filter((b) => b.time <= asOf); - return series.length ? series[series.length - 1]!.close : null; + const idx = findLastBarIndex(vnindex, asOf); + return idx !== -1 ? vnindex[idx]!.close : null; }; const vnindexBaseline = vnindexAt(intervalTurns[0]!); if (vnindexBaseline == null) throw new Error(`no VNINDEX data at first ${interval.label} turn`); @@ -312,8 +312,10 @@ export async function runBacktestSession( throwIfAborted(cb.signal); const dateIso = ictLabel(asOf); const priceOverride = (sym: string): number | null => { - const series = bars[sym]?.filter((b) => b.time <= asOf) ?? []; - return series.length ? series[series.length - 1]!.close : null; + const symBars = bars[sym]; + if (!symBars || symBars.length === 0) return null; + const idx = findLastBarIndex(symBars, asOf); + return idx !== -1 ? symBars[idx]!.close : null; }; broker.setPriceOverride(priceOverride); cb.onTurnStart?.({ asOf, dateIso }); diff --git a/src/data/sources/dnsePublic.ts b/src/data/sources/dnsePublic.ts index 151733c..c30ab21 100644 --- a/src/data/sources/dnsePublic.ts +++ b/src/data/sources/dnsePublic.ts @@ -61,6 +61,26 @@ export function seriesToBars(s: OhlcvSeries): Bar[] { return out; } +/** + * Binary search to find the last bar index with time <= targetTime. + * Assumes the bars array is chronologically sorted. + */ +export function findLastBarIndex(bars: Bar[], targetTime: number): number { + let low = 0; + let high = bars.length - 1; + let ans = -1; + while (low <= high) { + const mid = (low + high) >> 1; + if (bars[mid].time <= targetTime) { + ans = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return ans; +} + function clipBars(bars: Bar[]): Bar[] { // Clip when an as-of clock is active (ALS or module override). When neither // is set, fall through unchanged — DNSE only returns historical data anyway. @@ -68,7 +88,9 @@ function clipBars(bars: Bar[]): Bar[] { asOfClock.getStore()?.asOfSec != null || isAsOfOverridden(); if (!hasOverride) return bars; const asOf = nowSec(); - return bars.filter((b) => b.time <= asOf); + const index = findLastBarIndex(bars, asOf); + if (index === -1) return []; + return bars.slice(0, index + 1); } export async function getStockOhlcv(