diff --git a/packages/api/src/core/data-trace-stream-parser.ts b/packages/api/src/core/data-trace-stream-parser.ts new file mode 100644 index 000000000..cfd0adf50 --- /dev/null +++ b/packages/api/src/core/data-trace-stream-parser.ts @@ -0,0 +1,51 @@ +export interface TraceStreamProcessor { + (text: string): void + flush: () => void +} + +export function createTraceStreamProcessor( + onTraceValue: (nodeId: string, data: Record) => void, +): TraceStreamProcessor { + let lastIndex = 0 + let pendingLine = '' + + function parseLine(line: string) { + const trimmed = line.trim() + if (!trimmed) return + + const parsed = JSON.parse(trimmed) + if (parsed.nodeId !== undefined && parsed.type === 'TRACE_VALUE') { + onTraceValue(parsed.nodeId, parsed) + } + } + + const processChunk = ((text: string) => { + const newText = text.slice(lastIndex) + lastIndex = text.length + + const lines = (pendingLine + newText).split('\n') + pendingLine = lines.pop() ?? '' + + for (const line of lines) { + try { + parseLine(line) + } catch { + // Ignore malformed non-trace lines from the stream. + } + } + }) as TraceStreamProcessor + + processChunk.flush = () => { + if (!pendingLine.trim()) return + + try { + parseLine(pendingLine) + } catch { + // Ignore an incomplete final line. + } finally { + pendingLine = '' + } + } + + return processChunk +} diff --git a/packages/api/src/core/data-trace.ts b/packages/api/src/core/data-trace.ts index 533ac6f43..5bdc46f60 100644 --- a/packages/api/src/core/data-trace.ts +++ b/packages/api/src/core/data-trace.ts @@ -1,4 +1,5 @@ import { requestClient } from '../request' +import { createTraceStreamProcessor } from './data-trace-stream-parser' const BASE_URL = '/api/lineage/wide-table' @@ -98,27 +99,9 @@ export function getTraceData( callbacks: TraceStreamCallbacks, ): AbortController { const controller = new AbortController() - let lastIndex = 0 - - function processChunk(text: string) { - const newText = text.slice(lastIndex) - lastIndex = text.length - - const lines = newText.split('\n') - for (const line of lines) { - const trimmed = line.trim() - if (!trimmed) continue - try { - const parsed = JSON.parse(trimmed) - if (parsed.nodeId !== undefined && parsed.type === 'TRACE_VALUE') { - console.log('parsed', parsed) - callbacks.onNodeData?.(parsed.nodeId, parsed) - } - } catch { - // ignore incomplete line (will be completed in next chunk) - } - } - } + const processChunk = createTraceStreamProcessor((nodeId, parsed) => { + callbacks.onNodeData?.(nodeId, parsed) + }) requestClient .post(`${BASE_URL}/trace/stream`, data, { @@ -135,6 +118,7 @@ export function getTraceData( }, }) .then(() => { + processChunk.flush() callbacks.onDone?.() }) .catch((error: any) => {