diff --git a/studio/src/lib/csv.js b/studio/src/lib/csv.js index 55ddfb8..fc49527 100644 --- a/studio/src/lib/csv.js +++ b/studio/src/lib/csv.js @@ -48,11 +48,18 @@ export function normalize(row) { }; } +// Above this size, parse on a background worker so a big paste/upload doesn't +// freeze the UI. Small inputs parse on the main thread — the worker's +// serialization overhead makes it slower for them, and PapaParse falls back to +// synchronous parsing anyway when no Worker global exists (e.g. in tests). +const WORKER_THRESHOLD_BYTES = 512 * 1024; + export function parseCSV(text) { return new Promise((resolve, reject) => { Papa.parse(text, { header: true, skipEmptyLines: true, + worker: text.length > WORKER_THRESHOLD_BYTES, complete: (res) => { const rows = (res.data || []) .map(normalize) diff --git a/studio/src/lib/csv.test.js b/studio/src/lib/csv.test.js index 4c072bd..92a47f1 100644 --- a/studio/src/lib/csv.test.js +++ b/studio/src/lib/csv.test.js @@ -84,4 +84,15 @@ describe("parseCSV", () => { const rows = await parseCSV(csv); expect(rows[0].uid).toBe("999"); }); + + it("parses a large input (past the worker threshold) correctly", async () => { + const lines = ["text,likeCount"]; + for (let i = 0; i < 30000; i++) lines.push(`tweet number ${i},${i}`); + const csv = lines.join("\n"); + expect(csv.length).toBeGreaterThan(512 * 1024); // exercises worker:true path + const rows = await parseCSV(csv); + expect(rows).toHaveLength(30000); + expect(rows[0].text).toBe("tweet number 0"); + expect(rows[29999].likes).toBe(29999); + }); });