From 61c128dbe55e03731a32735dbf0994b10572b759 Mon Sep 17 00:00:00 2001 From: Bryan-Cee Date: Fri, 28 Mar 2025 10:14:46 +0300 Subject: [PATCH 1/2] add batched processing --- packages/webpack-stats-differ/src/diff.ts | 64 +++++++++++-------- .../src/generateDiffsAsync.ts | 58 +++++++++++------ packages/webpack-stats-differ/src/index.ts | 1 + 3 files changed, 77 insertions(+), 46 deletions(-) diff --git a/packages/webpack-stats-differ/src/diff.ts b/packages/webpack-stats-differ/src/diff.ts index d5aa6b1..37af3ce 100644 --- a/packages/webpack-stats-differ/src/diff.ts +++ b/packages/webpack-stats-differ/src/diff.ts @@ -19,6 +19,7 @@ import { GenerateDiffAsyncResult, GetFileDiffOptions, getFileDiffResult, + processPairsInBatches, } from "./generateDiffsAsync"; /** @@ -44,7 +45,7 @@ export async function diff( candidate: string | RemoteArtifact[]; hostUrl: string; }, - bundleStatsOwners?:Map | undefined, + bundleStatsOwners?: Map | undefined, useWorkers?: boolean ): Promise { const [filesA, filesB] = await Promise.all( @@ -94,44 +95,53 @@ const generateDiffs = async ( ): Promise => { let results: GenerateDiffAsyncResult[] = []; const numOfCpus = os.cpus().length; - const numOfWorkers = useWorkers ? Math.ceil(numOfCpus * 0.75) : 1; + // Limit workers to prevent memory issues + const numOfWorkers = useWorkers ? Math.min(8, Math.ceil(numOfCpus * 0.5)) : 1; const start = Date.now(); - console.log( - `Started diffing for ${filePairs.length} bundle pairs using ${numOfWorkers} workers` - ); + if (numOfWorkers <= 1) { - for (const pair of filePairs) { - const result = await getFileDiffResult({ pair, filter }); - results.push(result); - } + console.log(`Processing ${filePairs.length} bundle pairs in batches without workers`); + results = await processPairsInBatches(filePairs, filter); } else { - let pairIndex = 0; - const diffImageWorker = async () => { - const worker = new Worker(`${__dirname}/generateDiffsAsync.js`); + console.log(`Processing ${filePairs.length} bundle pairs in batches using ${numOfWorkers} workers`); + + // Split pairs into batches for each worker + const batchesPerWorker = Math.ceil(filePairs.length / numOfWorkers); + const workerBatches = Array.from({ length: numOfWorkers }, (_, i) => + filePairs.slice(i * batchesPerWorker, (i + 1) * batchesPerWorker) + ).filter(batch => batch.length > 0); - const generateDiffOptions: GetFileDiffOptions = { - pair: null, - filter, - }; + const diffWorker = async (workerPairs: FilePair[]) => { + const worker = new Worker(`${__dirname}/generateDiffsAsync.js`); + const workerResults: GenerateDiffAsyncResult[] = []; - while ((generateDiffOptions.pair = filePairs[pairIndex++])) { - await new Promise((resolve) => { - worker.once("message", (result: GenerateDiffAsyncResult) => { - results.push(result); - resolve(undefined); + for (let i = 0; i < workerPairs.length; i += 10) { + const batch = workerPairs.slice(i, i + 10); + console.log(`Worker processing batch ${Math.floor(i/10) + 1} of ${Math.ceil(workerPairs.length/10)} (${batch.length} pairs)`); + + for (const pair of batch) { + const generateDiffOptions: GetFileDiffOptions = { pair, filter }; + const result = await new Promise((resolve) => { + worker.once("message", (result: GenerateDiffAsyncResult) => { + resolve(result); + }); + worker.postMessage(generateDiffOptions); }); - - worker.postMessage(generateDiffOptions); - }); + workerResults.push(result); + } + + // Small delay between batches to allow for GC + await new Promise(resolve => setTimeout(resolve, 100)); } worker.terminate(); - return; + return workerResults; }; - const buckets = new Array(numOfWorkers).fill(1); - await Promise.all(buckets.map(diffImageWorker)); + const workerResults = await Promise.all(workerBatches.map(diffWorker)); + results = workerResults.flat(); } + console.log(`Diffing done in ${Date.now() - start} ms`); return { withDifferences: results diff --git a/packages/webpack-stats-differ/src/generateDiffsAsync.ts b/packages/webpack-stats-differ/src/generateDiffsAsync.ts index c515dbc..0209f6d 100644 --- a/packages/webpack-stats-differ/src/generateDiffsAsync.ts +++ b/packages/webpack-stats-differ/src/generateDiffsAsync.ts @@ -13,7 +13,7 @@ import { chain } from "stream-chain"; import { parser } from "stream-json"; import { streamObject } from "stream-json/streamers/StreamObject"; -const getWebpackStatJSON = async ( +export const getWebpackStatJSON = async ( filePath: string ): Promise => { try { @@ -22,24 +22,7 @@ const getWebpackStatJSON = async ( throw new Error(`Cannot access webpack stats file at ${filePath}: ${e}`); } - // Get file size - const stats = await fs.promises.stat(filePath); - const fileSizeInMB = stats.size / (1024 * 1024); - - // For small files, use the regular JSON.parse method - if (fileSizeInMB < 100) { - // Adjust threshold as needed - try { - const parsed: WebpackStatsJson = JSON.parse( - await fs.promises.readFile(filePath, { encoding: "utf-8" }) - ); - return parsed; - } catch (e: unknown) { - throw new Error(`Cannot parse webpack state file ${filePath}: ${e}`); - } - } - - // For large files, use streaming approach + // Always use streaming approach for webpack stats files since they can be very large return new Promise((resolve, reject) => { const result: WebpackStatsJson = {} as WebpackStatsJson; @@ -141,6 +124,43 @@ export const getFileDiffResult = async ({ } }; +// Add batch processing utilities +const BATCH_SIZE = 10; // Process 10 pairs at a time +const MAX_WORKERS = 8; // Limit number of workers to prevent memory issues + +export const processBatch = async ( + pairs: FilePair[], + filter?: string | string[] +): Promise => { + const results: GenerateDiffAsyncResult[] = []; + for (const pair of pairs) { + const result = await getFileDiffResult({ pair, filter }); + results.push(result); + } + return results; +}; + +export const processPairsInBatches = async ( + pairs: FilePair[], + filter?: string | string[] +): Promise => { + const results: GenerateDiffAsyncResult[] = []; + + for (let i = 0; i < pairs.length; i += BATCH_SIZE) { + const batch = pairs.slice(i, i + BATCH_SIZE); + console.log(`Processing batch ${Math.floor(i/BATCH_SIZE) + 1} of ${Math.ceil(pairs.length/BATCH_SIZE)} (${batch.length} pairs)`); + const batchResults = await processBatch(batch, filter); + results.push(...batchResults); + + // Force garbage collection between batches if available + if (global.gc) { + global.gc(); + } + } + + return results; +}; + if (!isMainThread) { (async () => { return new Promise(() => { diff --git a/packages/webpack-stats-differ/src/index.ts b/packages/webpack-stats-differ/src/index.ts index f4f08ae..7640cb8 100644 --- a/packages/webpack-stats-differ/src/index.ts +++ b/packages/webpack-stats-differ/src/index.ts @@ -8,3 +8,4 @@ export { FileDiffResultWithComparisonToolUrl, } from "./diff"; export { getFriendlyAssetName } from "./getFriendlyAssetName"; +export { getWebpackStatJSON } from "./generateDiffsAsync"; From 2e0085188702a208143b2ff2a8a204b8845dff68 Mon Sep 17 00:00:00 2001 From: Bryan-Cee Date: Fri, 28 Mar 2025 10:29:38 +0300 Subject: [PATCH 2/2] add streaming to getRemoteArtifactsManifest --- packages/webpack-stats-differ/src/diff.ts | 33 ++++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/webpack-stats-differ/src/diff.ts b/packages/webpack-stats-differ/src/diff.ts index 37af3ce..3dd43ca 100644 --- a/packages/webpack-stats-differ/src/diff.ts +++ b/packages/webpack-stats-differ/src/diff.ts @@ -21,6 +21,10 @@ import { getFileDiffResult, processPairsInBatches, } from "./generateDiffsAsync"; +import { parser } from "stream-json"; +import { streamArray } from "stream-json/streamers/StreamArray"; +import { chain } from "stream-chain"; +import { createReadStream } from "fs"; /** * Calculates the diff between two sets of bundle stats @@ -161,13 +165,28 @@ const generateDiffs = async ( const getRemoteArtifactsManifest = async ( filePath: string ): Promise => { - const data = await fs.promises.readFile(filePath, { encoding: "utf-8" }); - try { - const parsed: RemoteArtifact[] = JSON.parse(data); - return parsed; - } catch (e: unknown) { - throw new Error(`Cannot parse remote artifact manifest ${filePath}: ${e}`); - } + return new Promise((resolve, reject) => { + const artifacts: RemoteArtifact[] = []; + const pipeline = chain([ + createReadStream(filePath, { encoding: "utf8" }), + parser(), + streamArray(), + ]); + + pipeline.on("data", (data) => { + artifacts.push(data.value); + }); + + pipeline.on("end", () => { + resolve(artifacts); + }); + + pipeline.on("error", (err) => { + reject( + new Error(`Cannot parse remote artifact manifest ${filePath}: ${err}`) + ); + }); + }); }; const instanceOfRemoteArtifact = (obj: any): obj is RemoteArtifact[] =>