From e3bbbd47e447c1debd5a7ead040849fecbc50b10 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 22 Apr 2026 11:29:17 +0200 Subject: [PATCH] Preserve issue creationDate during migration via synthetic SCM changesets Instead of creating a single stub changeset per file with Date.now(), changeset extraction now uses issue creation dates to build per-line date mappings. The Compute Engine uses SCM blame dates to set issue creationDate, so by encoding the original dates into synthetic changesets, migrated issues retain their original creation timestamps. Applied across all pipeline versions (sq-9.9, sq-10.0, sq-10.4, sq-2025) for both main branch and branch extraction paths. Co-Authored-By: Claude Opus 4.6 --- .../helpers/build-stub-changeset.js | 63 +++++++++++--- .../sonarqube/extractors/changesets/index.js | 15 +++- .../helpers/checkpoint-phases-branch-part2.js | 2 +- .../helpers/checkpoint-phases-main-part2.js | 2 +- .../extractors/helpers/extract-all.js | 2 +- .../extractors/helpers/extract-branch.js | 2 +- .../changesets/helpers/extract-changesets.js | 65 +++++++++++++-- .../helpers/build-branch-source-phases.js | 2 +- .../helpers/build-main-source-phases.js | 2 +- .../extractors/helpers/extract-branch-data.js | 2 +- .../helpers/extract-branch-source-data.js | 4 +- .../extractors/helpers/extract-source-data.js | 2 +- .../helpers/create-stub-changeset.js | 69 +++++++++++++--- .../sonarqube/extractors/changesets/index.js | 17 +++- .../helpers/branch-phases-part2.js | 2 +- .../helpers/main-phases-part2.js | 2 +- .../helpers/extract-scm-and-highlighting.js | 2 +- .../helpers/extract-branch-scm.js | 4 +- .../helpers/extract-branch/index.js | 2 +- .../sq-9.9/sonarqube/extractors/changesets.js | 82 +++++++++++++------ .../helpers/checkpoint-phases-branch.js | 2 +- .../helpers/checkpoint-phases-main.js | 2 +- .../helpers/extract-branch-data.js | 2 +- .../helpers/run-all-extraction-steps.js | 2 +- 24 files changed, 272 insertions(+), 79 deletions(-) diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/changesets/helpers/build-stub-changeset.js b/src/pipelines/sq-10.0/sonarqube/extractors/changesets/helpers/build-stub-changeset.js index 4bdb76c2..d8b42cbc 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/changesets/helpers/build-stub-changeset.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/changesets/helpers/build-stub-changeset.js @@ -3,14 +3,57 @@ const STUB_REVISION = 'cloudvoyager000000000000000000000000000'; const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; -export function buildStubChangeset(lineCount) { - return { - componentRef: null, // Will be set by builder - changesets: [{ - revision: STUB_REVISION, - author: STUB_AUTHOR, - date: Date.now() - }], - changesetIndexByLine: new Array(lineCount).fill(0) - }; +/** + * Create changeset data for a single file, using issue creation dates + * to build per-line date mappings so that the Compute Engine assigns + * the correct creationDate to each migrated issue. + * + * @param {number} lineCount - Number of lines in the file + * @param {Array} [fileIssues=[]] - Issues located in this file, each with `line` and `creationDate` + */ +export function buildStubChangeset(lineCount, fileIssues = []) { + const fallbackTimestamp = Date.now(); + + // Build a map of line -> earliest issue creation date (as ms timestamp) + const lineDateMap = new Map(); + for (const issue of fileIssues) { + if (!issue.line || !issue.creationDate) continue; + const ts = new Date(issue.creationDate).getTime(); + if (Number.isNaN(ts)) continue; + const existing = lineDateMap.get(issue.line); + if (!existing || ts < existing) lineDateMap.set(issue.line, ts); + } + + // If no issues have usable dates, fall back to original single-changeset stub + if (lineDateMap.size === 0) { + return { + componentRef: null, + changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }], + changesetIndexByLine: new Array(lineCount).fill(0), + }; + } + + // Collect unique timestamps and assign each a changeset index. + // Index 0 is the fallback (for lines without issues). + const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); + const dateToIndex = new Map(); + dateToIndex.set(fallbackTimestamp, 0); + const changesets = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }]; + + for (const date of uniqueDates) { + if (dateToIndex.has(date)) continue; + dateToIndex.set(date, changesets.length); + changesets.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date }); + } + + // Map each line (1-based) to its changeset index + const changesetIndexByLine = new Array(lineCount).fill(0); + for (const [line, date] of lineDateMap) { + const arrayIdx = line - 1; + if (arrayIdx >= 0 && arrayIdx < lineCount) { + changesetIndexByLine[arrayIdx] = dateToIndex.get(date); + } + } + + return { componentRef: null, changesets, changesetIndexByLine }; } diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/changesets/index.js b/src/pipelines/sq-10.0/sonarqube/extractors/changesets/index.js index 8933ca17..0e877d5f 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/changesets/index.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/changesets/index.js @@ -4,10 +4,20 @@ import logger from '../../../../../shared/utils/logger.js'; import { buildStubChangeset } from './helpers/build-stub-changeset.js'; import { resolveLineCount } from './helpers/resolve-line-count.js'; -export async function extractChangesets(client, sourceFiles, components) { +export async function extractChangesets(client, sourceFiles, components, issues = []) { logger.info('Extracting SCM changeset data...'); const changesets = new Map(); + // Index issues by component key for fast lookup + const issuesByComponent = new Map(); + for (const issue of issues) { + if (!issue.component) continue; + if (!issuesByComponent.has(issue.component)) { + issuesByComponent.set(issue.component, []); + } + issuesByComponent.get(issue.component).push(issue); + } + for (const file of sourceFiles) { if (!file?.key) { logger.warn('Skipping file without key in changesets extraction'); @@ -15,7 +25,8 @@ export async function extractChangesets(client, sourceFiles, components) { } try { const lineCount = resolveLineCount(file, components); - changesets.set(file.key, buildStubChangeset(lineCount)); + const fileIssues = issuesByComponent.get(file.key) || []; + changesets.set(file.key, buildStubChangeset(lineCount, fileIssues)); } catch (error) { logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`); } diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-branch-part2.js b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-branch-part2.js index 1ae6c13f..56f6e86c 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-branch-part2.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-branch-part2.js @@ -19,7 +19,7 @@ export function buildBranchPhasesPart2(extractor, branch, metricKeys, ctx, opts) fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: opts.dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: `[${branch}] Extracting changesets`, - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: `[${branch}] Extracting symbols`, fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-main-part2.js b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-main-part2.js index 1d2af768..283bb5ba 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-main-part2.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-main-part2.js @@ -19,7 +19,7 @@ export function buildPhasesPart2(extractor, ctx, opts) { fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, null, { concurrency: opts.dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: 'Step 8: Extracting changesets', - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: 'Step 9: Extracting symbols', fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-all.js b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-all.js index c8b94c68..22a33864 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-all.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-all.js @@ -36,7 +36,7 @@ export async function extractAll(extractor) { data.sources = await extractSources(extractor.client, null, maxFiles, { concurrency: conc }); const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5; data.duplications = await extractDuplications(extractor.client, data.components, null, { concurrency: dupConc }); - data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components); + data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components, data.issues); data.symbols = await extractSymbols(extractor.client, sourceFilesList); data.syntaxHighlightings = await extractSyntaxHighlighting(extractor.client, sourceFilesList); logger.info(`Data extraction completed in ${((Date.now() - startTime) / 1000).toFixed(2)}s`); diff --git a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-branch.js b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-branch.js index 542dd3aa..e613a40e 100644 --- a/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-branch.js +++ b/src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-branch.js @@ -30,7 +30,7 @@ export async function extractBranch(extractor, branch, mainData) { const sources = await extractSources(extractor.client, branch, maxFiles, { concurrency: srcConc }); const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5; const duplications = await extractDuplications(extractor.client, components, branch, { concurrency: dupConc }); - const changesets = await extractChangesets(extractor.client, sourceFilesList, components); + const changesets = await extractChangesets(extractor.client, sourceFilesList, components, issues); const symbols = await extractSymbols(extractor.client, sourceFilesList); const syntaxHighlightings = await extractSyntaxHighlighting(extractor.client, sourceFilesList); const duration = ((Date.now() - startTime) / 1000).toFixed(2); diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/changesets/helpers/extract-changesets.js b/src/pipelines/sq-10.4/sonarqube/extractors/changesets/helpers/extract-changesets.js index bc155c8a..d7d1c50e 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/changesets/helpers/extract-changesets.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/changesets/helpers/extract-changesets.js @@ -7,22 +7,29 @@ const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; // -------- Main Logic -------- -// Extract SCM changeset (blame) data from SonarQube. -export async function extractChangesets(client, sourceFiles, components) { +// Extract SCM changeset (blame) data — uses issue creation dates to preserve original dates. +export async function extractChangesets(client, sourceFiles, components, issues = []) { logger.info('Extracting SCM changeset data...'); const changesets = new Map(); - const timestamp = Date.now(); + const fallbackTimestamp = Date.now(); + + // Index issues by component key for fast lookup + const issuesByComponent = new Map(); + for (const issue of issues) { + if (!issue.component) continue; + if (!issuesByComponent.has(issue.component)) { + issuesByComponent.set(issue.component, []); + } + issuesByComponent.get(issue.component).push(issue); + } for (const file of sourceFiles) { if (!file?.key) { logger.warn('Skipping file without key in changesets extraction'); continue; } try { const lineCount = resolveLineCount(file, components); - changesets.set(file.key, { - componentRef: null, - changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: timestamp }], - changesetIndexByLine: new Array(lineCount).fill(0) - }); + const fileIssues = issuesByComponent.get(file.key) || []; + changesets.set(file.key, buildChangeset(lineCount, fileIssues, fallbackTimestamp)); } catch (error) { logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`); } @@ -32,6 +39,48 @@ export async function extractChangesets(client, sourceFiles, components) { return changesets; } +// -------- Changeset Builder -------- + +function buildChangeset(lineCount, fileIssues, fallbackTimestamp) { + const lineDateMap = new Map(); + for (const issue of fileIssues) { + if (!issue.line || !issue.creationDate) continue; + const ts = new Date(issue.creationDate).getTime(); + if (Number.isNaN(ts)) continue; + const existing = lineDateMap.get(issue.line); + if (!existing || ts < existing) lineDateMap.set(issue.line, ts); + } + + if (lineDateMap.size === 0) { + return { + componentRef: null, + changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }], + changesetIndexByLine: new Array(lineCount).fill(0), + }; + } + + const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); + const dateToIndex = new Map(); + dateToIndex.set(fallbackTimestamp, 0); + const changesetsArr = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }]; + + for (const date of uniqueDates) { + if (dateToIndex.has(date)) continue; + dateToIndex.set(date, changesetsArr.length); + changesetsArr.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date }); + } + + const changesetIndexByLine = new Array(lineCount).fill(0); + for (const [line, date] of lineDateMap) { + const arrayIdx = line - 1; + if (arrayIdx >= 0 && arrayIdx < lineCount) { + changesetIndexByLine[arrayIdx] = dateToIndex.get(date); + } + } + + return { componentRef: null, changesets: changesetsArr, changesetIndexByLine }; +} + // -------- Helper Functions -------- function resolveLineCount(file, components) { diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-branch-source-phases.js b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-branch-source-phases.js index 07e8cdc3..d31e1303 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-branch-source-phases.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-branch-source-phases.js @@ -29,7 +29,7 @@ export function buildBranchSourcePhases(extractor, branch, ctx, opts) { fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: `[${branch}] Extracting changesets`, - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: `[${branch}] Extracting symbols`, fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-main-source-phases.js b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-main-source-phases.js index ee565a2a..c35bb5c4 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-main-source-phases.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-main-source-phases.js @@ -28,7 +28,7 @@ export function buildMainSourcePhases(extractor, ctx, opts) { fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, null, { concurrency: dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: 'Step 8: Extracting changesets', - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: 'Step 9: Extracting symbols', fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-data.js b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-data.js index 2b6551f3..0765810a 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-data.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-data.js @@ -31,7 +31,7 @@ export async function extractBranchData(extractor, branch, metricKeys) { logger.info(` [${branch}] Added ${hotspotIssues.length} hotspots (total: ${issues.length})`); } - const sourceData = await extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList); + const sourceData = await extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList, issues); return { components, issues, ...sourceData }; } diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-source-data.js b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-source-data.js index 19695cfd..aacd4412 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-source-data.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-source-data.js @@ -18,7 +18,7 @@ import { extractDuplications } from '../duplications.js'; * @param {Array} sourceFilesList - Source files list * @returns {Promise} Source data fields */ -export async function extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList) { +export async function extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList, issues = []) { const maxFiles = Math.max(0, Number.parseInt(process.env.MAX_SOURCE_FILES || '0', 10) || 0); const srcConc = extractor.performanceConfig.sourceExtraction?.concurrency || 10; const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5; @@ -33,7 +33,7 @@ export async function extractBranchSourceData(extractor, branch, metricKeys, com const duplications = await extractDuplications(extractor.client, components, branch, { concurrency: dupConc }); logger.info(` [${branch}] Extracting changesets...`); - const changesets = await extractChangesets(extractor.client, sourceFilesList, components); + const changesets = await extractChangesets(extractor.client, sourceFilesList, components, issues); logger.info(` [${branch}] Extracting symbols...`); const symbols = await extractSymbols(extractor.client, sourceFilesList); diff --git a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-source-data.js b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-source-data.js index 48e1fedf..d67a1f24 100644 --- a/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-source-data.js +++ b/src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-source-data.js @@ -33,7 +33,7 @@ export async function extractSourceData(extractor, data, sourceFilesList) { data.duplications = await extractDuplications(extractor.client, data.components, null, { concurrency: dupConc }); logger.info('Step 8/10: Extracting changesets...'); - data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components); + data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components, data.issues); logger.info('Step 9/10: Extracting symbols...'); data.symbols = await extractSymbols(extractor.client, sourceFilesList); diff --git a/src/pipelines/sq-2025/sonarqube/extractors/changesets/helpers/create-stub-changeset.js b/src/pipelines/sq-2025/sonarqube/extractors/changesets/helpers/create-stub-changeset.js index f0748e54..8cb62ab1 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/changesets/helpers/create-stub-changeset.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/changesets/helpers/create-stub-changeset.js @@ -3,17 +3,60 @@ const STUB_REVISION = 'cloudvoyager000000000000000000000000000'; const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; -/** Create minimal changeset data for a single file. */ -export function createStubChangeset(lineCount) { - const timestamp = Date.now(); - - return { - componentRef: null, // Will be set by builder - changesets: [{ - revision: STUB_REVISION, - author: STUB_AUTHOR, - date: timestamp, - }], - changesetIndexByLine: new Array(lineCount).fill(0), - }; +/** + * Create changeset data for a single file, using issue creation dates + * to build per-line date mappings so that the Compute Engine assigns + * the correct creationDate to each migrated issue. + * + * @param {number} lineCount - Number of lines in the file + * @param {Array} [fileIssues=[]] - Issues located in this file, each with `line` and `creationDate` + */ +export function createStubChangeset(lineCount, fileIssues = []) { + const fallbackTimestamp = Date.now(); + + // Build a map of line -> earliest issue creation date (as ms timestamp) + const lineDateMap = new Map(); + for (const issue of fileIssues) { + if (!issue.line || !issue.creationDate) continue; + const ts = new Date(issue.creationDate).getTime(); + if (Number.isNaN(ts)) continue; + const existing = lineDateMap.get(issue.line); + if (!existing || ts < existing) { + lineDateMap.set(issue.line, ts); + } + } + + // If no issues have usable dates, fall back to original single-changeset stub + if (lineDateMap.size === 0) { + return { + componentRef: null, + changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }], + changesetIndexByLine: new Array(lineCount).fill(0), + }; + } + + // Collect unique timestamps and assign each a changeset index. + // Index 0 is the fallback (for lines without issues). + const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); + const dateToIndex = new Map(); + dateToIndex.set(fallbackTimestamp, 0); + const changesets = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }]; + + for (const date of uniqueDates) { + if (dateToIndex.has(date)) continue; + const idx = changesets.length; + dateToIndex.set(date, idx); + changesets.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date }); + } + + // Map each line (1-based) to its changeset index + const changesetIndexByLine = new Array(lineCount).fill(0); + for (const [line, date] of lineDateMap) { + const arrayIdx = line - 1; // lines are 1-based, array is 0-based + if (arrayIdx >= 0 && arrayIdx < lineCount) { + changesetIndexByLine[arrayIdx] = dateToIndex.get(date); + } + } + + return { componentRef: null, changesets, changesetIndexByLine }; } diff --git a/src/pipelines/sq-2025/sonarqube/extractors/changesets/index.js b/src/pipelines/sq-2025/sonarqube/extractors/changesets/index.js index ed451d64..92107a6b 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/changesets/index.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/changesets/index.js @@ -4,11 +4,21 @@ import { resolveLineCount } from './helpers/resolve-line-count.js'; // -------- Extract Changesets -------- -/** Extract SCM changeset (blame) data — creates minimal stub data for each source file. */ -export async function extractChangesets(client, sourceFiles, components) { +/** Extract SCM changeset (blame) data — uses issue creation dates to preserve original dates. */ +export async function extractChangesets(client, sourceFiles, components, issues = []) { logger.info('Extracting SCM changeset data...'); const changesets = new Map(); + // Index issues by component key for fast lookup + const issuesByComponent = new Map(); + for (const issue of issues) { + if (!issue.component) continue; + if (!issuesByComponent.has(issue.component)) { + issuesByComponent.set(issue.component, []); + } + issuesByComponent.get(issue.component).push(issue); + } + for (const file of sourceFiles) { if (!file?.key) { logger.warn('Skipping file without key in changesets extraction'); @@ -16,7 +26,8 @@ export async function extractChangesets(client, sourceFiles, components) { } try { const lineCount = resolveLineCount(file, components); - changesets.set(file.key, createStubChangeset(lineCount)); + const fileIssues = issuesByComponent.get(file.key) || []; + changesets.set(file.key, createStubChangeset(lineCount, fileIssues)); } catch (error) { logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`); } diff --git a/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/branch-phases-part2.js b/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/branch-phases-part2.js index da973f1d..96b5b36f 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/branch-phases-part2.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/branch-phases-part2.js @@ -20,7 +20,7 @@ export function buildBranchPhasesPart2(extractor, ctx, branch, maxFiles, sourceC fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: `[${branch}] Extracting changesets`, - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: `[${branch}] Extracting symbols`, fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/main-phases-part2.js b/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/main-phases-part2.js index f1ce0b25..4631dc4f 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/main-phases-part2.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/main-phases-part2.js @@ -20,7 +20,7 @@ export function buildMainPhasesPart2(extractor, ctx, maxFiles, sourceConcurrency fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, null, { concurrency: dupConcurrency }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: 'Step 8: Extracting changesets', - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: 'Step 9: Extracting symbols', fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-all/helpers/extract-scm-and-highlighting.js b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-all/helpers/extract-scm-and-highlighting.js index 041538c7..23206569 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-all/helpers/extract-scm-and-highlighting.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-all/helpers/extract-scm-and-highlighting.js @@ -8,7 +8,7 @@ import logger from '../../../../../../../shared/utils/logger.js'; /** Steps 8-10: Extract changesets, symbols, and syntax highlighting. */ export async function extractScmAndHighlighting(ext, data) { logger.info('Step 8/10: Extracting changesets...'); - data.changesets = await extractChangesets(ext.client, data._sourceFilesList, data.components); + data.changesets = await extractChangesets(ext.client, data._sourceFilesList, data.components, data.issues); logger.info('Step 9/10: Extracting symbols...'); data.symbols = await extractSymbols(ext.client, data._sourceFilesList); diff --git a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/helpers/extract-branch-scm.js b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/helpers/extract-branch-scm.js index e11dfc47..99b0a5bd 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/helpers/extract-branch-scm.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/helpers/extract-branch-scm.js @@ -6,9 +6,9 @@ import logger from '../../../../../../../shared/utils/logger.js'; // -------- Extract Branch SCM -------- /** Extract changesets, symbols, and syntax highlighting for a branch. */ -export async function extractBranchScm(ext, branch, sourceFilesList, components) { +export async function extractBranchScm(ext, branch, sourceFilesList, components, issues = []) { logger.info(` [${branch}] Extracting changesets...`); - const changesets = await extractChangesets(ext.client, sourceFilesList, components); + const changesets = await extractChangesets(ext.client, sourceFilesList, components, issues); logger.info(` [${branch}] Extracting symbols...`); const symbols = await extractSymbols(ext.client, sourceFilesList); diff --git a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/index.js b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/index.js index 8ab7cd67..1ea0b1ca 100644 --- a/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/index.js +++ b/src/pipelines/sq-2025/sonarqube/extractors/helpers/extract-branch/index.js @@ -23,7 +23,7 @@ export async function extractBranch(extractor, branch, mainData) { const measures = await extractMeasures(extractor.client, metricKeys, branch); const sources = await extractBranchSources(extractor, branch); const duplications = await extractBranchDuplications(extractor, branch, components); - const scmData = await extractBranchScm(extractor, branch, sourceFilesList, components); + const scmData = await extractBranchScm(extractor, branch, sourceFilesList, components, issues); const duration = ((Date.now() - startTime) / 1000).toFixed(2); logger.info(` [${branch}] Branch extraction completed in ${duration}s — ${issues.length} issues, ${components.length} components, ${sources.length} sources`); diff --git a/src/pipelines/sq-9.9/sonarqube/extractors/changesets.js b/src/pipelines/sq-9.9/sonarqube/extractors/changesets.js index 80724266..778dc01c 100644 --- a/src/pipelines/sq-9.9/sonarqube/extractors/changesets.js +++ b/src/pipelines/sq-9.9/sonarqube/extractors/changesets.js @@ -1,42 +1,38 @@ import logger from '../../../../shared/utils/logger.js'; +const STUB_REVISION = 'cloudvoyager000000000000000000000000000'; +const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; + /** - * Extract SCM changeset (blame) data from SonarQube - * Creates minimal stub data for each source file + * Extract SCM changeset (blame) data — uses issue creation dates to preserve original dates. * @param {SonarQubeClient} client - SonarQube client * @param {Array} sourceFiles - List of source files with component keys * @param {Object} components - Component tree with metadata + * @param {Array} [issues=[]] - Extracted issues with component, line, and creationDate * @returns {Promise>} Map of component key to changeset data */ -export async function extractChangesets(client, sourceFiles, components) { +export async function extractChangesets(client, sourceFiles, components, issues = []) { logger.info('Extracting SCM changeset data...'); const changesets = new Map(); - const timestamp = Date.now(); // Current timestamp in milliseconds - const stubRevision = 'cloudvoyager000000000000000000000000000'; // 40-char stub hash - const stubAuthor = 'cloudvoyager-migration@sonarcloud.io'; + const fallbackTimestamp = Date.now(); + + // Index issues by component key for fast lookup + const issuesByComponent = new Map(); + for (const issue of issues) { + if (!issue.component) continue; + if (!issuesByComponent.has(issue.component)) { + issuesByComponent.set(issue.component, []); + } + issuesByComponent.get(issue.component).push(issue); + } - // Create minimal changeset data for each source file for (const file of sourceFiles) { if (!file || !file.key) { logger.warn('Skipping file without key in changesets extraction'); continue; } try { - // Use actual source file line count const lineCount = file.lines ? file.lines.length : 1; - - // Create minimal changeset: single changeset for all lines - // changesetIndexByLine maps each line to its changeset index (0-based) - // With a single changeset, all lines point to index 0 - const changesetData = { - componentRef: null, // Will be set by builder - changesets: [{ - revision: stubRevision, - author: stubAuthor, - date: timestamp - }], - changesetIndexByLine: new Array(lineCount).fill(0) // All lines -> changeset[0] - }; - - changesets.set(file.key, changesetData); + const fileIssues = issuesByComponent.get(file.key) || []; + changesets.set(file.key, buildChangeset(lineCount, fileIssues, fallbackTimestamp)); } catch (error) { logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`); } @@ -45,3 +41,43 @@ export async function extractChangesets(client, sourceFiles, components) { logger.info(`Created ${changesets.size} changeset entries`); return changesets; } + +function buildChangeset(lineCount, fileIssues, fallbackTimestamp) { + const lineDateMap = new Map(); + for (const issue of fileIssues) { + if (!issue.line || !issue.creationDate) continue; + const ts = new Date(issue.creationDate).getTime(); + if (Number.isNaN(ts)) continue; + const existing = lineDateMap.get(issue.line); + if (!existing || ts < existing) lineDateMap.set(issue.line, ts); + } + + if (lineDateMap.size === 0) { + return { + componentRef: null, + changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }], + changesetIndexByLine: new Array(lineCount).fill(0), + }; + } + + const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); + const dateToIndex = new Map(); + dateToIndex.set(fallbackTimestamp, 0); + const changesetsArr = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }]; + + for (const date of uniqueDates) { + if (dateToIndex.has(date)) continue; + dateToIndex.set(date, changesetsArr.length); + changesetsArr.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date }); + } + + const changesetIndexByLine = new Array(lineCount).fill(0); + for (const [line, date] of lineDateMap) { + const arrayIdx = line - 1; + if (arrayIdx >= 0 && arrayIdx < lineCount) { + changesetIndexByLine[arrayIdx] = dateToIndex.get(date); + } + } + + return { componentRef: null, changesets: changesetsArr, changesetIndexByLine }; +} diff --git a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-branch.js b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-branch.js index cc541f2e..bb758501 100644 --- a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-branch.js +++ b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-branch.js @@ -34,7 +34,7 @@ export function buildBranchPhases(extractor, branch, mainData, ctx) { fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: dupConc }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: `[${branch}] Extracting changesets`, - fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: `[${branch}] Extracting symbols`, fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-main.js b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-main.js index 2b35498a..87f957ac 100644 --- a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-main.js +++ b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/checkpoint-phases-main.js @@ -38,7 +38,7 @@ export function buildMainBranchPhases(extractor, ctx) { fn: async () => { ctx.duplications = await api.extractDuplications(extractor.client, ctx.components, null, { concurrency: dupConc }); return ctx.duplications; }, restore: (d) => { ctx.duplications = d; } }, { name: 'extract:changesets', label: 'Step 8: Extracting changesets', - fn: async () => { ctx.changesets = await api.extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; }, + fn: async () => { ctx.changesets = await api.extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; }, restore: (d) => { ctx.changesets = d; } }, { name: 'extract:symbols', label: 'Step 9: Extracting symbols', fn: async () => { ctx.symbols = await api.extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; }, diff --git a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/extract-branch-data.js b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/extract-branch-data.js index d101314b..20cb10de 100644 --- a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/extract-branch-data.js +++ b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/extract-branch-data.js @@ -30,7 +30,7 @@ export async function extractBranchData(extractor, branch, metricKeys, maxFiles) const duplications = await extractDuplications(client, components, branch, { concurrency: performanceConfig.sourceExtraction?.concurrency || 5, }); - const changesets = await extractChangesets(client, sourceFilesList, components); + const changesets = await extractChangesets(client, sourceFilesList, components, issues); const symbols = await extractSymbols(client, sourceFilesList); const syntaxHighlightings = await extractSyntaxHighlighting(client, sourceFilesList); diff --git a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/run-all-extraction-steps.js b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/run-all-extraction-steps.js index 5b5666b2..a29b3d24 100644 --- a/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/run-all-extraction-steps.js +++ b/src/pipelines/sq-9.9/sonarqube/extractors/data-extractor/helpers/run-all-extraction-steps.js @@ -42,7 +42,7 @@ export async function runAllExtractionSteps(extractor, data) { data.duplications = await extractDuplications(client, data.components, null, { concurrency: performanceConfig.sourceExtraction?.concurrency || 5, }); - data.changesets = await extractChangesets(client, sourceFilesList, data.components); + data.changesets = await extractChangesets(client, sourceFilesList, data.components, data.issues); data.symbols = await extractSymbols(client, sourceFilesList); data.syntaxHighlightings = await extractSyntaxHighlighting(client, sourceFilesList); }