Skip to content

Preserve issue creationDate during migration via synthetic SCM change… #125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joshua-quek-sonarsource merged 1 commit into from
Apr 23, 2026
Merged

Preserve issue creationDate during migration via synthetic SCM change… #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 53 additions & 10 deletions src/pipelines/sq-10.0/sonarqube/extractors/changesets/helpers/build-stub-changeset.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,57 @@
const STUB_REVISION = 'cloudvoyager000000000000000000000000000';
const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io';

export function buildStubChangeset(lineCount) {
return {
componentRef: null, // Will be set by builder
changesets: [{
revision: STUB_REVISION,
author: STUB_AUTHOR,
date: Date.now()
}],
changesetIndexByLine: new Array(lineCount).fill(0)
};
/**
* Create changeset data for a single file, using issue creation dates
* to build per-line date mappings so that the Compute Engine assigns
* the correct creationDate to each migrated issue.
*
* @param {number} lineCount - Number of lines in the file
* @param {Array} [fileIssues=[]] - Issues located in this file, each with `line` and `creationDate`
*/
export function buildStubChangeset(lineCount, fileIssues = []) {
const fallbackTimestamp = Date.now();

// Build a map of line -> earliest issue creation date (as ms timestamp)
const lineDateMap = new Map();
for (const issue of fileIssues) {
if (!issue.line || !issue.creationDate) continue;
const ts = new Date(issue.creationDate).getTime();
if (Number.isNaN(ts)) continue;
const existing = lineDateMap.get(issue.line);
if (!existing || ts < existing) lineDateMap.set(issue.line, ts);
}

// If no issues have usable dates, fall back to original single-changeset stub
if (lineDateMap.size === 0) {
return {
componentRef: null,
changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }],
changesetIndexByLine: new Array(lineCount).fill(0),
};
}

// Collect unique timestamps and assign each a changeset index.
// Index 0 is the fallback (for lines without issues).
const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b);
const dateToIndex = new Map();
dateToIndex.set(fallbackTimestamp, 0);
const changesets = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }];

for (const date of uniqueDates) {
if (dateToIndex.has(date)) continue;
dateToIndex.set(date, changesets.length);
changesets.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date });
}

// Map each line (1-based) to its changeset index
const changesetIndexByLine = new Array(lineCount).fill(0);
for (const [line, date] of lineDateMap) {
const arrayIdx = line - 1;
if (arrayIdx >= 0 && arrayIdx < lineCount) {
changesetIndexByLine[arrayIdx] = dateToIndex.get(date);
}
}

return { componentRef: null, changesets, changesetIndexByLine };
}
15 changes: 13 additions & 2 deletions src/pipelines/sq-10.0/sonarqube/extractors/changesets/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,29 @@ import logger from '../../../../../shared/utils/logger.js';
import { buildStubChangeset } from './helpers/build-stub-changeset.js';
import { resolveLineCount } from './helpers/resolve-line-count.js';

export async function extractChangesets(client, sourceFiles, components) {
export async function extractChangesets(client, sourceFiles, components, issues = []) {
logger.info('Extracting SCM changeset data...');
const changesets = new Map();

// Index issues by component key for fast lookup
const issuesByComponent = new Map();
for (const issue of issues) {
if (!issue.component) continue;
if (!issuesByComponent.has(issue.component)) {
issuesByComponent.set(issue.component, []);
}
issuesByComponent.get(issue.component).push(issue);
}

for (const file of sourceFiles) {
if (!file?.key) {
logger.warn('Skipping file without key in changesets extraction');
continue;
}
try {
const lineCount = resolveLineCount(file, components);
changesets.set(file.key, buildStubChangeset(lineCount));
const fileIssues = issuesByComponent.get(file.key) || [];
changesets.set(file.key, buildStubChangeset(lineCount, fileIssues));
} catch (error) {
logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`);
}
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-branch-part2.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export function buildBranchPhasesPart2(extractor, branch, metricKeys, ctx, opts)
fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: opts.dupConcurrency }); return ctx.duplications; },
restore: (d) => { ctx.duplications = d; } },
{ name: 'extract:changesets', label: `[${branch}] Extracting changesets`,
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; },
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; },
restore: (d) => { ctx.changesets = d; } },
{ name: 'extract:symbols', label: `[${branch}] Extracting symbols`,
fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; },
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.0/sonarqube/extractors/helpers/checkpoint-phases-main-part2.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export function buildPhasesPart2(extractor, ctx, opts) {
fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, null, { concurrency: opts.dupConcurrency }); return ctx.duplications; },
restore: (d) => { ctx.duplications = d; } },
{ name: 'extract:changesets', label: 'Step 8: Extracting changesets',
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; },
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; },

@cursor cursor Bot Apr 22, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hotspot issues excluded from changeset date mapping in checkpoint paths

Medium Severity

In all checkpoint-based extraction paths, ctx.issues is passed to extractChangesets before hotspot issues have been merged in. Hotspots are extracted into ctx.hotspotIssues and only merged into ctx.issues after all phases complete (e.g., in extract-all-with-checkpoints.js). This means hotspot creation dates are silently excluded from the synthetic changeset date mappings. Non-checkpoint paths correctly merge hotspots before calling extractChangesets, creating an inconsistency. This affects all four pipeline versions (sq-9.9, sq-10.0, sq-10.4, sq-2025) for both main and branch extraction.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit e3bbbd4. Configure here.

restore: (d) => { ctx.changesets = d; } },
{ name: 'extract:symbols', label: 'Step 9: Extracting symbols',
fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; },
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-all.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export async function extractAll(extractor) {
data.sources = await extractSources(extractor.client, null, maxFiles, { concurrency: conc });
const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5;
data.duplications = await extractDuplications(extractor.client, data.components, null, { concurrency: dupConc });
data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components);
data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components, data.issues);
data.symbols = await extractSymbols(extractor.client, sourceFilesList);
data.syntaxHighlightings = await extractSyntaxHighlighting(extractor.client, sourceFilesList);
logger.info(`Data extraction completed in ${((Date.now() - startTime) / 1000).toFixed(2)}s`);
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.0/sonarqube/extractors/helpers/extract-branch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export async function extractBranch(extractor, branch, mainData) {
const sources = await extractSources(extractor.client, branch, maxFiles, { concurrency: srcConc });
const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5;
const duplications = await extractDuplications(extractor.client, components, branch, { concurrency: dupConc });
const changesets = await extractChangesets(extractor.client, sourceFilesList, components);
const changesets = await extractChangesets(extractor.client, sourceFilesList, components, issues);
const symbols = await extractSymbols(extractor.client, sourceFilesList);
const syntaxHighlightings = await extractSyntaxHighlighting(extractor.client, sourceFilesList);
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
Expand Down
65 changes: 57 additions & 8 deletions src/pipelines/sq-10.4/sonarqube/extractors/changesets/helpers/extract-changesets.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,29 @@ const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io';

// -------- Main Logic --------

// Extract SCM changeset (blame) data from SonarQube.
export async function extractChangesets(client, sourceFiles, components) {
// Extract SCM changeset (blame) data — uses issue creation dates to preserve original dates.
export async function extractChangesets(client, sourceFiles, components, issues = []) {
logger.info('Extracting SCM changeset data...');

const changesets = new Map();
const timestamp = Date.now();
const fallbackTimestamp = Date.now();

// Index issues by component key for fast lookup
const issuesByComponent = new Map();
for (const issue of issues) {
if (!issue.component) continue;
if (!issuesByComponent.has(issue.component)) {
issuesByComponent.set(issue.component, []);
}
issuesByComponent.get(issue.component).push(issue);
}

for (const file of sourceFiles) {
if (!file?.key) { logger.warn('Skipping file without key in changesets extraction'); continue; }
try {
const lineCount = resolveLineCount(file, components);
changesets.set(file.key, {
componentRef: null,
changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: timestamp }],
changesetIndexByLine: new Array(lineCount).fill(0)
});
const fileIssues = issuesByComponent.get(file.key) || [];
changesets.set(file.key, buildChangeset(lineCount, fileIssues, fallbackTimestamp));
} catch (error) {
logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`);
}
Expand All @@ -32,6 +39,48 @@ export async function extractChangesets(client, sourceFiles, components) {
return changesets;
}

// -------- Changeset Builder --------

function buildChangeset(lineCount, fileIssues, fallbackTimestamp) {
const lineDateMap = new Map();
for (const issue of fileIssues) {
if (!issue.line || !issue.creationDate) continue;
const ts = new Date(issue.creationDate).getTime();
if (Number.isNaN(ts)) continue;
const existing = lineDateMap.get(issue.line);
if (!existing || ts < existing) lineDateMap.set(issue.line, ts);
}

if (lineDateMap.size === 0) {
return {
componentRef: null,
changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }],
changesetIndexByLine: new Array(lineCount).fill(0),
};
}

const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b);
const dateToIndex = new Map();
dateToIndex.set(fallbackTimestamp, 0);
const changesetsArr = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }];

for (const date of uniqueDates) {
if (dateToIndex.has(date)) continue;
dateToIndex.set(date, changesetsArr.length);
changesetsArr.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date });
}

const changesetIndexByLine = new Array(lineCount).fill(0);
for (const [line, date] of lineDateMap) {
const arrayIdx = line - 1;
if (arrayIdx >= 0 && arrayIdx < lineCount) {
changesetIndexByLine[arrayIdx] = dateToIndex.get(date);
}
}

return { componentRef: null, changesets: changesetsArr, changesetIndexByLine };
}

// -------- Helper Functions --------

function resolveLineCount(file, components) {
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-branch-source-phases.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export function buildBranchSourcePhases(extractor, branch, ctx, opts) {
fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: dupConcurrency }); return ctx.duplications; },
restore: (d) => { ctx.duplications = d; } },
{ name: 'extract:changesets', label: `[${branch}] Extracting changesets`,
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; },
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; },
restore: (d) => { ctx.changesets = d; } },
{ name: 'extract:symbols', label: `[${branch}] Extracting symbols`,
fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; },
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.4/sonarqube/extractors/helpers/build-main-source-phases.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export function buildMainSourcePhases(extractor, ctx, opts) {
fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, null, { concurrency: dupConcurrency }); return ctx.duplications; },
restore: (d) => { ctx.duplications = d; } },
{ name: 'extract:changesets', label: 'Step 8: Extracting changesets',
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; },
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; },
restore: (d) => { ctx.changesets = d; } },
{ name: 'extract:symbols', label: 'Step 9: Extracting symbols',
fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; },
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export async function extractBranchData(extractor, branch, metricKeys) {
logger.info(` [${branch}] Added ${hotspotIssues.length} hotspots (total: ${issues.length})`);
}

const sourceData = await extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList);
const sourceData = await extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList, issues);

return { components, issues, ...sourceData };
}
4 changes: 2 additions & 2 deletions src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-branch-source-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { extractDuplications } from '../duplications.js';
* @param {Array} sourceFilesList - Source files list
* @returns {Promise<object>} Source data fields
*/
export async function extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList) {
export async function extractBranchSourceData(extractor, branch, metricKeys, components, sourceFilesList, issues = []) {
const maxFiles = Math.max(0, Number.parseInt(process.env.MAX_SOURCE_FILES || '0', 10) || 0);
const srcConc = extractor.performanceConfig.sourceExtraction?.concurrency || 10;
const dupConc = extractor.performanceConfig.sourceExtraction?.concurrency || 5;
Expand All @@ -33,7 +33,7 @@ export async function extractBranchSourceData(extractor, branch, metricKeys, com
const duplications = await extractDuplications(extractor.client, components, branch, { concurrency: dupConc });

logger.info(` [${branch}] Extracting changesets...`);
const changesets = await extractChangesets(extractor.client, sourceFilesList, components);
const changesets = await extractChangesets(extractor.client, sourceFilesList, components, issues);

logger.info(` [${branch}] Extracting symbols...`);
const symbols = await extractSymbols(extractor.client, sourceFilesList);
Expand Down
2 changes: 1 addition & 1 deletion src/pipelines/sq-10.4/sonarqube/extractors/helpers/extract-source-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export async function extractSourceData(extractor, data, sourceFilesList) {
data.duplications = await extractDuplications(extractor.client, data.components, null, { concurrency: dupConc });

logger.info('Step 8/10: Extracting changesets...');
data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components);
data.changesets = await extractChangesets(extractor.client, sourceFilesList, data.components, data.issues);

logger.info('Step 9/10: Extracting symbols...');
data.symbols = await extractSymbols(extractor.client, sourceFilesList);
Expand Down
69 changes: 56 additions & 13 deletions src/pipelines/sq-2025/sonarqube/extractors/changesets/helpers/create-stub-changeset.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,60 @@
const STUB_REVISION = 'cloudvoyager000000000000000000000000000';
const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io';

/** Create minimal changeset data for a single file. */
export function createStubChangeset(lineCount) {
const timestamp = Date.now();

return {
componentRef: null, // Will be set by builder
changesets: [{
revision: STUB_REVISION,
author: STUB_AUTHOR,
date: timestamp,
}],
changesetIndexByLine: new Array(lineCount).fill(0),
};
/**
* Create changeset data for a single file, using issue creation dates
* to build per-line date mappings so that the Compute Engine assigns
* the correct creationDate to each migrated issue.
*
* @param {number} lineCount - Number of lines in the file
* @param {Array} [fileIssues=[]] - Issues located in this file, each with `line` and `creationDate`
*/
export function createStubChangeset(lineCount, fileIssues = []) {
const fallbackTimestamp = Date.now();

// Build a map of line -> earliest issue creation date (as ms timestamp)
const lineDateMap = new Map();
for (const issue of fileIssues) {
if (!issue.line || !issue.creationDate) continue;
const ts = new Date(issue.creationDate).getTime();
if (Number.isNaN(ts)) continue;
const existing = lineDateMap.get(issue.line);
if (!existing || ts < existing) {
lineDateMap.set(issue.line, ts);
}
}

// If no issues have usable dates, fall back to original single-changeset stub
if (lineDateMap.size === 0) {
return {
componentRef: null,
changesets: [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }],
changesetIndexByLine: new Array(lineCount).fill(0),
};
}

// Collect unique timestamps and assign each a changeset index.
// Index 0 is the fallback (for lines without issues).
const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b);
const dateToIndex = new Map();
dateToIndex.set(fallbackTimestamp, 0);
const changesets = [{ revision: STUB_REVISION, author: STUB_AUTHOR, date: fallbackTimestamp }];

for (const date of uniqueDates) {
if (dateToIndex.has(date)) continue;
const idx = changesets.length;
dateToIndex.set(date, idx);
changesets.push({ revision: STUB_REVISION, author: STUB_AUTHOR, date });
}

// Map each line (1-based) to its changeset index
const changesetIndexByLine = new Array(lineCount).fill(0);
for (const [line, date] of lineDateMap) {
const arrayIdx = line - 1; // lines are 1-based, array is 0-based
if (arrayIdx >= 0 && arrayIdx < lineCount) {
changesetIndexByLine[arrayIdx] = dateToIndex.get(date);
}
}

return { componentRef: null, changesets, changesetIndexByLine };
}
17 changes: 14 additions & 3 deletions src/pipelines/sq-2025/sonarqube/extractors/changesets/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,30 @@ import { resolveLineCount } from './helpers/resolve-line-count.js';

// -------- Extract Changesets --------

/** Extract SCM changeset (blame) data — creates minimal stub data for each source file. */
export async function extractChangesets(client, sourceFiles, components) {
/** Extract SCM changeset (blame) data — uses issue creation dates to preserve original dates. */
export async function extractChangesets(client, sourceFiles, components, issues = []) {
logger.info('Extracting SCM changeset data...');
const changesets = new Map();

// Index issues by component key for fast lookup
const issuesByComponent = new Map();
for (const issue of issues) {
if (!issue.component) continue;
if (!issuesByComponent.has(issue.component)) {
issuesByComponent.set(issue.component, []);
}
issuesByComponent.get(issue.component).push(issue);
}

for (const file of sourceFiles) {
if (!file?.key) {
logger.warn('Skipping file without key in changesets extraction');
continue;
}
try {
const lineCount = resolveLineCount(file, components);
changesets.set(file.key, createStubChangeset(lineCount));
const fileIssues = issuesByComponent.get(file.key) || [];
changesets.set(file.key, createStubChangeset(lineCount, fileIssues));
} catch (error) {
logger.warn(`Failed to create changeset for ${file.key}: ${error.message}`);
}
Expand Down
2 changes: 1 addition & 1 deletion ...ipelines/sq-2025/sonarqube/extractors/checkpoint-extractor/helpers/branch-phases-part2.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export function buildBranchPhasesPart2(extractor, ctx, branch, maxFiles, sourceC
fn: async () => { ctx.duplications = await extractDuplications(extractor.client, ctx.components, branch, { concurrency: dupConcurrency }); return ctx.duplications; },
restore: (d) => { ctx.duplications = d; } },
{ name: 'extract:changesets', label: `[${branch}] Extracting changesets`,
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components); return ctx.changesets; },
fn: async () => { ctx.changesets = await extractChangesets(extractor.client, ctx.sourceFilesList, ctx.components, ctx.issues); return ctx.changesets; },
restore: (d) => { ctx.changesets = d; } },
{ name: 'extract:symbols', label: `[${branch}] Extracting symbols`,
fn: async () => { ctx.symbols = await extractSymbols(extractor.client, ctx.sourceFilesList); return ctx.symbols; },
Expand Down
Loading