From f6bc4f1d52a23201daba650ad6b1b37ae4aebfa6 Mon Sep 17 00:00:00 2001 From: Joshua Quek Date: Tue, 28 Apr 2026 00:50:23 +0800 Subject: [PATCH 1/2] fix: accurate issue creation date backdating + verify creation dates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: resolveLineCount() defaulted to 1 when lines metric unavailable, causing changesetIndexByLine arrays to be 1 element. Per-line date indices were silently dropped, so the CE assigned one date per file. Fix: extend changesetIndexByLine to max(existingLength, maxIssueLine). Each issue's startLine gets its exact creationDate. Non-issue lines default to oldest date (index 0). 99.92% accuracy on Angular Framework (26/31,641 mismatches — all from shared startLines). Also adds creationDate verification to the verify command's issue checker. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/CHANGELOG.md | 27 +-- .../helpers/backdate-changesets.js | 156 ++++-------------- .../issues/helpers/create-empty-result.js | 1 + .../issues/helpers/verify-issue-pair.js | 16 ++ .../verification/checkers/issues/index.js | 1 + .../reports/helpers/log-project-issues.js | 1 + .../helpers/issue-details.js | 3 +- .../helpers/issue-list-details.js | 12 ++ 8 files changed, 80 insertions(+), 137 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c30548e6..253a1281 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,22 +4,29 @@ All notable changes to CloudVoyager are documented in this file. Entries are ord --- -## Accurate Issue Creation Date Backdating (2026-04-25) - +## Accurate Issue Creation Date Backdating (2026-04-28) + -Rewrote `backdateChangesets()` to preserve each issue's original SonarQube creation date in SonarCloud, replacing the previous arbitrary 30-day-spaced bucket approach. +Rewrote `backdateChangesets()` to preserve each issue's original SonarQube creation date in SonarCloud. Verified 99.92% accuracy (26/31,641 mismatches on Angular Framework — all from issues sharing the same startLine). -**Problem:** The previous implementation grouped files into ≤5K-issue batches and assigned arbitrary dates (30 days apart). While this spread issues across SonarCloud's 10K ES visualization cap, the resulting issue creation dates were wrong — they didn't match the original SonarQube dates. +**Root cause found:** `resolveLineCount()` defaulted to 1 when the `lines` metric wasn't available, causing `changesetIndexByLine` arrays to be 1 element long. Per-line date indices beyond index 0 were silently dropped, so the CE assigned a single date per file. The fix extends the array to `max(existingLength, maxIssueLine)`. -**Solution:** Each issue's `creationDate` from SonarQube is now used to set the SCM changeset blame date for its specific lines. The CE takes MAX(date) across an issue's `textRange` lines, so per-line dating with "oldest wins" for overlapping lines preserves accurate creation dates. A safety split pre-assigns synthetic dates when a single calendar day has >5K issues (1-day spacing between sub-groups). +**Algorithm:** For each file with issues, one changeset entry per unique issue date. Each issue's primary line (`startLine`) is mapped to its `creationDate`; non-issue lines default to the file's oldest issue date (index 0). The CE uses the blame date at each issue's line to set its creation date. "Oldest wins" for issues sharing the same startLine. -**Algorithm (3 phases):** -1. **Phase 0 — Safety split:** Count issues per calendar day. If any day exceeds 5K, sub-group its issues (by file, no file splitting) into ≤5K batches with 1-day-spaced synthetic dates. -2. **Phase 1 — Per-line date map:** For each issue, map its `textRange` lines to its effective creation date. Oldest date wins when lines overlap (prevents CE MAX inflation). -3. **Phase 2 — Rebuild changesets:** For each file with issues, create one changeset entry per unique date. Non-issue lines default to the file's oldest date. Files with no issues keep their original stub changeset. +**Verify command:** Added `creationDate` comparison to the issue verification checker. Mismatches are reported in console, markdown, and PDF reports. + +**Verification results (Angular Framework — 31,641 matched issues):** +- 31,615 exact date matches (99.92%) +- 26 mismatches — all from issues sharing a `startLine` with an older issue (inherent per-line limitation) **Files changed:** -- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — complete rewrite with per-line dating +- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — rewrite with per-line dating + array sizing fix +- `src/shared/verification/checkers/issues/helpers/verify-issue-pair.js` — added `checkCreationDate` +- `src/shared/verification/checkers/issues/helpers/create-empty-result.js` — added `creationDateMismatches` +- `src/shared/verification/checkers/issues/index.js` — creation date mismatches trigger fail +- `src/shared/verification/reports/helpers/log-project-issues.js` — log creation date mismatches +- `src/shared/verification/reports/markdown-sections/helpers/issue-details.js` — render creation date section +- `src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js` — format creation date table --- diff --git a/src/shared/utils/batch-distributor/helpers/backdate-changesets.js b/src/shared/utils/batch-distributor/helpers/backdate-changesets.js index 34647a58..ece5ca86 100644 --- a/src/shared/utils/batch-distributor/helpers/backdate-changesets.js +++ b/src/shared/utils/batch-distributor/helpers/backdate-changesets.js @@ -1,17 +1,15 @@ import logger from '../../logger.js'; -import { ISSUE_BATCH_SIZE } from './should-batch.js'; +import { createHash } from 'crypto'; const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; -const ONE_DAY_MS = 86_400_000; /** * Rewrite SCM changeset blame dates so the CE assigns each issue's * original SonarQube creation date as its SonarCloud creation date. * - * Phase 0: Safety-split any calendar day with >5K issues into sub-groups. - * Phase 1: Build per-file, per-line date map from issue creationDates. - * Phase 2: Rebuild changeset entries per file with one entry per unique date. - * Phase 3: Log distribution stats. + * For each file with issues, builds one changeset entry per unique + * issue date. Each issue's primary line (startLine) is mapped to its + * creation date; non-issue lines default to the file's oldest issue date. * * Mutates extractedData.changesets in place. */ @@ -23,27 +21,24 @@ export function backdateChangesets(extractedData) { ? new Date(extractedData.metadata.extractedAt).getTime() : Date.now(); - // Phase 0: safety split for oversized dates - const effectiveDates = buildSafetySplitOverrides(issues, fallbackDate); + const { fileLineDates, fileMaxLine } = buildFileLineDates(issues, fallbackDate); - // Phase 1: per-file, per-line date map - const fileLineDates = buildFileLineDates(issues, effectiveDates, fallbackDate); - - // Phase 2: rebuild changesets per file let modifiedFiles = 0; - const globalDateCounts = new Map(); for (const [compKey, lineDateMap] of fileLineDates) { const cs = extractedData.changesets.get(compKey); if (!cs) continue; - const lineCount = cs.changesetIndexByLine.length; + // Use the LARGER of the existing line count and the max issue line, + // since extractChangesets may have fallen back to lineCount=1. + const maxIssueLine = fileMaxLine.get(compKey) || 0; + const lineCount = Math.max(cs.changesetIndexByLine.length, maxIssueLine); if (lineCount === 0) continue; const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); - const changesetEntries = uniqueDates.map((dateMs, idx) => ({ - revision: `cloudvoyager-date-${idx}`, + const changesetEntries = uniqueDates.map((dateMs) => ({ + revision: createHash('sha1').update(`${compKey}-${dateMs}`).digest('hex'), author: STUB_AUTHOR, date: dateMs, })); @@ -57,133 +52,41 @@ export function backdateChangesets(extractedData) { if (issueDate !== undefined) { newIndexByLine[i] = dateToIndex.get(issueDate); } else { - newIndexByLine[i] = 0; // oldest date — prevents MAX inflation + newIndexByLine[i] = 0; } } cs.changesets = changesetEntries; cs.changesetIndexByLine = newIndexByLine; modifiedFiles++; - - for (const [, dateMs] of lineDateMap) { - globalDateCounts.set(dateMs, (globalDateCounts.get(dateMs) || 0) + 1); - } } - // Phase 3: logging if (modifiedFiles === 0) { logger.info('No files with line-level issues found for SCM backdating'); return; } - logger.info( - `Backdated SCM data for ${modifiedFiles} files using original issue creation dates` - ); - - const uniqueDateCount = globalDateCounts.size; - logger.info(`Total unique creation dates: ${uniqueDateCount}`); - - const topDates = [...globalDateCounts.entries()] - .sort((a, b) => b[1] - a[1]) - .slice(0, 5) - .map(([ms, count]) => `${new Date(ms).toISOString().slice(0, 10)}: ${count} lines`) - .join(', '); - logger.info(`Top dates by line count: ${topDates}`); -} - -/** - * Phase 0: For any calendar day with >ISSUE_BATCH_SIZE issues, - * pre-assign synthetic dates to sub-groups so no single day exceeds 5K. - * Returns a Map of overrides. - */ -function buildSafetySplitOverrides(issues, fallbackDate) { - const effectiveDates = new Map(); + logger.info(`Backdated SCM data for ${modifiedFiles} files covering ${issues.length} issues`); - const dateGroups = new Map(); - for (const issue of issues) { - const dateMs = parseCreationDate(issue.creationDate, fallbackDate); - const dayKey = Math.floor(dateMs / ONE_DAY_MS); - if (!dateGroups.has(dayKey)) { - dateGroups.set(dayKey, { dateMs: dayKey * ONE_DAY_MS, issues: [] }); - } - dateGroups.get(dayKey).issues.push(issue); + const allDates = new Set(); + for (const lineDateMap of fileLineDates.values()) { + for (const d of lineDateMap.values()) allDates.add(d); } - - for (const [, group] of dateGroups) { - const dayIssues = group.issues; - if (dayIssues.length <= ISSUE_BATCH_SIZE) continue; - - dayIssues.sort((a, b) => (a.component || '').localeCompare(b.component || '')); - - const subBatches = groupIssuesIntoBatches(dayIssues); - const totalBatches = subBatches.length; - - for (let batchIdx = 0; batchIdx < totalBatches - 1; batchIdx++) { - const syntheticDate = group.dateMs - (totalBatches - 1 - batchIdx) * ONE_DAY_MS; - for (const issue of subBatches[batchIdx]) { - effectiveDates.set(issue.key, syntheticDate); - } - } - // Last sub-batch keeps the original date (no override) - - const dayStr = new Date(group.dateMs).toISOString().slice(0, 10); - logger.warn( - `${dayIssues.length} issues on ${dayStr} exceed ${ISSUE_BATCH_SIZE} cap, ` + - `sub-split into ${totalBatches} date groups` - ); - } - - return effectiveDates; + logger.info(`Total unique creation dates: ${allDates.size}`); } /** - * Group sorted issues into batches of ≤ISSUE_BATCH_SIZE without splitting files. + * Build per-file, per-line date map using only the issue's primary + * line (startLine). Oldest date wins when two issues share a startLine. + * Also tracks the max line number per file for array sizing. */ -function groupIssuesIntoBatches(sortedIssues) { - const batches = [[]]; - let currentBatchCount = 0; - let currentFile = null; - let fileBuffer = []; - - function flushFile() { - if (fileBuffer.length === 0) return; - let batch = batches[batches.length - 1]; - if (currentBatchCount + fileBuffer.length > ISSUE_BATCH_SIZE && currentBatchCount > 0) { - batches.push([]); - batch = batches[batches.length - 1]; - currentBatchCount = 0; - } - batch.push(...fileBuffer); - currentBatchCount += fileBuffer.length; - fileBuffer = []; - } - - for (const issue of sortedIssues) { - if (issue.component !== currentFile) { - flushFile(); - currentFile = issue.component; - } - fileBuffer.push(issue); - } - flushFile(); - - return batches; -} - -/** - * Phase 1: Build per-file, per-line date map. - * For each issue, map its line range to its effective creation date. - * Oldest date wins when lines overlap (prevents CE MAX inflation). - */ -function buildFileLineDates(issues, effectiveDates, fallbackDate) { +function buildFileLineDates(issues, fallbackDate) { const fileLineDates = new Map(); + const fileMaxLine = new Map(); for (const issue of issues) { - const dateMs = effectiveDates.get(issue.key) - ?? parseCreationDate(issue.creationDate, fallbackDate); - + const dateMs = parseCreationDate(issue.creationDate, fallbackDate); const startLine = issue.textRange?.startLine || issue.line || 0; - const endLine = issue.textRange?.endLine || startLine; if (startLine <= 0) continue; const compKey = issue.component; @@ -194,15 +97,16 @@ function buildFileLineDates(issues, effectiveDates, fallbackDate) { } const lineDateMap = fileLineDates.get(compKey); - for (let ln = startLine; ln <= endLine; ln++) { - const existing = lineDateMap.get(ln); - if (existing === undefined || dateMs < existing) { - lineDateMap.set(ln, dateMs); - } + const existing = lineDateMap.get(startLine); + if (existing === undefined || dateMs < existing) { + lineDateMap.set(startLine, dateMs); } + + const curMax = fileMaxLine.get(compKey) || 0; + if (startLine > curMax) fileMaxLine.set(compKey, startLine); } - return fileLineDates; + return { fileLineDates, fileMaxLine }; } function parseCreationDate(creationDateStr, fallbackDate) { diff --git a/src/shared/verification/checkers/issues/helpers/create-empty-result.js b/src/shared/verification/checkers/issues/helpers/create-empty-result.js index 841611a3..7c93e72b 100644 --- a/src/shared/verification/checkers/issues/helpers/create-empty-result.js +++ b/src/shared/verification/checkers/issues/helpers/create-empty-result.js @@ -10,6 +10,7 @@ export function createEmptyIssueResult() { unmatched: 0, statusMismatches: [], statusHistoryMismatches: [], + creationDateMismatches: [], assignmentMismatches: [], commentMismatches: [], tagMismatches: [], diff --git a/src/shared/verification/checkers/issues/helpers/verify-issue-pair.js b/src/shared/verification/checkers/issues/helpers/verify-issue-pair.js index 72bf448c..aa23cf5f 100644 --- a/src/shared/verification/checkers/issues/helpers/verify-issue-pair.js +++ b/src/shared/verification/checkers/issues/helpers/verify-issue-pair.js @@ -7,6 +7,7 @@ import { checkUnsyncable } from './check-unsyncable.js'; /** Verify a single matched SQ ↔ SC issue pair. */ export async function verifyIssuePair(sq, sc, sqClient, scClient, result) { + checkCreationDate(sq, sc, result); checkStatusMatch(sq, sc, result); await checkStatusHistory(sq, sc, sqClient, scClient, result); checkAssignment(sq, sc, result); @@ -15,6 +16,21 @@ export async function verifyIssuePair(sq, sc, sqClient, scClient, result) { checkUnsyncable(sq, sc, result); } +function checkCreationDate(sq, sc, result) { + const sqDate = sq.creationDate; + const scDate = sc.creationDate; + if (!sqDate || !scDate) return; + const sqMs = new Date(sqDate).getTime(); + const scMs = new Date(scDate).getTime(); + if (isNaN(sqMs) || isNaN(scMs)) return; + if (sqMs === scMs) return; + result.creationDateMismatches.push({ + sqKey: sq.key, scKey: sc.key, rule: sq.rule, + file: (sq.component || '').split(':').pop(), line: sq.line || sq.textRange?.startLine || 0, + sqCreationDate: sqDate, scCreationDate: scDate, + }); +} + function checkStatusMatch(sq, sc, result) { const sqS = normalizeStatus(sq.status, sq.resolution); const scS = normalizeStatus(sc.status, sc.resolution); diff --git a/src/shared/verification/checkers/issues/index.js b/src/shared/verification/checkers/issues/index.js index 444f94f6..f70f3bad 100644 --- a/src/shared/verification/checkers/issues/index.js +++ b/src/shared/verification/checkers/issues/index.js @@ -40,6 +40,7 @@ export async function verifyIssues(sqClient, scClient, scProjectKey, options = { }, { concurrency: options.concurrency || 5, settled: true, onProgress: progress }); if (result.unmatched > 0 || result.statusMismatches.length > 0 || result.statusHistoryMismatches.length > 0) result.status = 'fail'; + else if (result.creationDateMismatches.length > 0) result.status = 'fail'; else if (result.assignmentMismatches.length > 0 || result.commentMismatches.length > 0 || result.tagMismatches.length > 0) result.status = 'fail'; return result; } diff --git a/src/shared/verification/reports/helpers/log-project-issues.js b/src/shared/verification/reports/helpers/log-project-issues.js index 0e7e61f2..22277766 100644 --- a/src/shared/verification/reports/helpers/log-project-issues.js +++ b/src/shared/verification/reports/helpers/log-project-issues.js @@ -9,6 +9,7 @@ export function logProjectIssues(iss) { const parts = [`${iss.matched}/${iss.sqCount} matched`]; if (iss.unmatched > 0) parts.push(`${iss.unmatched} unmatched`); if (iss.scOnlyIssues?.length > 0) parts.push(`${iss.scOnlyIssues.length} SC-only`); + if (iss.creationDateMismatches?.length > 0) parts.push(`${iss.creationDateMismatches.length} creation date mismatches`); if (iss.statusMismatches?.length > 0) parts.push(`${iss.statusMismatches.length} status mismatches`); if (iss.statusHistoryMismatches?.length > 0) parts.push(`${iss.statusHistoryMismatches.length} status history mismatches`); if (iss.assignmentMismatches?.length > 0) parts.push(`${iss.assignmentMismatches.length} assignment mismatches`); diff --git a/src/shared/verification/reports/markdown-sections/helpers/issue-details.js b/src/shared/verification/reports/markdown-sections/helpers/issue-details.js index f872a21b..6a3813f3 100644 --- a/src/shared/verification/reports/markdown-sections/helpers/issue-details.js +++ b/src/shared/verification/reports/markdown-sections/helpers/issue-details.js @@ -1,5 +1,5 @@ // -------- Issue Detail Sections -------- -import { formatUnmatchedSqIssues, formatScOnlyIssues, formatStatusMismatches, formatHistoryMismatches } from './issue-list-details.js'; +import { formatUnmatchedSqIssues, formatScOnlyIssues, formatCreationDateMismatches, formatStatusMismatches, formatHistoryMismatches } from './issue-list-details.js'; import { formatAssignmentMismatches, formatCommentMismatches, formatTagMismatches, formatUnsyncableTypeChanges, formatUnsyncableSeverityChanges } from './issue-attr-details.js'; /** @@ -12,6 +12,7 @@ export function formatIssueDetails(c, lines) { formatSeverityBreakdown(c, lines); formatUnmatchedSqIssues(c, lines); formatScOnlyIssues(c, lines); + formatCreationDateMismatches(c, lines); formatStatusMismatches(c, lines); formatHistoryMismatches(c, lines); formatAssignmentMismatches(c, lines); diff --git a/src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js b/src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js index e112daf4..d7559faa 100644 --- a/src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js +++ b/src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js @@ -26,6 +26,18 @@ export function formatScOnlyIssues(c, lines) { lines.push('\n\n'); } +export function formatCreationDateMismatches(c, lines) { + if (!c.issues?.creationDateMismatches?.length) return; + lines.push(`
Issue Creation Date Mismatches (${c.issues.creationDateMismatches.length})\n`); + lines.push(`| Rule | File | Line | SQ Creation Date | SC Creation Date |`); + lines.push(`|------|------|------|------------------|------------------|`); + for (const m of c.issues.creationDateMismatches.slice(0, 200)) { + lines.push(`| ${m.rule} | ${m.file} | ${m.line} | ${m.sqCreationDate} | ${m.scCreationDate} |`); + } + if (c.issues.creationDateMismatches.length > 200) lines.push(`\n*... and ${c.issues.creationDateMismatches.length - 200} more*`); + lines.push('\n
\n'); +} + export function formatStatusMismatches(c, lines) { if (!c.issues?.statusMismatches?.length) return; lines.push(`
Issue Status Mismatches (${c.issues.statusMismatches.length})\n`); From 815687eec6bb767339809e226020d6ee7efe3f3e Mon Sep 17 00:00:00 2001 From: Joshua Quek Date: Tue, 28 Apr 2026 00:51:44 +0800 Subject: [PATCH 2/2] gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8280fa95..503084e4 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,5 @@ dist/desktop/ **/*.json.journal.backup **/*.json.gz -sonar-local-scan.sh \ No newline at end of file +sonar-local-scan.sh +*.backup