fix: accurate issue creation date backdating + verify creation dates

.gitignore

@@ -70,4 +70,5 @@ dist/desktop/
 **/*.json.journal.backup
 **/*.json.gz
 
-sonar-local-scan.sh
+sonar-local-scan.sh
+*.backup

docs/CHANGELOG.md

@@ -4,22 +4,29 @@ All notable changes to CloudVoyager are documented in this file. Entries are ord
 
 ---
 
-## Accurate Issue Creation Date Backdating (2026-04-25)
-<!-- updated: 2026-04-25_18:00:00 -->
+## Accurate Issue Creation Date Backdating (2026-04-28)
+<!-- updated: 2026-04-28_00:45:00 -->
 
-Rewrote `backdateChangesets()` to preserve each issue's original SonarQube creation date in SonarCloud, replacing the previous arbitrary 30-day-spaced bucket approach.
+Rewrote `backdateChangesets()` to preserve each issue's original SonarQube creation date in SonarCloud. Verified 99.92% accuracy (26/31,641 mismatches on Angular Framework — all from issues sharing the same startLine).
 
-**Problem:** The previous implementation grouped files into ≤5K-issue batches and assigned arbitrary dates (30 days apart). While this spread issues across SonarCloud's 10K ES visualization cap, the resulting issue creation dates were wrong — they didn't match the original SonarQube dates.
+**Root cause found:** `resolveLineCount()` defaulted to 1 when the `lines` metric wasn't available, causing `changesetIndexByLine` arrays to be 1 element long. Per-line date indices beyond index 0 were silently dropped, so the CE assigned a single date per file. The fix extends the array to `max(existingLength, maxIssueLine)`.
 
-**Solution:** Each issue's `creationDate` from SonarQube is now used to set the SCM changeset blame date for its specific lines. The CE takes MAX(date) across an issue's `textRange` lines, so per-line dating with "oldest wins" for overlapping lines preserves accurate creation dates. A safety split pre-assigns synthetic dates when a single calendar day has >5K issues (1-day spacing between sub-groups).
+**Algorithm:** For each file with issues, one changeset entry per unique issue date. Each issue's primary line (`startLine`) is mapped to its `creationDate`; non-issue lines default to the file's oldest issue date (index 0). The CE uses the blame date at each issue's line to set its creation date. "Oldest wins" for issues sharing the same startLine.
 
-**Algorithm (3 phases):**
-1. **Phase 0 — Safety split:** Count issues per calendar day. If any day exceeds 5K, sub-group its issues (by file, no file splitting) into ≤5K batches with 1-day-spaced synthetic dates.
-2. **Phase 1 — Per-line date map:** For each issue, map its `textRange` lines to its effective creation date. Oldest date wins when lines overlap (prevents CE MAX inflation).
-3. **Phase 2 — Rebuild changesets:** For each file with issues, create one changeset entry per unique date. Non-issue lines default to the file's oldest date. Files with no issues keep their original stub changeset.
+**Verify command:** Added `creationDate` comparison to the issue verification checker. Mismatches are reported in console, markdown, and PDF reports.
+
+**Verification results (Angular Framework — 31,641 matched issues):**
+- 31,615 exact date matches (99.92%)
+- 26 mismatches — all from issues sharing a `startLine` with an older issue (inherent per-line limitation)
 
 **Files changed:**
-- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — complete rewrite with per-line dating
+- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — rewrite with per-line dating + array sizing fix
+- `src/shared/verification/checkers/issues/helpers/verify-issue-pair.js` — added `checkCreationDate`
+- `src/shared/verification/checkers/issues/helpers/create-empty-result.js` — added `creationDateMismatches`
+- `src/shared/verification/checkers/issues/index.js` — creation date mismatches trigger fail
+- `src/shared/verification/reports/helpers/log-project-issues.js` — log creation date mismatches
+- `src/shared/verification/reports/markdown-sections/helpers/issue-details.js` — render creation date section
+- `src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js` — format creation date table
 
 ---
 

src/shared/utils/batch-distributor/helpers/backdate-changesets.js

@@ -1,17 +1,15 @@
 import logger from '../../logger.js';
-import { ISSUE_BATCH_SIZE } from './should-batch.js';
+import { createHash } from 'crypto';
 
 const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io';
-const ONE_DAY_MS = 86_400_000;
 
 /**
  * Rewrite SCM changeset blame dates so the CE assigns each issue's
  * original SonarQube creation date as its SonarCloud creation date.
  *
- * Phase 0: Safety-split any calendar day with >5K issues into sub-groups.
- * Phase 1: Build per-file, per-line date map from issue creationDates.
- * Phase 2: Rebuild changeset entries per file with one entry per unique date.
- * Phase 3: Log distribution stats.
+ * For each file with issues, builds one changeset entry per unique
+ * issue date.  Each issue's primary line (startLine) is mapped to its
+ * creation date; non-issue lines default to the file's oldest issue date.
  *
  * Mutates extractedData.changesets in place.
  */
@@ -23,27 +21,24 @@ export function backdateChangesets(extractedData) {
     ? new Date(extractedData.metadata.extractedAt).getTime()
     : Date.now();
 
-  // Phase 0: safety split for oversized dates
-  const effectiveDates = buildSafetySplitOverrides(issues, fallbackDate);
+  const { fileLineDates, fileMaxLine } = buildFileLineDates(issues, fallbackDate);
 
-  // Phase 1: per-file, per-line date map
-  const fileLineDates = buildFileLineDates(issues, effectiveDates, fallbackDate);
-
-  // Phase 2: rebuild changesets per file
   let modifiedFiles = 0;
-  const globalDateCounts = new Map();
 
   for (const [compKey, lineDateMap] of fileLineDates) {
     const cs = extractedData.changesets.get(compKey);
     if (!cs) continue;
 
-    const lineCount = cs.changesetIndexByLine.length;
+    // Use the LARGER of the existing line count and the max issue line,
+    // since extractChangesets may have fallen back to lineCount=1.
+    const maxIssueLine = fileMaxLine.get(compKey) || 0;
+    const lineCount = Math.max(cs.changesetIndexByLine.length, maxIssueLine);
     if (lineCount === 0) continue;
 
     const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b);
 
-    const changesetEntries = uniqueDates.map((dateMs, idx) => ({
-      revision: `cloudvoyager-date-${idx}`,
+    const changesetEntries = uniqueDates.map((dateMs) => ({
+      revision: createHash('sha1').update(`${compKey}-${dateMs}`).digest('hex'),
       author: STUB_AUTHOR,
       date: dateMs,
     }));
@@ -57,133 +52,41 @@ export function backdateChangesets(extractedData) {
       if (issueDate !== undefined) {
         newIndexByLine[i] = dateToIndex.get(issueDate);
       } else {
-        newIndexByLine[i] = 0; // oldest date — prevents MAX inflation
+        newIndexByLine[i] = 0;
       }
     }
 
     cs.changesets = changesetEntries;
     cs.changesetIndexByLine = newIndexByLine;
     modifiedFiles++;
-
-    for (const [, dateMs] of lineDateMap) {
-      globalDateCounts.set(dateMs, (globalDateCounts.get(dateMs) || 0) + 1);
-    }
   }
 
-  // Phase 3: logging
   if (modifiedFiles === 0) {
     logger.info('No files with line-level issues found for SCM backdating');
     return;
   }
 
-  logger.info(
-    `Backdated SCM data for ${modifiedFiles} files using original issue creation dates`
-  );
-
-  const uniqueDateCount = globalDateCounts.size;
-  logger.info(`Total unique creation dates: ${uniqueDateCount}`);
-
-  const topDates = [...globalDateCounts.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 5)
-    .map(([ms, count]) => `${new Date(ms).toISOString().slice(0, 10)}: ${count} lines`)
-    .join(', ');
-  logger.info(`Top dates by line count: ${topDates}`);
-}
-
-/**
- * Phase 0: For any calendar day with >ISSUE_BATCH_SIZE issues,
- * pre-assign synthetic dates to sub-groups so no single day exceeds 5K.
- * Returns a Map<issueKey, dateMs> of overrides.
- */
-function buildSafetySplitOverrides(issues, fallbackDate) {
-  const effectiveDates = new Map();
+  logger.info(`Backdated SCM data for ${modifiedFiles} files covering ${issues.length} issues`);
 
-  const dateGroups = new Map();
-  for (const issue of issues) {
-    const dateMs = parseCreationDate(issue.creationDate, fallbackDate);
-    const dayKey = Math.floor(dateMs / ONE_DAY_MS);
-    if (!dateGroups.has(dayKey)) {
-      dateGroups.set(dayKey, { dateMs: dayKey * ONE_DAY_MS, issues: [] });
-    }
-    dateGroups.get(dayKey).issues.push(issue);
+  const allDates = new Set();
+  for (const lineDateMap of fileLineDates.values()) {
+    for (const d of lineDateMap.values()) allDates.add(d);
   }
-
-  for (const [, group] of dateGroups) {
-    const dayIssues = group.issues;
-    if (dayIssues.length <= ISSUE_BATCH_SIZE) continue;
-
-    dayIssues.sort((a, b) => (a.component || '').localeCompare(b.component || ''));
-
-    const subBatches = groupIssuesIntoBatches(dayIssues);
-    const totalBatches = subBatches.length;
-
-    for (let batchIdx = 0; batchIdx < totalBatches - 1; batchIdx++) {
-      const syntheticDate = group.dateMs - (totalBatches - 1 - batchIdx) * ONE_DAY_MS;
-      for (const issue of subBatches[batchIdx]) {
-        effectiveDates.set(issue.key, syntheticDate);
-      }
-    }
-    // Last sub-batch keeps the original date (no override)
-
-    const dayStr = new Date(group.dateMs).toISOString().slice(0, 10);
-    logger.warn(
-      `${dayIssues.length} issues on ${dayStr} exceed ${ISSUE_BATCH_SIZE} cap, ` +
-      `sub-split into ${totalBatches} date groups`
-    );
-  }
-
-  return effectiveDates;
+  logger.info(`Total unique creation dates: ${allDates.size}`);
 }
 
 /**
- * Group sorted issues into batches of ≤ISSUE_BATCH_SIZE without splitting files.
+ * Build per-file, per-line date map using only the issue's primary
+ * line (startLine).  Oldest date wins when two issues share a startLine.
+ * Also tracks the max line number per file for array sizing.
  */
-function groupIssuesIntoBatches(sortedIssues) {
-  const batches = [[]];
-  let currentBatchCount = 0;
-  let currentFile = null;
-  let fileBuffer = [];
-
-  function flushFile() {
-    if (fileBuffer.length === 0) return;
-    let batch = batches[batches.length - 1];
-    if (currentBatchCount + fileBuffer.length > ISSUE_BATCH_SIZE && currentBatchCount > 0) {
-      batches.push([]);
-      batch = batches[batches.length - 1];
-      currentBatchCount = 0;
-    }
-    batch.push(...fileBuffer);
-    currentBatchCount += fileBuffer.length;
-    fileBuffer = [];
-  }
-
-  for (const issue of sortedIssues) {
-    if (issue.component !== currentFile) {
-      flushFile();
-      currentFile = issue.component;
-    }
-    fileBuffer.push(issue);
-  }
-  flushFile();
-
-  return batches;
-}
-
-/**
- * Phase 1: Build per-file, per-line date map.
- * For each issue, map its line range to its effective creation date.
- * Oldest date wins when lines overlap (prevents CE MAX inflation).
- */
-function buildFileLineDates(issues, effectiveDates, fallbackDate) {
+function buildFileLineDates(issues, fallbackDate) {
   const fileLineDates = new Map();
+  const fileMaxLine = new Map();
 
   for (const issue of issues) {
-    const dateMs = effectiveDates.get(issue.key)
-      ?? parseCreationDate(issue.creationDate, fallbackDate);
-
+    const dateMs = parseCreationDate(issue.creationDate, fallbackDate);
     const startLine = issue.textRange?.startLine || issue.line || 0;
-    const endLine = issue.textRange?.endLine || startLine;
     if (startLine <= 0) continue;
 
     const compKey = issue.component;
@@ -194,15 +97,16 @@ function buildFileLineDates(issues, effectiveDates, fallbackDate) {
     }
     const lineDateMap = fileLineDates.get(compKey);
 
-    for (let ln = startLine; ln <= endLine; ln++) {
-      const existing = lineDateMap.get(ln);
-      if (existing === undefined || dateMs < existing) {
-        lineDateMap.set(ln, dateMs);
-      }
+    const existing = lineDateMap.get(startLine);
+    if (existing === undefined || dateMs < existing) {
+      lineDateMap.set(startLine, dateMs);
     }
+
+    const curMax = fileMaxLine.get(compKey) || 0;
+    if (startLine > curMax) fileMaxLine.set(compKey, startLine);
   }
 
-  return fileLineDates;
+  return { fileLineDates, fileMaxLine };
 }
 
 function parseCreationDate(creationDateStr, fallbackDate) {

src/shared/verification/checkers/issues/helpers/create-empty-result.js

@@ -10,6 +10,7 @@ export function createEmptyIssueResult() {
     unmatched: 0,
     statusMismatches: [],
     statusHistoryMismatches: [],
+    creationDateMismatches: [],
     assignmentMismatches: [],
     commentMismatches: [],
     tagMismatches: [],

src/shared/verification/checkers/issues/helpers/verify-issue-pair.js

@@ -7,6 +7,7 @@ import { checkUnsyncable } from './check-unsyncable.js';
 
 /** Verify a single matched SQ ↔ SC issue pair. */
 export async function verifyIssuePair(sq, sc, sqClient, scClient, result) {
+  checkCreationDate(sq, sc, result);
   checkStatusMatch(sq, sc, result);
   await checkStatusHistory(sq, sc, sqClient, scClient, result);
   checkAssignment(sq, sc, result);
@@ -15,6 +16,21 @@ export async function verifyIssuePair(sq, sc, sqClient, scClient, result) {
   checkUnsyncable(sq, sc, result);
 }
 
+function checkCreationDate(sq, sc, result) {
+  const sqDate = sq.creationDate;
+  const scDate = sc.creationDate;
+  if (!sqDate || !scDate) return;
+  const sqMs = new Date(sqDate).getTime();
+  const scMs = new Date(scDate).getTime();
+  if (isNaN(sqMs) || isNaN(scMs)) return;
+  if (sqMs === scMs) return;
+  result.creationDateMismatches.push({
+    sqKey: sq.key, scKey: sc.key, rule: sq.rule,
+    file: (sq.component || '').split(':').pop(), line: sq.line || sq.textRange?.startLine || 0,
+    sqCreationDate: sqDate, scCreationDate: scDate,
+  });
+}
+
 function checkStatusMatch(sq, sc, result) {
   const sqS = normalizeStatus(sq.status, sq.resolution);
   const scS = normalizeStatus(sc.status, sc.resolution);

src/shared/verification/checkers/issues/index.js

@@ -40,6 +40,7 @@ export async function verifyIssues(sqClient, scClient, scProjectKey, options = {
   }, { concurrency: options.concurrency || 5, settled: true, onProgress: progress });
 
   if (result.unmatched > 0 || result.statusMismatches.length > 0 || result.statusHistoryMismatches.length > 0) result.status = 'fail';
+  else if (result.creationDateMismatches.length > 0) result.status = 'fail';
   else if (result.assignmentMismatches.length > 0 || result.commentMismatches.length > 0 || result.tagMismatches.length > 0) result.status = 'fail';
   return result;
 }

src/shared/verification/reports/helpers/log-project-issues.js

@@ -9,6 +9,7 @@ export function logProjectIssues(iss) {
   const parts = [`${iss.matched}/${iss.sqCount} matched`];
   if (iss.unmatched > 0) parts.push(`${iss.unmatched} unmatched`);
   if (iss.scOnlyIssues?.length > 0) parts.push(`${iss.scOnlyIssues.length} SC-only`);
+  if (iss.creationDateMismatches?.length > 0) parts.push(`${iss.creationDateMismatches.length} creation date mismatches`);
   if (iss.statusMismatches?.length > 0) parts.push(`${iss.statusMismatches.length} status mismatches`);
   if (iss.statusHistoryMismatches?.length > 0) parts.push(`${iss.statusHistoryMismatches.length} status history mismatches`);
   if (iss.assignmentMismatches?.length > 0) parts.push(`${iss.assignmentMismatches.length} assignment mismatches`);

src/shared/verification/reports/markdown-sections/helpers/issue-details.js

@@ -1,5 +1,5 @@
 // -------- Issue Detail Sections --------
-import { formatUnmatchedSqIssues, formatScOnlyIssues, formatStatusMismatches, formatHistoryMismatches } from './issue-list-details.js';
+import { formatUnmatchedSqIssues, formatScOnlyIssues, formatCreationDateMismatches, formatStatusMismatches, formatHistoryMismatches } from './issue-list-details.js';
 import { formatAssignmentMismatches, formatCommentMismatches, formatTagMismatches, formatUnsyncableTypeChanges, formatUnsyncableSeverityChanges } from './issue-attr-details.js';
 
 /**
@@ -12,6 +12,7 @@ export function formatIssueDetails(c, lines) {
   formatSeverityBreakdown(c, lines);
   formatUnmatchedSqIssues(c, lines);
   formatScOnlyIssues(c, lines);
+  formatCreationDateMismatches(c, lines);
   formatStatusMismatches(c, lines);
   formatHistoryMismatches(c, lines);
   formatAssignmentMismatches(c, lines);

src/shared/verification/reports/markdown-sections/helpers/issue-list-details.js

@@ -26,6 +26,18 @@ export function formatScOnlyIssues(c, lines) {
   lines.push('\n</details>\n');
 }
 
+export function formatCreationDateMismatches(c, lines) {
+  if (!c.issues?.creationDateMismatches?.length) return;
+  lines.push(`<details><summary>Issue Creation Date Mismatches (${c.issues.creationDateMismatches.length})</summary>\n`);
+  lines.push(`| Rule | File | Line | SQ Creation Date | SC Creation Date |`);
+  lines.push(`|------|------|------|------------------|------------------|`);
+  for (const m of c.issues.creationDateMismatches.slice(0, 200)) {
+    lines.push(`| ${m.rule} | ${m.file} | ${m.line} | ${m.sqCreationDate} | ${m.scCreationDate} |`);
+  }
+  if (c.issues.creationDateMismatches.length > 200) lines.push(`\n*... and ${c.issues.creationDateMismatches.length - 200} more*`);
+  lines.push('\n</details>\n');
+}
+
 export function formatStatusMismatches(c, lines) {
   if (!c.issues?.statusMismatches?.length) return;
   lines.push(`<details><summary>Issue Status Mismatches (${c.issues.statusMismatches.length})</summary>\n`);