From 43c89a847083a23b66045dce3bafe47104780518 Mon Sep 17 00:00:00 2001 From: Joshua Quek Date: Sat, 25 Apr 2026 19:07:19 +0800 Subject: [PATCH] update docs --- docs/CHANGELOG.md | 37 +-- docs/architecture.md | 100 ++------ docs/bespoke-algorithms.md | 141 +++++------ docs/key-capabilities.md | 4 +- docs/technical-details.md | 119 +++------ docs/troubleshooting.md | 33 ++- .../helpers/backdate-changesets.js | 237 ++++++++++++------ 7 files changed, 316 insertions(+), 355 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e964100..c30548e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to CloudVoyager are documented in this file. Entries are ord --- +## Accurate Issue Creation Date Backdating (2026-04-25) + + +Rewrote `backdateChangesets()` to preserve each issue's original SonarQube creation date in SonarCloud, replacing the previous arbitrary 30-day-spaced bucket approach. + +**Problem:** The previous implementation grouped files into ≤5K-issue batches and assigned arbitrary dates (30 days apart). While this spread issues across SonarCloud's 10K ES visualization cap, the resulting issue creation dates were wrong — they didn't match the original SonarQube dates. + +**Solution:** Each issue's `creationDate` from SonarQube is now used to set the SCM changeset blame date for its specific lines. The CE takes MAX(date) across an issue's `textRange` lines, so per-line dating with "oldest wins" for overlapping lines preserves accurate creation dates. A safety split pre-assigns synthetic dates when a single calendar day has >5K issues (1-day spacing between sub-groups). + +**Algorithm (3 phases):** +1. **Phase 0 — Safety split:** Count issues per calendar day. If any day exceeds 5K, sub-group its issues (by file, no file splitting) into ≤5K batches with 1-day-spaced synthetic dates. +2. **Phase 1 — Per-line date map:** For each issue, map its `textRange` lines to its effective creation date. Oldest date wins when lines overlap (prevents CE MAX inflation). +3. **Phase 2 — Rebuild changesets:** For each file with issues, create one changeset entry per unique date. Non-issue lines default to the file's oldest date. Files with no issues keep their original stub changeset. + +**Files changed:** +- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — complete rewrite with per-line dating + +--- + ## Regression Testing System — Phase 1 (2026-04-25) @@ -40,24 +59,12 @@ Fixed issue migration data loss and implemented SCM-based date-bucket distributi **Bug 2 — Missing IN_SANDBOX status in SQ 2025 pipeline:** The `issueStatuses` parameter was missing `IN_SANDBOX` (new in SQ 2025) and incorrectly included `CLOSED`. Fixed in both `issue-methods.js` and `issues-hotspots.js`. -**Feature — SCM date-bucket distribution:** SonarCloud's UI caps the issues list at 10K per creation-date bucket. To work around this, issues are sorted by file and grouped into ≤5K-issue batches. Each batch's files have their SCM changeset blame dates set to a different date (30 days apart). The CE uses SCM blame dates to set creation dates for new issues, spreading them across multiple date buckets within a single analysis. - -**Verification results (Angular Framework — 31,642 issues):** -| Date Bucket | Issues | -|-------------|--------| -| Oct 2025 | 4,854 | -| Nov 2025 | 4,816 | -| Dec 2025 | 4,959 | -| Jan 2026 | 4,767 | -| Feb 2026 | 4,921 | -| Mar 2026 | 4,958 | -| Apr 2026 | 2,366 | -| **Total** | **31,641** (SQ: 31,642) | +**Feature — SCM date-bucket distribution:** Replaced in v1.3.1 by accurate per-issue creation date backdating (see entry above). **Files changed:** - `src/shared/utils/batch-distributor/helpers/should-batch.js` — disabled multi-analysis batching -- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — new: SCM date-bucket distribution -- `src/shared/utils/batch-distributor/helpers/compute-batch-date.js` — 30-day spacing between batches +- `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` — SCM date-bucket distribution (superseded by v1.3.1 rewrite) +- `src/shared/utils/batch-distributor/helpers/compute-batch-date.js` — 30-day spacing between batches (no longer used by backdateChangesets) - `src/pipelines/sq-2025/sonarqube/api-client/helpers/issue-methods.js` — fixed status list - `src/pipelines/sq-2025/sonarqube/api/issues-hotspots.js` — fixed status constant - All 6 pipeline `transfer-branch` entry points — call `backdateChangesets` before protobuf build diff --git a/docs/architecture.md b/docs/architecture.md index 8a5bdeb..0d6e1a6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # 🏗️ Architecture - + ## 📁 Project Structure @@ -74,14 +74,14 @@ src/ │ │ │ ├── build-windows.js # Initial window partitioning │ │ │ ├── fetch-window.js # Fetch issues within a single window │ │ │ └── merge-results.js # Deduplicate and merge sliced results - │ ├── batch-distributor/ # SCM date-bucket distribution (5K per date bucket) - │ │ ├── index.js # Re-exports all batch-distributor helpers + │ ├── batch-distributor/ # SCM date backdating for accurate issue creation dates + │ │ ├── index.js # Re-exports all helpers │ │ ├── helpers/ - │ │ │ ├── backdate-changesets.js # Core: modifies SCM blame dates to spread issue creation dates - │ │ │ ├── should-batch.js # ISSUE_BATCH_SIZE constant; shouldBatch() always returns false - │ │ │ ├── compute-batch-plan.js # Returns batch descriptors (legacy, multi-analysis path) - │ │ │ ├── compute-batch-date.js # Computes backdated ISO date per batch (30-day spacing) - │ │ │ └── create-batch-extracted-data.js # Shallow-clones extracted data (legacy) + │ │ │ ├── should-batch.js # ISSUE_BATCH_SIZE constant (5000); shouldBatch() returns false + │ │ │ ├── backdate-changesets.js # Per-line date backdating from issue.creationDate + │ │ │ ├── compute-batch-plan.js # (legacy) Returns batch descriptors with start/end indices + │ │ │ ├── compute-batch-date.js # (legacy) Computes backdated ISO date per batch + │ │ │ └── create-batch-extracted-data.js # (legacy) Shallow-clones extracted data with sliced issues │ ├── issue-sync/ # Shared issue sync utilities │ │ ├── has-manual-changes.js # Detects human-authored changes on an SQ issue │ │ ├── fetch-sq-changelogs.js # Batch-fetches SQ changelogs concurrently @@ -131,7 +131,7 @@ sq-{version}/ │ └── fetch-and-sync-hotspots.js # Fetches SQ hotspots, syncs to SC ├── transfer-branch.js # Re-export → transfer-branch/index.js ├── transfer-branch/ -│ ├── index.js # Orchestrates per-branch transfer; calls backdateChangesets() before protobuf build +│ ├── index.js # Orchestrates per-branch transfer; gates to batched path when issues > 5K │ └── helpers/ # build-and-encode-report, upload-report, compute-branch-stats, transfer-branch-batched, ... ├── migrate-pipeline.js # Re-export → migrate-pipeline/index.js ├── migrate-pipeline/ @@ -249,24 +249,25 @@ sq-{version}/ **404 JS files** across the sq-10.4 pipeline, all ≤50 lines. Classes converted to factory functions (`createSonarQubeClient`, `createSonarCloudClient`, `createProtobufBuilder`, `createDataExtractor`) with thin class wrappers for backward compatibility. - -### Shared Utilities — SCM Date-Bucket Distribution + +### Shared Utilities — SCM Date Backdating -The **batch-distributor** (`src/shared/utils/batch-distributor/`) works around SonarCloud's Elasticsearch 10K-per-date-bucket visualization limit. When a branch has more than 5K issues, `backdateChangesets()` modifies SCM changeset blame dates within a **single analysis** so the CE assigns different creation dates to different groups of issues, spreading them across multiple date buckets. +The **batch-distributor** (`src/shared/utils/batch-distributor/`) preserves each issue's original SonarQube creation date in SonarCloud by rewriting SCM changeset blame dates in the protobuf report. -> **Note:** Multi-analysis batching (separate scanner report uploads) was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot — issues from prior analyses not in the current one are closed. +The primary function is `backdateChangesets(extractedData)`, which: -Key helpers: +1. **Safety-splits** any calendar day with >5K issues into sub-groups with 1-day-spaced synthetic dates +2. **Maps** each issue's `creationDate` to its `textRange` lines (oldest date wins for overlapping lines) +3. **Rebuilds** each file's changeset with one entry per unique date, and `changesetIndexByLine` pointing each line to the correct date | Function | Purpose | |----------|---------| -| `backdateChangesets(extractedData)` | Core: sorts issues by file, groups files into ≤5K batches, sets ALL lines of each file to the batch date. No-op when issues ≤ 5K. | -| `shouldBatch(extractedData)` | Always returns `false` (multi-analysis batching disabled). Exports `ISSUE_BATCH_SIZE` constant. | -| `computeBatchDate(baseDateISO, batchIndex, totalBatches)` | Computes a backdated ISO date per batch with 30-day spacing | -| `computeBatchPlan(totalIssues)` | Legacy: returns batch descriptors (used by disabled multi-analysis path) | -| `createBatchExtractedData(...)` | Legacy: shallow-clones extracted data (used by disabled multi-analysis path) | +| `backdateChangesets(extractedData)` | Per-line date backdating from `issue.creationDate` — mutates `extractedData.changesets` in place | +| `shouldBatch(extractedData)` | Returns `false` (multi-analysis batching disabled); exports `ISSUE_BATCH_SIZE` constant (5000) | -**Integration:** All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function mutates `extractedData.changesets` in place. +Legacy helpers (`computeBatchPlan`, `computeBatchDate`, `createBatchExtractedData`) remain in the module but are no longer used by `backdateChangesets`. + +**Integration:** All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before the protobuf build step. The function handles all project sizes — no gate or threshold for activation. ## 🔄 Version Routing @@ -589,65 +590,6 @@ The renderer uses vanilla HTML/CSS/JS with no build step. Security follows Elect > **New desktop components:** `progress-parser.js` parses CLI log output to compute real-time progress percentages and ETA for all three pipeline types (migrate, transfer, verify). `whale-animator.js` renders a pixel-art whale sprite animation with starfield, cloud parallax, and typewriter phase labels during execution. -## Regression Testing Architecture - - -Regression tests live in `test/regression/` and validate the full migration pipeline end-to-end against ephemeral SonarQube instances. - -### Directory Structure - -``` -test/regression/ -├── helpers/ # Shared test utilities (SQC client wrappers, retry/backoff) -├── enrichment/ # Scripts that seed SQ with realistic test data -│ ├── add-comments.js # Add issue/hotspot comments -│ ├── change-statuses.js # Transition issue/hotspot statuses -│ └── add-hotspot-data.js # Create hotspot review entries -├── assert-migrate.js # Assertions for migrate command output -├── assert-sync-metadata.js # Assertions for sync-metadata command output -├── assert-verify.js # Assertions for verify command output -└── sample-projects/ # Angular and other sample codebases scanned during CI -``` - -### Ephemeral SQ Docker Architecture - -Each CI job provisions a disposable environment — no shared state between jobs: - -``` -┌─────────────────────────────────────────────────────┐ -│ GitHub Actions Runner │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ SonarQube │ │ PostgreSQL │ │ -│ │ Enterprise │◄──┤ (Docker) │ │ -│ │ (Docker) │ └──────────────┘ │ -│ └──────┬───────┘ │ -│ │ │ -│ 1. Scan Angular sample project │ -│ 2. Run enrichment scripts (comments, statuses, │ -│ hotspot data) │ -│ 3. Run CloudVoyager migrate/sync/verify │ -│ 4. Run assertion scripts against SonarCloud │ -└─────────────────────────────────────────────────────┘ -``` - -### CI Matrix - -Phase 1 scenarios are tested across all 4 supported SonarQube versions (9.9, 10.0, 10.4, 2025.1), producing **5 scenarios × 4 SQ versions = 20 matrix jobs**. Each job is independent and runs in parallel with `fail-fast: false`. - -### Private CI Repository - -Sensitive workflows (secrets, license keys, Docker credentials) run in a private repository (`sonar-solutions/cloudvoyager-ci`). This repo: -- Syncs from the public repo automatically every 15 minutes -- Hosts the regression workflow files and encrypted secrets -- Reports status back to the public repo via commit statuses - -### Assertion and Enrichment Details - -**Assertion scripts** use the shared SonarCloud API client (`helpers/`) with built-in retry and exponential backoff to account for eventual consistency in SonarCloud's CE pipeline. Each `assert-*.js` script validates command-specific invariants (issue counts, status mappings, hotspot states, metadata parity). - -**Enrichment scripts** run after the initial SonarQube scan but before migration, seeding the SQ instance with realistic data that exercises metadata sync paths: issue comments, status transitions (confirm, false-positive, won't-fix), and hotspot review entries with assigned reviewers. - ## 📚 Further Reading - [Configuration Reference](configuration.md) — all config options, environment variables, npm scripts diff --git a/docs/bespoke-algorithms.md b/docs/bespoke-algorithms.md index d2efc0b..458c120 100644 --- a/docs/bespoke-algorithms.md +++ b/docs/bespoke-algorithms.md @@ -190,109 +190,92 @@ CloudVoyager checkpoints migration progress so a run can be resumed after any in --- -## 5. SCM Date-Bucket Distribution (Upload-Side 10K Mitigation) +## 5. Accurate Issue Creation Date Backdating - + + ### Problem -SonarCloud's Elasticsearch visualization layer caps results at **10,000 issues per date bucket**. When CloudVoyager migrates a branch, the scanner report is uploaded with a single `analysis_date`. If the branch carries more than 10K issues, only the first 10,000 are visible in the SonarCloud UI — the rest silently disappear from the Issues tab. This is an ES index-time bucketing limitation, not an API pagination issue, so it cannot be solved client-side. +SonarCloud's Compute Engine assigns creation dates to NEW issues from SCM blame data. Each file's changeset protobuf has `changesets[]` (array of `{revision, author, date}`) and `changesetIndexByLine[]` (maps each line to a changeset index). The CE takes **MAX(date)** across an issue's `textRange.startLine..endLine` to determine its creation date. -### Why Multi-Analysis Batching Failed - -The original approach split issues across multiple scanner report uploads with distinct analysis dates. This failed because SonarCloud's CE issue tracker treats each analysis as a **complete snapshot** — issues from prior analyses not present in the current one are **closed**. Only the last batch's issues survived, silently destroying all prior batches' data. - -### Current Solution: Single-Analysis SCM Backdating - -All issues are uploaded in a **single analysis**. The CE assigns issue creation dates from SCM blame data, so by modifying the changeset blame dates per file, we control which date bucket each issue lands in — all within one report. +Without backdating, all issues in a migrated project would get the same creation date (the extraction timestamp). The goal is **1:1 accuracy** — each issue's creation date in SonarCloud should match its original `creationDate` from SonarQube. A 5K-per-day safety split handles rare cases where a single calendar day has >5K issues (SonarCloud's ES visualization cap is 10K per date bucket). ### Algorithm ``` backdateChangesets(extractedData): issues = extractedData.issues - IF issues.length <= 5000: RETURN // no-op for small projects - - baseDate = extractedData.metadata.extractedAt - - // Sort issues by file so files are contiguous - issues.sort(by component key) - - // Group files into batches of ≤5K issues (no file splitting) - fileBatches = buildFileBatches(issues) - IF fileBatches.length <= 1: RETURN - - // Backdate ALL lines of each file in non-final batches - FOR EACH batch (except last): - batchDate = computeBatchDate(baseDate, batchIdx, totalBatches) - FOR EACH file IN batch: - changeset = extractedData.changesets.get(file) - changeset.changesets = [{ revision: batchRevision, author: stub, date: batchDate }] - changeset.changesetIndexByLine.fill(0) // all lines → single backdated entry - - // Last batch keeps original dates (untouched) + fallbackDate = extractedData.metadata.extractedAt || Date.now() + + // Phase 0: Safety split for oversized dates + effectiveDates = Map() + dateGroups = group issues by creationDate (truncated to calendar day) + FOR EACH (day, dayIssues) WHERE dayIssues.length > 5000: + sort dayIssues by component + subBatches = groupFilesIntoBatches(dayIssues, 5000) // no file splitting + FOR EACH subBatch (except last): + syntheticDate = day - (totalBatches - 1 - batchIdx) * 1 day + FOR EACH issue IN subBatch: + effectiveDates.set(issue.key, syntheticDate) + // Last sub-batch keeps original date + + // Phase 1: Per-file, per-line date map + fileLineDates = Map> + FOR EACH issue: + dateMs = effectiveDates[issue.key] ?? parse(issue.creationDate) ?? fallbackDate + startLine = issue.textRange?.startLine || issue.line || 0 + endLine = issue.textRange?.endLine || startLine + IF startLine <= 0: SKIP // file-level issue + FOR ln = startLine TO endLine: + IF !fileLineDates[component][ln] OR dateMs < existing: + fileLineDates[component][ln] = dateMs // OLDEST wins + + // Phase 2: Rebuild changesets per file + FOR EACH (compKey, lineDateMap) IN fileLineDates: + cs = extractedData.changesets.get(compKey) + IF !cs OR lineCount == 0: CONTINUE + uniqueDates = sorted unique values from lineDateMap + cs.changesets = one {revision, author, date} per unique date + FOR EACH line i (0..lineCount-1): + IF lineDateMap has (i+1): newIndexByLine[i] = dateToIndex[lineDateMap[i+1]] + ELSE: newIndexByLine[i] = 0 // oldest date (safe default) + cs.changesetIndexByLine = newIndexByLine ``` -### Critical Insight: Full-File Backdating +### Why "Oldest Wins" for Overlapping Lines -Setting only individual issue lines is **insufficient**. The CE uses the MAX of SCM dates across an issue's line range — surrounding lines with the current date override any backdated line. The solution sets **every line** of each file to the batch date, ensuring the CE has no newer line to fall back on. +The CE takes MAX across a multi-line issue's range: +- **Older issue**: all its lines ≤ its date → MAX = its date (correct) +- **Newer issue**: overlapping lines have older date, but non-overlapping lines have its correct date → MAX = correct date (correct) +- **Exception**: newer issue entirely contained within older issue's range → inherits older date (unavoidable CE MAX limitation — rare for real code issues) -### Key Invariants +Non-issue lines default to the file's oldest issue date to prevent accidental MAX inflation for any multi-line issue spanning them. -- **Batch size = 5,000 (constant)** — 50% safety margin under the 10K ES limit. -- **Single analysis** — all issues uploaded in one report, preserving data integrity. No CE issue tracker conflicts. -- **File-level granularity** — all lines of a file share the same batch date. Files are never split across batches. -- **30-day spacing** — batches are spaced 30 days apart (via `computeBatchDate`) for clear separation in the SonarCloud UI date facet. -- **Last batch untouched** — the final batch keeps original SCM dates; only earlier batches are backdated. -- **`shouldBatch` always returns false** — multi-analysis batching is permanently disabled. `backdateChangesets()` is called unconditionally before protobuf build; it no-ops when issues ≤ 5K. - -### Example - -A branch with 31,641 issues extracted on 2026-04-23: +### Key Invariants -| Batch | Files | Issues | SCM Blame Date | -|-------|-------|--------|----------------| -| 1/7 | Files A–F | ~4,854 | Oct 2025 | -| 2/7 | Files G–K | ~4,816 | Nov 2025 | -| 3/7 | Files L–P | ~4,959 | Dec 2025 | -| 4/7 | Files Q–T | ~4,767 | Jan 2026 | -| 5/7 | Files U–W | ~4,921 | Feb 2026 | -| 6/7 | Files X–Y | ~4,958 | Mar 2026 | -| 7/7 | Files Z+ | ~2,366 | Apr 2026 (original) | +- **All projects are backdated** — no early return for small projects. Every project gets accurate dates. +- **Per-line granularity** — each line in a file can have a different date, enabling multiple issues with different creation dates in the same file. +- **Safety split threshold = 5,000** — matches the `ISSUE_BATCH_SIZE` constant. Days exceeding this get sub-grouped with 1-day spacing. +- **No file splitting** — the safety split groups whole files into sub-batches; a file's issues are never split across different synthetic dates. +- **Fallback chain** — `effectiveDates[key]` → `parse(issue.creationDate)` → `extractedAt` → `Date.now()`. +- **Files with no issues unchanged** — only files appearing in `fileLineDates` get their changesets rewritten; others keep their original stub. ### Pipeline Integration -All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function is a no-op for projects with ≤5K issues. - -``` -transfer-branch.js - └── backdateChangesets(extractedData) // mutates SCM dates in-place - └── buildProtobufMessages(extractedData) - └── encodeAndUpload(messages) -``` - -### Implementation Files - -| File | Role | -|------|------| -| `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` | Core: sorts issues by file, groups into batches, backdates all lines per file | -| `src/shared/utils/batch-distributor/helpers/should-batch.js` | Exports `ISSUE_BATCH_SIZE` constant; `shouldBatch()` always returns false (multi-analysis disabled) | -| `src/shared/utils/batch-distributor/helpers/compute-batch-date.js` | Computes backdated ISO date per batch (30-day spacing) | +All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before the protobuf build step. No signature change — the function mutates `extractedData.changesets` in place. ### Implementation ``` -src/shared/utils/batch-distributor/ - index.js — Re-exports all helpers - helpers/ - should-batch.js — Gate: issues.length > 5000 - compute-batch-plan.js — Returns [{startIndex, endIndex, batchIndex, isLast}] - compute-batch-date.js — Backdates: baseDate - N days - create-batch-extracted-data.js — Shallow clone with sliced issues + metadata override - -src/pipelines//transfer-pipeline/helpers/ - transfer-branch.js — shouldBatch gate, delegates to batched or single path - transfer-branch-batched.js — Batch loop: build → encode → uploadAndWait per batch +src/shared/utils/batch-distributor/helpers/ + backdate-changesets.js — Per-line date backdating from issue.creationDate + should-batch.js — ISSUE_BATCH_SIZE constant (5000); shouldBatch() returns false + +Legacy files (unchanged, no longer used by backdateChangesets): + compute-batch-plan.js — Returns batch descriptors with start/end indices + compute-batch-date.js — Computes 30-day-spaced backdated dates + create-batch-extracted-data.js — Shallow clone with sliced issues ``` --- diff --git a/docs/key-capabilities.md b/docs/key-capabilities.md index aa0eb15..5d948c0 100644 --- a/docs/key-capabilities.md +++ b/docs/key-capabilities.md @@ -217,9 +217,9 @@ The report is submitted to SonarCloud's CE endpoint as a multipart form upload w Each report includes an `scm_revision_id` (git commit hash) in its metadata. SonarCloud uses this to detect and reject duplicate submissions, preventing accidental data duplication across multiple migration runs. -### Issue Batching for Large Projects +### Accurate Issue Creation Date Preservation -When a branch has more than 5,000 issues, CloudVoyager automatically splits the issues into batches of 5,000 and submits each batch as a separate scanner report with a distinct `analysis_date` going backwards from today. This prevents hitting SonarCloud's Elasticsearch visualization limit of 10K results per date bucket, ensuring all migrated issues are visible in the UI. The batching is transparent to the caller — `transferBranch` automatically detects when batching is needed and routes accordingly. Non-final batches strip sources, changesets, and duplications to minimize upload size, while keeping components and active rules for issue resolution. Each batch uses a unique `scmRevisionId` to prevent CE deduplication. +CloudVoyager preserves each issue's original SonarQube creation date in SonarCloud. `backdateChangesets()` reads each issue's `creationDate` and maps it to per-line SCM blame dates in the changeset protobuf. The CE takes MAX(date) across an issue's `textRange` lines, so "oldest wins" for overlapping lines ensures accurate creation dates. A safety split handles calendar days with >5K issues (sub-groups with 1-day-spaced synthetic dates, no file splitting). This produces a realistic historical distribution in SonarCloud's creation date facet matching the original SonarQube project history, instead of arbitrary batch-spaced clusters. ### Branch Name Resolution diff --git a/docs/technical-details.md b/docs/technical-details.md index d7601ef..f8dc410 100644 --- a/docs/technical-details.md +++ b/docs/technical-details.md @@ -34,14 +34,14 @@ sequenceDiagram Note over CLI: ➕B Build & encode protobuf report - Note over CLI: ➕B2 SCM date-bucket distribution (if >5K issues) - CLI->>CLI: backdateChangesets(extractedData) — spreads SCM blame dates across 30-day buckets - CLI->>CLI: build Issue / ExternalIssue / AdHocRule messages - CLI->>CLI: build Measure / Component / Changeset messages - CLI->>CLI: zip → scanner-report.zip - - Note over CLI,SC: ➕C Upload & CE retry - CLI->>SC: POST /api/ce/submit (scanner-report.zip) + Note over CLI: backdateChangesets() — rewrite SCM blame dates per-line from issue.creationDate + else single upload + CLI->>CLI: build Issue / ExternalIssue / AdHocRule messages + CLI->>CLI: build Measure / Component / Changeset messages + CLI->>CLI: zip → scanner-report.zip + + Note over CLI,SC: ➕C Upload & CE retry + CLI->>SC: POST /api/ce/submit (scanner-report.zip) SC-->>CLI: CE task ID (or timeout → poll /api/ce/activity) end @@ -139,7 +139,7 @@ sequenceDiagram | ➕D | Issue sync | `Generate a mermaid sequence diagram showing the full issue sync pipeline in CloudVoyager including pre-filter, SC indexing wait, and changelog replay` | | ➕E | Org mapping / CSVs | `Generate a mermaid sequence diagram showing how CloudVoyager maps SonarQube projects to SonarCloud organizations and generates dry-run CSV files` | | ➕F | Quality profiles | `Generate a mermaid sequence diagram for CloudVoyager quality profile migration including backup XML, rename of built-in profiles, and diff report` | -| ➕G | SCM date-bucket distribution | `Generate a mermaid sequence diagram showing how CloudVoyager backdateChangesets() modifies SCM blame dates within a single analysis to spread issues across ≤5K date buckets` | +| ➕G | Batch distribution | `Generate a mermaid sequence diagram showing how CloudVoyager batch-distributor splits large issue sets into ≤5K batches with backdated analysis dates and sequential CE uploads` | ## 📡 Protobuf Encoding @@ -229,7 +229,7 @@ Components use a flat structure - all files are direct children of the project c The tool includes `scm_revision_id` (git commit hash) in metadata. SonarCloud uses this to detect and reject duplicate reports, enabling proper analysis history tracking. -**SCM date-bucket distribution**: When a branch has >5,000 issues, `backdateChangesets()` modifies SCM changeset blame dates within a single analysis so the CE assigns different creation dates to different groups of issues. See [SCM Date-Bucket Distribution](#-scm-date-bucket-distribution-5k-per-date-bucket) for details. +**Batch uploads**: When issue batching is active (>5,000 issues), each batch receives a unique `scmRevisionId` generated via `randomBytes(20).toString('hex')` instead of the original git commit hash. This prevents the SonarCloud CE from deduplicating successive batch uploads as identical reports. See [Issue Batching for Upload](#-issue-batching-for-upload-5k-per-date-bucket) for details. ## 🌿 Branch Sync @@ -259,65 +259,42 @@ The extractors handle these lower limits automatically. **metricKeys batching**: SQ 9.9 through 10.8 limits `metricKeys` to 15 per request (must batch). SQ 2025.1+ has no batching limit. - -## 📦 SCM Date-Bucket Distribution (5K Per Date Bucket) + +## 📦 Accurate Issue Creation Date Backdating -SonarCloud's Elasticsearch caps **visualization** at 10,000 issues per date bucket. When a branch has more than 10K issues, only the first 10,000 are visible in the UI — even though all issues exist in the database. +SonarCloud's CE assigns creation dates to NEW issues from SCM blame data. `backdateChangesets()` rewrites each file's changeset protobuf so that the CE assigns each issue its original SonarQube creation date. -**Solution:** `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` modifies SCM changeset blame dates within a **single analysis** so the CE assigns different creation dates to different groups of issues, spreading them across multiple date buckets. +**How it works:** The function reads each issue's `creationDate` field (preserved from SonarQube extraction) and maps it to the issue's `textRange` lines in the file's changeset data. The CE takes MAX(date) across an issue's line range, so using "oldest wins" for overlapping lines ensures accurate dates. A safety split handles days with >5K issues. -> **Note:** The original multi-analysis batching approach (splitting issues across separate scanner report uploads) was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot — issues from prior analyses not present in the current one are closed. Only single-analysis SCM backdating preserves all issues. +### Algorithm (3 Phases) -### Algorithm - -1. **Gate** — if `issues.length <= 5000`, no-op (no backdating needed) -2. **Sort** — issues sorted by component key so files are contiguous -3. **Group** — `buildFileBatches()` walks sorted issues, grouping files into batches of ≤5K issues. Files are never split across batches. -4. **Backdate** — for each batch (except the last), ALL lines of each file have their changeset replaced with a single entry at the batch date. `computeBatchDate()` spaces batches 30 days apart going backwards from the extraction date. -5. **Last batch untouched** — keeps original SCM dates - -### Why Full-File Backdating - -Setting only individual issue lines is insufficient. The CE uses the **MAX** of SCM dates across an issue's line range — surrounding lines with the current date override any backdated line. Setting **every line** of each file ensures the CE has no newer line to fall back on. - -### Example - -A branch with 31,641 issues extracted on 2026-04-23: - -| Batch | Issues | SCM Blame Date | CE Creation Date | -|-------|--------|----------------|-----------------| -| 1/7 | ~4,854 | Oct 2025 | Oct 2025 | -| 2/7 | ~4,816 | Nov 2025 | Nov 2025 | -| 3/7 | ~4,959 | Dec 2025 | Dec 2025 | -| 4/7 | ~4,767 | Jan 2026 | Jan 2026 | -| 5/7 | ~4,921 | Feb 2026 | Feb 2026 | -| 6/7 | ~4,958 | Mar 2026 | Mar 2026 | -| 7/7 | ~2,366 | Apr 2026 (original) | Apr 2026 | +1. **Phase 0 — Safety split:** Group issues by calendar day. For any day exceeding 5,000 issues, sort by component and sub-group into ≤5K batches with 1-day-spaced synthetic dates (no file splitting). +2. **Phase 1 — Per-line date map:** For each issue, map its `textRange.startLine..endLine` to its effective creation date. Oldest date wins when lines overlap. File-level issues (line=0/null) are skipped. +3. **Phase 2 — Rebuild changesets:** For each file with issues, create one changeset entry per unique date. `changesetIndexByLine[i]` points to the date for line `i+1`, or to the oldest date (index 0) for non-issue lines. Files with no issues keep their original stub. ### Design Decisions | Decision | Rationale | |----------|-----------| -| Batch size = 5,000 (hardcoded) | 50% safety margin under ES 10K visualization limit | -| Single analysis (not multi-analysis) | CE issue tracker closes issues from prior analyses not in current one | -| Full-file line backdating | CE uses MAX of SCM dates across issue line range; partial backdating is overridden | -| 30-day spacing between batches | Clear separation in SonarCloud UI date facet | -| File-level granularity (no file splitting) | Simpler, avoids partial-file changeset complexity | -| Last batch keeps original dates | Most recent issues appear at the correct time | +| Per-line dating (not per-file) | Enables multiple issues with different creation dates in the same file | +| Oldest date wins for overlapping lines | CE takes MAX across line range — older date on shared lines doesn't inflate newer issues that have their own non-overlapping lines | +| Non-issue lines get oldest date | Prevents accidental MAX inflation for multi-line issues spanning non-issue lines | +| Safety split at 5,000/day | 50% margin under SonarCloud's 10K ES visualization cap per date bucket | +| No file splitting in safety split | A file's issues stay together on the same synthetic date | +| All projects backdated | No early return for small projects — every project gets accurate dates | ### Helper Files (`src/shared/utils/batch-distributor/helpers/`) | File | Role | |------|------| -| `backdate-changesets.js` | Core: sorts issues, groups files into batches, backdates all lines per file | -| `should-batch.js` | Exports `ISSUE_BATCH_SIZE` constant; `shouldBatch()` always returns false (multi-analysis disabled) | -| `compute-batch-date.js` | Computes backdated ISO date per batch (30-day spacing) | -| `compute-batch-plan.js` | Returns batch descriptors (legacy, used by disabled multi-analysis path) | -| `create-batch-extracted-data.js` | Shallow-clones extracted data (legacy, used by disabled multi-analysis path) | +| `backdate-changesets.js` | Per-line date backdating from `issue.creationDate` (complete rewrite) | +| `should-batch.js` | `ISSUE_BATCH_SIZE` constant (5000); `shouldBatch()` returns false | + +Legacy files (unchanged, no longer used by `backdateChangesets`): `compute-batch-plan.js`, `compute-batch-date.js`, `create-batch-extracted-data.js`. ### Pipeline Integration -All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function mutates `extractedData.changesets` in place and is a no-op for projects with ≤5K issues. +All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before the protobuf build step. The function mutates `extractedData.changesets` in place — no signature change. ## 🔍 Search Slicing for 10K+ Issues @@ -699,44 +676,6 @@ All errors extend `CloudVoyagerError` base class (11 classes total): SIGINT/SIGTERM triggers registered cleanup handlers (journal save, lock release) then exits with code 0. A second signal forces immediate exit. The `shutdownCheck()` callback is passed through the pipeline for interrupt safety, allowing long-running operations to check for pending shutdown. -## Ephemeral SonarQube Container Architecture (Regression Testing) - - -Each regression test CI job provisions a disposable SonarQube Enterprise environment from scratch, runs the CloudVoyager pipeline against it, asserts outcomes, and tears everything down. No shared state survives between jobs. - -### Per-Job Lifecycle - -``` -Job starts → PostgreSQL container → SQ Enterprise container → Health check → License inject → Token create → Enrichment → Migration → Assertion → Teardown -``` - -### Container Provisioning - -- Each CI job spins up **two containers**: a PostgreSQL container (backing store) and a SonarQube Enterprise container (version-specific, e.g. `sonarqube:10.4-enterprise`). -- Both containers communicate via a Docker **user-defined bridge network** (`sq-net`), which provides DNS-based service discovery between the database and SonarQube. - -### Startup & Health Check - -- SonarQube startup takes **30–90 seconds** depending on the version and runner hardware. -- The setup script polls `GET /api/system/status` in a loop until the response payload contains `"status": "UP"`, indicating the instance is fully operational and ready to accept API calls. - -### License Injection - -- The SonarQube Enterprise license key is injected via `POST /api/editions/set_license` immediately after the health check passes. -- The license key is a CI secret, masked in logs via GitHub Actions' `::add-mask::` directive to prevent accidental exposure. - -### Admin Token Generation - -- An admin API token is created via `POST /api/user_tokens/generate` so that CloudVoyager can authenticate against the ephemeral SonarQube instance using bearer token auth rather than basic auth. - -### Project Key Namespacing - -- SQC target project keys use the `cv-regression-{scenario}-{sq-version}` namespace (e.g. `cv-regression-large-project-10.4`) to prevent collision with existing regression workflows that may target the same SonarCloud organization. - -### Composite Action - -The `setup-sonarqube` composite action (`.github/actions/setup-sonarqube/action.yml`) encapsulates the entire provisioning sequence — container creation, networking, health polling, license injection, and token generation — as a reusable building block for all regression test jobs. - ## 📚 Further Reading - [Configuration Reference](configuration.md) — all config options, environment variables, npm scripts diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 88d12de..6a78055 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -585,12 +585,12 @@ The modern statuses (`FALSE_POSITIVE`, `ACCEPTED`, `FIXED`) do **not** exist in --- - + ## 📊 Projects with 10,000+ Issues SonarQube's `/api/issues/search` endpoint caps results at 10,000 due to an Elasticsearch hard limit. -> **Note (v1.3+):** Large-project issue handling now has two complementary mechanisms. The **search slicer** handles retrieval of >10K issues from SonarQube by splitting the date range into windows that each stay under the 10K API limit. The **SCM date-bucket distribution** (`backdateChangesets()`) handles uploading >5K issues to SonarCloud by modifying SCM changeset blame dates within a single analysis, so the CE assigns different creation dates to different groups of issues. This prevents the Elasticsearch visualization limit (10K per date bucket) from hiding issues in the UI. Together, these two features ensure that projects of any size are fully extracted from SonarQube and fully visible in SonarCloud. +> **Note (v1.3+):** Large-project issue handling now has two complementary mechanisms. The **search slicer** handles retrieval of >10K issues from SonarQube by splitting the date range into windows that each stay under the 10K API limit. The **SCM date backdating** (`backdateChangesets`) preserves each issue's original SonarQube creation date in SonarCloud by writing per-line blame dates into the changeset protobuf. A safety split ensures no single calendar day exceeds 5K issues (50% margin under the 10K ES visualization cap). Together, these two features ensure that projects of any size are fully extracted from SonarQube and fully visible in SonarCloud with accurate historical creation dates. ### Error: `10001th result asked` @@ -618,33 +618,28 @@ The search slicer algorithm: --- - -## 📦 SCM Date-Bucket Distribution (Large Issue Sets) - -When a branch has more than 5,000 issues, CloudVoyager automatically modifies SCM changeset blame dates so the CE assigns different creation dates to different groups of issues. This spreads issues across multiple date buckets within a **single analysis**, preventing SonarCloud's Elasticsearch visualization limit (10K per date bucket) from hiding issues. + +## 📦 Issue Creation Date Accuracy -> **This is automatic and transparent.** `backdateChangesets()` is called before every protobuf build and is a no-op for branches with ≤5K issues. Users do not need to enable it or take any special action. +CloudVoyager preserves each issue's original SonarQube creation date in SonarCloud by rewriting SCM changeset blame dates in the protobuf report. This is automatic and transparent — no user configuration needed. -> **Note:** An earlier version used multi-analysis batching (separate scanner report uploads per batch). This was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot — issues from prior analyses not in the current one are **closed**, causing silent data loss. +> **How it works (v1.3.1+):** `backdateChangesets()` reads each issue's `creationDate` field from SonarQube and maps it to the issue's `textRange` lines in the file's changeset data. The CE takes MAX(date) across an issue's line range to determine its creation date, so per-line dating with "oldest wins" for overlapping lines preserves accurate dates. A safety split handles calendar days with >5K issues. -### Why do I see issues spread across multiple dates? +### Issue creation dates don't match SonarQube -**Expected behavior.** If a branch has more than 5,000 issues, CloudVoyager logs: +Check that the migration was run with v1.3.1+. Earlier versions used arbitrary 30-day-spaced batch dates instead of original creation dates. Re-transfer affected projects to get accurate dates. -``` -Backdating SCM data: 31641 issues → 7 date buckets of ≤5000 -Modified SCM data for 2847 files across 7 date buckets -``` +### Issue counts look correct in the API but not in the UI -Issues are grouped by file into ≤5K batches. Each batch's files have their SCM blame dates set to a different month (30 days apart). The CE uses SCM blame dates to assign issue creation dates, so issues land in separate date buckets. The last batch keeps the original dates. +SonarCloud's Elasticsearch caps visualization at 10K per date bucket. With accurate creation dates, issues are naturally distributed across their original dates. The safety split ensures no single day exceeds 5K issues. If you see this problem, verify the migration used v1.3.1+. -### Issue counts look correct in the API but not in the UI +### Warning: "N issues on DATE exceed 5K cap" -This is the exact problem SCM date-bucket distribution solves. Without it, all issues share one creation date and SonarCloud's Elasticsearch only visualizes the first 10K per date bucket. Re-transfer the affected projects — `backdateChangesets()` will spread the issues automatically. +**Expected behavior.** This means a single calendar day had more than 5,000 issues with the same creation date. The safety split automatically sub-groups them into ≤5K batches with 1-day-spaced synthetic dates. The sub-groups will appear as separate adjacent dates in SonarCloud's creation date facet. -### Can I change the batch size? +### Can I change the safety split threshold? -The batch size is hardcoded at 5,000 (50% safety margin under the 10K ES limit). It is not configurable. +The threshold is hardcoded at 5,000 (50% safety margin under the 10K ES visualization limit). It is not configurable. --- diff --git a/src/shared/utils/batch-distributor/helpers/backdate-changesets.js b/src/shared/utils/batch-distributor/helpers/backdate-changesets.js index bb3bd16..34647a5 100644 --- a/src/shared/utils/batch-distributor/helpers/backdate-changesets.js +++ b/src/shared/utils/batch-distributor/helpers/backdate-changesets.js @@ -1,117 +1,212 @@ import logger from '../../logger.js'; import { ISSUE_BATCH_SIZE } from './should-batch.js'; -import { computeBatchDate } from './compute-batch-date.js'; - -// -------- Backdate Changesets -------- const STUB_AUTHOR = 'cloudvoyager-migration@sonarcloud.io'; +const ONE_DAY_MS = 86_400_000; /** - * Modify SCM changeset blame dates so the CE assigns different creation - * dates to different groups of issues. This spreads issues across - * multiple date buckets within a **single** analysis. + * Rewrite SCM changeset blame dates so the CE assigns each issue's + * original SonarQube creation date as its SonarCloud creation date. * - * Strategy: sort issues by file, group files into ≤5K-issue batches, - * then set ALL lines of each file to that batch's date. Setting every - * line ensures the CE has no "newer" line to fall back on. + * Phase 0: Safety-split any calendar day with >5K issues into sub-groups. + * Phase 1: Build per-file, per-line date map from issue creationDates. + * Phase 2: Rebuild changeset entries per file with one entry per unique date. + * Phase 3: Log distribution stats. * - * Mutates extractedData.issues (sort order) and - * extractedData.changesets (dates) in place. + * Mutates extractedData.changesets in place. */ export function backdateChangesets(extractedData) { const issues = extractedData.issues || []; - if (issues.length <= ISSUE_BATCH_SIZE) return; + if (issues.length === 0) return; - const baseDateISO = extractedData.metadata?.extractedAt || new Date().toISOString(); + const fallbackDate = extractedData.metadata?.extractedAt + ? new Date(extractedData.metadata.extractedAt).getTime() + : Date.now(); - // Sort issues by component so files are contiguous - issues.sort((a, b) => (a.component || '').localeCompare(b.component || '')); + // Phase 0: safety split for oversized dates + const effectiveDates = buildSafetySplitOverrides(issues, fallbackDate); - // Group files into batches of ≤ISSUE_BATCH_SIZE issues. - // Each file goes entirely into one batch (no splitting a file across batches). - const fileBatches = buildFileBatches(issues); - const batchCount = fileBatches.length; + // Phase 1: per-file, per-line date map + const fileLineDates = buildFileLineDates(issues, effectiveDates, fallbackDate); - if (batchCount <= 1) return; + // Phase 2: rebuild changesets per file + let modifiedFiles = 0; + const globalDateCounts = new Map(); - logger.info( - `Backdating SCM data: ${issues.length} issues → ${batchCount} date buckets of ≤${ISSUE_BATCH_SIZE}` - ); + for (const [compKey, lineDateMap] of fileLineDates) { + const cs = extractedData.changesets.get(compKey); + if (!cs) continue; - let modifiedFiles = 0; + const lineCount = cs.changesetIndexByLine.length; + if (lineCount === 0) continue; - for (let batchIdx = 0; batchIdx < batchCount; batchIdx++) { - if (batchIdx === batchCount - 1) break; // last batch keeps original date + const uniqueDates = [...new Set(lineDateMap.values())].sort((a, b) => a - b); - const batchDateMs = new Date( - computeBatchDate(baseDateISO, batchIdx, batchCount) - ).getTime(); + const changesetEntries = uniqueDates.map((dateMs, idx) => ({ + revision: `cloudvoyager-date-${idx}`, + author: STUB_AUTHOR, + date: dateMs, + })); - const batchRevision = `cloudvoyager-batch-${String(batchIdx + 1).padStart(4, '0')}`; + const dateToIndex = new Map(); + uniqueDates.forEach((d, i) => dateToIndex.set(d, i)); - for (const compKey of fileBatches[batchIdx].files) { - const cs = extractedData.changesets.get(compKey); - if (!cs) continue; + const newIndexByLine = new Array(lineCount); + for (let i = 0; i < lineCount; i++) { + const issueDate = lineDateMap.get(i + 1); + if (issueDate !== undefined) { + newIndexByLine[i] = dateToIndex.get(issueDate); + } else { + newIndexByLine[i] = 0; // oldest date — prevents MAX inflation + } + } - // Replace the single stub changeset with the backdated one - cs.changesets = [{ - revision: batchRevision, - author: STUB_AUTHOR, - date: batchDateMs, - }]; - // Point every line to changeset index 0 (the only entry) - cs.changesetIndexByLine.fill(0); + cs.changesets = changesetEntries; + cs.changesetIndexByLine = newIndexByLine; + modifiedFiles++; - modifiedFiles++; + for (const [, dateMs] of lineDateMap) { + globalDateCounts.set(dateMs, (globalDateCounts.get(dateMs) || 0) + 1); } } + // Phase 3: logging + if (modifiedFiles === 0) { + logger.info('No files with line-level issues found for SCM backdating'); + return; + } + logger.info( - `Modified SCM data for ${modifiedFiles} files across ${batchCount} date buckets` + `Backdated SCM data for ${modifiedFiles} files using original issue creation dates` ); + + const uniqueDateCount = globalDateCounts.size; + logger.info(`Total unique creation dates: ${uniqueDateCount}`); + + const topDates = [...globalDateCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([ms, count]) => `${new Date(ms).toISOString().slice(0, 10)}: ${count} lines`) + .join(', '); + logger.info(`Top dates by line count: ${topDates}`); } /** - * Walk the sorted issues array and collect files into batches. - * Each batch accumulates files until the issue count exceeds - * ISSUE_BATCH_SIZE, then a new batch starts. + * Phase 0: For any calendar day with >ISSUE_BATCH_SIZE issues, + * pre-assign synthetic dates to sub-groups so no single day exceeds 5K. + * Returns a Map of overrides. */ -function buildFileBatches(issues) { - const batches = [{ files: new Set(), issueCount: 0 }]; - let currentFile = null; - let currentFileIssueCount = 0; +function buildSafetySplitOverrides(issues, fallbackDate) { + const effectiveDates = new Map(); + const dateGroups = new Map(); for (const issue of issues) { - const compKey = issue.component; - if (!compKey) continue; + const dateMs = parseCreationDate(issue.creationDate, fallbackDate); + const dayKey = Math.floor(dateMs / ONE_DAY_MS); + if (!dateGroups.has(dayKey)) { + dateGroups.set(dayKey, { dateMs: dayKey * ONE_DAY_MS, issues: [] }); + } + dateGroups.get(dayKey).issues.push(issue); + } + + for (const [, group] of dateGroups) { + const dayIssues = group.issues; + if (dayIssues.length <= ISSUE_BATCH_SIZE) continue; + + dayIssues.sort((a, b) => (a.component || '').localeCompare(b.component || '')); - if (compKey !== currentFile) { - // New file — flush previous file's count and decide batch placement - if (currentFile && currentFileIssueCount > 0) { - let batch = batches[batches.length - 1]; - if (batch.issueCount + currentFileIssueCount > ISSUE_BATCH_SIZE && batch.issueCount > 0) { - batches.push({ files: new Set(), issueCount: 0 }); - batch = batches[batches.length - 1]; - } - batch.files.add(currentFile); - batch.issueCount += currentFileIssueCount; + const subBatches = groupIssuesIntoBatches(dayIssues); + const totalBatches = subBatches.length; + + for (let batchIdx = 0; batchIdx < totalBatches - 1; batchIdx++) { + const syntheticDate = group.dateMs - (totalBatches - 1 - batchIdx) * ONE_DAY_MS; + for (const issue of subBatches[batchIdx]) { + effectiveDates.set(issue.key, syntheticDate); } - currentFile = compKey; - currentFileIssueCount = 0; } - currentFileIssueCount++; + // Last sub-batch keeps the original date (no override) + + const dayStr = new Date(group.dateMs).toISOString().slice(0, 10); + logger.warn( + `${dayIssues.length} issues on ${dayStr} exceed ${ISSUE_BATCH_SIZE} cap, ` + + `sub-split into ${totalBatches} date groups` + ); } - // Flush last file - if (currentFile && currentFileIssueCount > 0) { + return effectiveDates; +} + +/** + * Group sorted issues into batches of ≤ISSUE_BATCH_SIZE without splitting files. + */ +function groupIssuesIntoBatches(sortedIssues) { + const batches = [[]]; + let currentBatchCount = 0; + let currentFile = null; + let fileBuffer = []; + + function flushFile() { + if (fileBuffer.length === 0) return; let batch = batches[batches.length - 1]; - if (batch.issueCount + currentFileIssueCount > ISSUE_BATCH_SIZE && batch.issueCount > 0) { - batches.push({ files: new Set(), issueCount: 0 }); + if (currentBatchCount + fileBuffer.length > ISSUE_BATCH_SIZE && currentBatchCount > 0) { + batches.push([]); batch = batches[batches.length - 1]; + currentBatchCount = 0; + } + batch.push(...fileBuffer); + currentBatchCount += fileBuffer.length; + fileBuffer = []; + } + + for (const issue of sortedIssues) { + if (issue.component !== currentFile) { + flushFile(); + currentFile = issue.component; } - batch.files.add(currentFile); - batch.issueCount += currentFileIssueCount; + fileBuffer.push(issue); } + flushFile(); return batches; } + +/** + * Phase 1: Build per-file, per-line date map. + * For each issue, map its line range to its effective creation date. + * Oldest date wins when lines overlap (prevents CE MAX inflation). + */ +function buildFileLineDates(issues, effectiveDates, fallbackDate) { + const fileLineDates = new Map(); + + for (const issue of issues) { + const dateMs = effectiveDates.get(issue.key) + ?? parseCreationDate(issue.creationDate, fallbackDate); + + const startLine = issue.textRange?.startLine || issue.line || 0; + const endLine = issue.textRange?.endLine || startLine; + if (startLine <= 0) continue; + + const compKey = issue.component; + if (!compKey) continue; + + if (!fileLineDates.has(compKey)) { + fileLineDates.set(compKey, new Map()); + } + const lineDateMap = fileLineDates.get(compKey); + + for (let ln = startLine; ln <= endLine; ln++) { + const existing = lineDateMap.get(ln); + if (existing === undefined || dateMs < existing) { + lineDateMap.set(ln, dateMs); + } + } + } + + return fileLineDates; +} + +function parseCreationDate(creationDateStr, fallbackDate) { + if (!creationDateStr) return fallbackDate; + const ms = new Date(creationDateStr).getTime(); + return isNaN(ms) ? fallbackDate : ms; +}