From 5a0498e3f356f966dda95c8f95cebe294b7b0db2 Mon Sep 17 00:00:00 2001 From: Joshua Quek Date: Sat, 25 Apr 2026 17:30:01 +0800 Subject: [PATCH 1/2] edit docs --- docs/architecture.md | 38 +++++++------ docs/bespoke-algorithms.md | 107 ++++++++++++++++++++----------------- docs/technical-details.md | 102 ++++++++++++++++------------------- docs/troubleshooting.md | 28 +++++----- 4 files changed, 138 insertions(+), 137 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 83117a9..ac145b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # šŸ—ļø Architecture - + ## šŸ“ Project Structure @@ -74,13 +74,14 @@ src/ ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ build-windows.js # Initial window partitioning ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ fetch-window.js # Fetch issues within a single window ā”‚ ā”‚ ā”‚ ā””ā”€ā”€ merge-results.js # Deduplicate and merge sliced results - ā”‚ ā”œā”€ā”€ batch-distributor/ # Issue batching for upload (5K per date bucket) - ā”‚ ā”‚ ā”œā”€ā”€ index.js # shouldBatch + createBatchExtractedData orchestrator + ā”‚ ā”œā”€ā”€ batch-distributor/ # SCM date-bucket distribution (5K per date bucket) + ā”‚ ā”‚ ā”œā”€ā”€ index.js # Re-exports all batch-distributor helpers ā”‚ ā”‚ ā”œā”€ā”€ helpers/ - ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ should-batch.js # Predicate: issues.length > 5000 - ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ compute-batch-plan.js # Returns batch descriptors with start/end indices - ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ compute-batch-date.js # Computes backdated ISO date per batch - ā”‚ ā”‚ ā”‚ ā””ā”€ā”€ create-batch-extracted-data.js # Shallow-clones extracted data with sliced issues + ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ backdate-changesets.js # Core: modifies SCM blame dates to spread issue creation dates + ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ should-batch.js # ISSUE_BATCH_SIZE constant; shouldBatch() always returns false + ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ compute-batch-plan.js # Returns batch descriptors (legacy, multi-analysis path) + ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ compute-batch-date.js # Computes backdated ISO date per batch (30-day spacing) + ā”‚ ā”‚ ā”‚ ā””ā”€ā”€ create-batch-extracted-data.js # Shallow-clones extracted data (legacy) ā”‚ ā”œā”€ā”€ issue-sync/ # Shared issue sync utilities ā”‚ ā”‚ ā”œā”€ā”€ has-manual-changes.js # Detects human-authored changes on an SQ issue ā”‚ ā”‚ ā”œā”€ā”€ fetch-sq-changelogs.js # Batch-fetches SQ changelogs concurrently @@ -130,7 +131,7 @@ sq-{version}/ ā”‚ ā””ā”€ā”€ fetch-and-sync-hotspots.js # Fetches SQ hotspots, syncs to SC ā”œā”€ā”€ transfer-branch.js # Re-export ā†’ transfer-branch/index.js ā”œā”€ā”€ transfer-branch/ -ā”‚ ā”œā”€ā”€ index.js # Orchestrates per-branch transfer; gates to batched path when issues > 5K +ā”‚ ā”œā”€ā”€ index.js # Orchestrates per-branch transfer; calls backdateChangesets() before protobuf build ā”‚ ā””ā”€ā”€ helpers/ # build-and-encode-report, upload-report, compute-branch-stats, transfer-branch-batched, ... ā”œā”€ā”€ migrate-pipeline.js # Re-export ā†’ migrate-pipeline/index.js ā”œā”€ā”€ migrate-pipeline/ @@ -248,21 +249,24 @@ sq-{version}/ **404 JS files** across the sq-10.4 pipeline, all ā‰¤50 lines. Classes converted to factory functions (`createSonarQubeClient`, `createSonarCloudClient`, `createProtobufBuilder`, `createDataExtractor`) with thin class wrappers for backward compatibility. - -### Shared Utilities ā€” Batch Distributor + +### Shared Utilities ā€” SCM Date-Bucket Distribution -The **batch-distributor** (`src/shared/utils/batch-distributor/`) is a shared utility that splits large issue sets across multiple scanner report uploads to work around SonarCloud's Elasticsearch 10K-per-date-bucket visualization limit. Without batching, branches with more than 10,000 issues on a single analysis date would have issues hidden in the SonarCloud UI. +The **batch-distributor** (`src/shared/utils/batch-distributor/`) works around SonarCloud's Elasticsearch 10K-per-date-bucket visualization limit. When a branch has more than 5K issues, `backdateChangesets()` modifies SCM changeset blame dates within a **single analysis** so the CE assigns different creation dates to different groups of issues, spreading them across multiple date buckets. -It exposes four pure-function helpers: +> **Note:** Multi-analysis batching (separate scanner report uploads) was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot ā€” issues from prior analyses not in the current one are closed. + +Key helpers: | Function | Purpose | |----------|---------| -| `shouldBatch(extractedData)` | Predicate ā€” returns `true` when `issues.length > 5000` | -| `computeBatchPlan(totalIssues)` | Returns an array of batch descriptors, each with `startIndex`, `endIndex`, `batchIndex`, and `isLast` | -| `computeBatchDate(baseDateISO, batchIndex, totalBatches)` | Computes a backdated ISO date string per batch, stepping one day back per batch from the base date | -| `createBatchExtractedData(originalData, batchDescriptor, batchDate, batchScmRevisionId)` | Shallow-clones the extracted data with a sliced issues array and overridden metadata; non-final batches strip sources, changesets, and duplications to reduce upload size | +| `backdateChangesets(extractedData)` | Core: sorts issues by file, groups files into ā‰¤5K batches, sets ALL lines of each file to the batch date. No-op when issues ā‰¤ 5K. | +| `shouldBatch(extractedData)` | Always returns `false` (multi-analysis batching disabled). Exports `ISSUE_BATCH_SIZE` constant. | +| `computeBatchDate(baseDateISO, batchIndex, totalBatches)` | Computes a backdated ISO date per batch with 30-day spacing | +| `computeBatchPlan(totalIssues)` | Legacy: returns batch descriptors (used by disabled multi-analysis path) | +| `createBatchExtractedData(...)` | Legacy: shallow-clones extracted data (used by disabled multi-analysis path) | -**Integration:** All four pipeline versions (`sq-9.9`, `sq-10.0`, `sq-10.4`, `sq-2025`) integrate via a `shouldBatch` gate in `transferBranch`. When a branch has more than 5,000 issues, `transferBranch` computes a batch plan and uploads each batch as a separate scanner report with a unique backdated analysis date and `scmRevisionId`. Only the final batch includes sources, changesets, and duplications. +**Integration:** All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function mutates `extractedData.changesets` in place. ## šŸ”„ Version Routing diff --git a/docs/bespoke-algorithms.md b/docs/bespoke-algorithms.md index 775535a..d2efc0b 100644 --- a/docs/bespoke-algorithms.md +++ b/docs/bespoke-algorithms.md @@ -190,83 +190,94 @@ CloudVoyager checkpoints migration progress so a run can be resumed after any in --- -## 5. Issue Batch Distribution (Upload-Side 10K Mitigation) +## 5. SCM Date-Bucket Distribution (Upload-Side 10K Mitigation) - - + ### Problem -SonarCloud's Elasticsearch visualization layer caps results at **10,000 issues per date bucket**. When CloudVoyager migrates a branch, the scanner report is uploaded with a single `analysis_date` (the `extractedAt` timestamp from the source SonarQube instance). If the branch carries more than 10K issues, only the first 10,000 are visible in the SonarCloud UI ā€” the rest silently disappear from the Issues tab. This is an ES index-time bucketing limitation, not an API pagination issue, so it cannot be solved client-side. +SonarCloud's Elasticsearch visualization layer caps results at **10,000 issues per date bucket**. When CloudVoyager migrates a branch, the scanner report is uploaded with a single `analysis_date`. If the branch carries more than 10K issues, only the first 10,000 are visible in the SonarCloud UI ā€” the rest silently disappear from the Issues tab. This is an ES index-time bucketing limitation, not an API pagination issue, so it cannot be solved client-side. -The batch distributor solves this by splitting the issues across multiple scanner report uploads, each assigned a **distinct analysis date**, so no single date bucket exceeds the ES cap. +### Why Multi-Analysis Batching Failed -### Algorithm +The original approach split issues across multiple scanner report uploads with distinct analysis dates. This failed because SonarCloud's CE issue tracker treats each analysis as a **complete snapshot** ā€” issues from prior analyses not present in the current one are **closed**. Only the last batch's issues survived, silently destroying all prior batches' data. -``` -transferBranch(extractedData, opts): - // --- shouldBatch gate (in each pipeline's transfer-branch.js) --- - IF shouldBatch(extractedData): // i.e. issues.length > 5000 - ceTask = transferBranchBatched(extractedData, opts) - RETURN { stats: computeBranchStats(extractedData), ceTask } +### Current Solution: Single-Analysis SCM Backdating - // Otherwise: normal single-report path - ... +All issues are uploaded in a **single analysis**. The CE assigns issue creation dates from SCM blame data, so by modifying the changeset blame dates per file, we control which date bucket each issue lands in ā€” all within one report. + +### Algorithm -transferBranchBatched(extractedData, opts): - plan = computeBatchPlan(extractedData.issues.length) - // plan = [{startIndex, endIndex, batchIndex, isLast}, ...] - // batchCount = ceil(totalIssues / 5000) +``` +backdateChangesets(extractedData): + issues = extractedData.issues + IF issues.length <= 5000: RETURN // no-op for small projects baseDate = extractedData.metadata.extractedAt - FOR EACH batch IN plan: - // Last batch gets baseDate; earlier batches are backdated - batchDate = baseDate - (plan.length - 1 - batch.batchIndex) days + // Sort issues by file so files are contiguous + issues.sort(by component key) - // Unique SCM revision prevents CE deduplication - batchScmId = randomBytes(20).toString('hex') + // Group files into batches of ā‰¤5K issues (no file splitting) + fileBatches = buildFileBatches(issues) + IF fileBatches.length <= 1: RETURN - // Shallow clone with sliced issues + overridden metadata - batchData = clone(extractedData) - batchData.issues = extractedData.issues[batch.startIndex..batch.endIndex] - batchData.metadata.extractedAt = batchDate - batchData.metadata.scmRevisionId = batchScmId + // Backdate ALL lines of each file in non-final batches + FOR EACH batch (except last): + batchDate = computeBatchDate(baseDate, batchIdx, totalBatches) + FOR EACH file IN batch: + changeset = extractedData.changesets.get(file) + changeset.changesets = [{ revision: batchRevision, author: stub, date: batchDate }] + changeset.changesetIndexByLine.fill(0) // all lines ā†’ single backdated entry - // Strip heavy payloads from non-final batches - IF NOT batch.isLast: - batchData.sources = [] - batchData.changesets = empty Map - batchData.duplications = empty Map + // Last batch keeps original dates (untouched) +``` - // Build protobuf, encode, upload ā€” MUST wait for CE completion - ceTask = buildEncodeUpload(batchData, { wait: true }) +### Critical Insight: Full-File Backdating - RETURN lastCeTask -``` +Setting only individual issue lines is **insufficient**. The CE uses the MAX of SCM dates across an issue's line range ā€” surrounding lines with the current date override any backdated line. The solution sets **every line** of each file to the batch date, ensuring the CE has no newer line to fall back on. ### Key Invariants - **Batch size = 5,000 (constant)** ā€” 50% safety margin under the 10K ES limit. -- **`shouldBatch` gate** ā€” `transferBranch` checks `issues.length > 5000` before entering the batch path; projects at or below the threshold take the normal single-report path with zero overhead. -- **Oldest batch submitted first** ā€” batch 0 gets the most-backdated date; the final (newest) batch carries the original `extractedAt` date and full project data (sources, changesets, duplications). -- **Non-final batches are lightweight** ā€” `sources`, `changesets`, and `duplications` are stripped (set to empty arrays/maps) to reduce upload size. Only the last batch carries these heavy payloads, since SonarCloud keeps only the latest analysis's source snapshots. -- **Sequential upload with wait** ā€” each batch calls `uploadAndWait` before the next begins. CE processes reports per-project sequentially; concurrent uploads would race and produce indeterminate results. -- **Unique `scmRevisionId` per batch** ā€” generated via `randomBytes(20).toString('hex')`. Without a unique revision, CE deduplicates and silently rejects subsequent batches for the same branch. -- **Stats computed from original data** ā€” `computeBranchStats` runs on the full `extractedData` (before slicing), not on any single batch, ensuring accurate totals. +- **Single analysis** ā€” all issues uploaded in one report, preserving data integrity. No CE issue tracker conflicts. +- **File-level granularity** ā€” all lines of a file share the same batch date. Files are never split across batches. +- **30-day spacing** ā€” batches are spaced 30 days apart (via `computeBatchDate`) for clear separation in the SonarCloud UI date facet. +- **Last batch untouched** ā€” the final batch keeps original SCM dates; only earlier batches are backdated. +- **`shouldBatch` always returns false** ā€” multi-analysis batching is permanently disabled. `backdateChangesets()` is called unconditionally before protobuf build; it no-ops when issues ā‰¤ 5K. + +### Example + +A branch with 31,641 issues extracted on 2026-04-23: + +| Batch | Files | Issues | SCM Blame Date | +|-------|-------|--------|----------------| +| 1/7 | Files Aā€“F | ~4,854 | Oct 2025 | +| 2/7 | Files Gā€“K | ~4,816 | Nov 2025 | +| 3/7 | Files Lā€“P | ~4,959 | Dec 2025 | +| 4/7 | Files Qā€“T | ~4,767 | Jan 2026 | +| 5/7 | Files Uā€“W | ~4,921 | Feb 2026 | +| 6/7 | Files Xā€“Y | ~4,958 | Mar 2026 | +| 7/7 | Files Z+ | ~2,366 | Apr 2026 (original) | ### Pipeline Integration -Each pipeline version (sq-9.9, sq-10.0, sq-10.4, sq-2025) has a `transfer-branch.js` that imports `shouldBatch` from the shared batch-distributor module. The flow is: +All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function is a no-op for projects with ā‰¤5K issues. ``` transfer-branch.js - ā””ā”€ā”€ shouldBatch(extractedData)? - ā”œā”€ā”€ YES ā†’ transfer-branch-batched.js (batch loop) - ā””ā”€ā”€ NO ā†’ normal single-report build ā†’ encode ā†’ upload + ā””ā”€ā”€ backdateChangesets(extractedData) // mutates SCM dates in-place + ā””ā”€ā”€ buildProtobufMessages(extractedData) + ā””ā”€ā”€ encodeAndUpload(messages) ``` -The `*-batched.js` file in each pipeline wraps that pipeline's specific `ProtobufBuilder` / `ProtobufEncoder` / `ReportUploader` classes inside the batch loop. The shared utilities (`computeBatchPlan`, `computeBatchDate`, `createBatchExtractedData`) are pipeline-agnostic. +### Implementation Files + +| File | Role | +|------|------| +| `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` | Core: sorts issues by file, groups into batches, backdates all lines per file | +| `src/shared/utils/batch-distributor/helpers/should-batch.js` | Exports `ISSUE_BATCH_SIZE` constant; `shouldBatch()` always returns false (multi-analysis disabled) | +| `src/shared/utils/batch-distributor/helpers/compute-batch-date.js` | Computes backdated ISO date per batch (30-day spacing) | ### Implementation diff --git a/docs/technical-details.md b/docs/technical-details.md index 04e2129..dedb23c 100644 --- a/docs/technical-details.md +++ b/docs/technical-details.md @@ -34,24 +34,14 @@ sequenceDiagram Note over CLI: āž•B Build & encode protobuf report - alt issues > 5,000 (batch distribution) - CLI->>CLI: shouldBatch() ā†’ true - CLI->>CLI: computeBatchPlan(totalIssues) ā†’ N batches of ā‰¤5,000 - loop For each batch (oldest first) - CLI->>CLI: computeBatchDate() ā†’ backdated analysis_date - CLI->>CLI: createBatchExtractedData() (slice issues, unique scmRevisionId) - CLI->>CLI: build protobuf messages + zip ā†’ scanner-report.zip - Note over CLI,SC: āž•C Upload & CE retry (per batch) - CLI->>SC: POST /api/ce/submit (batch scanner-report.zip) - SC-->>CLI: CE task ID ā†’ wait for completion before next batch - end - else issues ā‰¤ 5,000 (single upload) - CLI->>CLI: build Issue / ExternalIssue / AdHocRule messages - CLI->>CLI: build Measure / Component / Changeset messages - CLI->>CLI: zip ā†’ scanner-report.zip - - Note over CLI,SC: āž•C Upload & CE retry - CLI->>SC: POST /api/ce/submit (scanner-report.zip) + Note over CLI: āž•B2 SCM date-bucket distribution (if >5K issues) + CLI->>CLI: backdateChangesets(extractedData) ā€” spreads SCM blame dates across 30-day buckets + CLI->>CLI: build Issue / ExternalIssue / AdHocRule messages + CLI->>CLI: build Measure / Component / Changeset messages + CLI->>CLI: zip ā†’ scanner-report.zip + + Note over CLI,SC: āž•C Upload & CE retry + CLI->>SC: POST /api/ce/submit (scanner-report.zip) SC-->>CLI: CE task ID (or timeout ā†’ poll /api/ce/activity) end @@ -149,7 +139,7 @@ sequenceDiagram | āž•D | Issue sync | `Generate a mermaid sequence diagram showing the full issue sync pipeline in CloudVoyager including pre-filter, SC indexing wait, and changelog replay` | | āž•E | Org mapping / CSVs | `Generate a mermaid sequence diagram showing how CloudVoyager maps SonarQube projects to SonarCloud organizations and generates dry-run CSV files` | | āž•F | Quality profiles | `Generate a mermaid sequence diagram for CloudVoyager quality profile migration including backup XML, rename of built-in profiles, and diff report` | -| āž•G | Batch distribution | `Generate a mermaid sequence diagram showing how CloudVoyager batch-distributor splits large issue sets into ā‰¤5K batches with backdated analysis dates and sequential CE uploads` | +| āž•G | SCM date-bucket distribution | `Generate a mermaid sequence diagram showing how CloudVoyager backdateChangesets() modifies SCM blame dates within a single analysis to spread issues across ā‰¤5K date buckets` | ## šŸ“” Protobuf Encoding @@ -239,7 +229,7 @@ Components use a flat structure - all files are direct children of the project c The tool includes `scm_revision_id` (git commit hash) in metadata. SonarCloud uses this to detect and reject duplicate reports, enabling proper analysis history tracking. -**Batch uploads**: When issue batching is active (>5,000 issues), each batch receives a unique `scmRevisionId` generated via `randomBytes(20).toString('hex')` instead of the original git commit hash. This prevents the SonarCloud CE from deduplicating successive batch uploads as identical reports. See [Issue Batching for Upload](#-issue-batching-for-upload-5k-per-date-bucket) for details. +**SCM date-bucket distribution**: When a branch has >5,000 issues, `backdateChangesets()` modifies SCM changeset blame dates within a single analysis so the CE assigns different creation dates to different groups of issues. See [SCM Date-Bucket Distribution](#-scm-date-bucket-distribution-5k-per-date-bucket) for details. ## šŸŒæ Branch Sync @@ -269,67 +259,65 @@ The extractors handle these lower limits automatically. **metricKeys batching**: SQ 9.9 through 10.8 limits `metricKeys` to 15 per request (must batch). SQ 2025.1+ has no batching limit. - -## šŸ“¦ Issue Batching for Upload (5K Per Date Bucket) + +## šŸ“¦ SCM Date-Bucket Distribution (5K Per Date Bucket) -SonarCloud's Elasticsearch caps **visualization** at 10,000 results per date bucket. When a branch has more than 5,000 issues, submitting them all in a single scanner report (with one `analysis_date`) means only 10K are visible ā€” even though all issues exist in the database. +SonarCloud's Elasticsearch caps **visualization** at 10,000 issues per date bucket. When a branch has more than 10K issues, only the first 10,000 are visible in the UI ā€” even though all issues exist in the database. -**Solution:** `src/shared/utils/batch-distributor/` splits large issue sets into batches of up to 5,000, each submitted as a separate scanner report with a distinct `analysis_date` going backwards from today. - -### Algorithm +**Solution:** `src/shared/utils/batch-distributor/helpers/backdate-changesets.js` modifies SCM changeset blame dates within a **single analysis** so the CE assigns different creation dates to different groups of issues, spreading them across multiple date buckets. -1. **Gate** ā€” `should-batch.js`: if `extractedData.issues.length > 5000`, batching activates -2. **Plan** ā€” `compute-batch-plan.js`: `batchCount = ceil(totalIssues / 5000)`, returns `[{startIndex, endIndex, batchIndex, isLast}]` -3. **Date** ā€” `compute-batch-date.js`: last batch (newest) gets the original date; earlier batches are backdated one day each (`baseDate - (totalBatches - 1 - batchIndex)` days) -4. **Clone** ā€” `create-batch-extracted-data.js`: shallow-clones `extractedData` with sliced `issues` array, overridden `metadata.extractedAt` and unique `metadata.scmRevisionId` (via `randomBytes(20)`) -5. **Trim** ā€” Non-final batches strip `sources`, `changesets`, and `duplications` (set to `[]` / `new Map()`) to reduce upload size. `components` and `activeRules` are kept for issue resolution. -6. **Upload** ā€” Each batch is built, encoded, and uploaded sequentially with `wait: true` (CE must complete before the next batch) +> **Note:** The original multi-analysis batching approach (splitting issues across separate scanner report uploads) was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot ā€” issues from prior analyses not present in the current one are closed. Only single-analysis SCM backdating preserves all issues. -### Example +### Algorithm -A branch with 20,590 issues on 2026-05-12: +1. **Gate** ā€” if `issues.length <= 5000`, no-op (no backdating needed) +2. **Sort** ā€” issues sorted by component key so files are contiguous +3. **Group** ā€” `buildFileBatches()` walks sorted issues, grouping files into batches of ā‰¤5K issues. Files are never split across batches. +4. **Backdate** ā€” for each batch (except the last), ALL lines of each file have their changeset replaced with a single entry at the batch date. `computeBatchDate()` spaces batches 30 days apart going backwards from the extraction date. +5. **Last batch untouched** ā€” keeps original SCM dates -| Batch | Issues | analysis_date | Submitted | -|-------|--------|---------------|-----------| -| 1/5 | 1ā€“5,000 | 2026-05-08 | First (oldest) | -| 2/5 | 5,001ā€“10,000 | 2026-05-09 | | -| 3/5 | 10,001ā€“15,000 | 2026-05-10 | | -| 4/5 | 15,001ā€“20,000 | 2026-05-11 | | -| 5/5 | 20,001ā€“20,590 | 2026-05-12 | Last (newest, carries full project data) | +### Why Full-File Backdating -### Backdating Strategy - +Setting only individual issue lines is insufficient. The CE uses the **MAX** of SCM dates across an issue's line range ā€” surrounding lines with the current date override any backdated line. Setting **every line** of each file ensures the CE has no newer line to fall back on. -Each batch needs a distinct `analysis_date` so that SonarCloud's Elasticsearch distributes issues across separate date buckets (each bucket can display up to 10K results). The dates are computed by `computeBatchDate()` going **backwards from the original date**: +### Example -- The **final batch** (highest index, newest) keeps the **original `analysis_date`** from the extraction. This ensures the latest analysis in SonarCloud carries the full project data (sources, changesets, duplications) and represents the "current" state. -- **Earlier batches** are backdated by one calendar day each: `originalDate - (totalBatches - 1 - batchIndex)` days. Batch 0 (first submitted) gets the oldest date; batch N-1 (last submitted) gets the original date. +A branch with 31,641 issues extracted on 2026-04-23: -This means for 5 batches with an original date of 2026-05-12, the dates are 2026-05-08, 2026-05-09, 2026-05-10, 2026-05-11, 2026-05-12. The sequential submission (oldest first) ensures SonarCloud's analysis timeline reads chronologically. +| Batch | Issues | SCM Blame Date | CE Creation Date | +|-------|--------|----------------|-----------------| +| 1/7 | ~4,854 | Oct 2025 | Oct 2025 | +| 2/7 | ~4,816 | Nov 2025 | Nov 2025 | +| 3/7 | ~4,959 | Dec 2025 | Dec 2025 | +| 4/7 | ~4,767 | Jan 2026 | Jan 2026 | +| 5/7 | ~4,921 | Feb 2026 | Feb 2026 | +| 6/7 | ~4,958 | Mar 2026 | Mar 2026 | +| 7/7 | ~2,366 | Apr 2026 (original) | Apr 2026 | ### Design Decisions | Decision | Rationale | |----------|-----------| | Batch size = 5,000 (hardcoded) | 50% safety margin under ES 10K visualization limit | -| Submit oldest batch first | Final (newest) analysis carries full project data for UI display | -| Strip sources from non-final batches | Biggest payload reduction; components stay for issue resolution | -| Unique `scmRevisionId` per batch | Prevents CE deduplication across batches ā€” each batch uses `randomBytes(20).toString('hex')` | -| Always wait between batches | CE processes reports sequentially per project; avoids race conditions | -| Backdate one day per batch | Distributes issues across separate ES date buckets; final batch retains the real date | +| Single analysis (not multi-analysis) | CE issue tracker closes issues from prior analyses not in current one | +| Full-file line backdating | CE uses MAX of SCM dates across issue line range; partial backdating is overridden | +| 30-day spacing between batches | Clear separation in SonarCloud UI date facet | +| File-level granularity (no file splitting) | Simpler, avoids partial-file changeset complexity | +| Last batch keeps original dates | Most recent issues appear at the correct time | ### Helper Files (`src/shared/utils/batch-distributor/helpers/`) | File | Role | |------|------| -| `should-batch.js` | Predicate: `issues.length > 5000` | -| `compute-batch-plan.js` | Returns batch descriptors with start/end indices | -| `compute-batch-date.js` | Computes backdated ISO date per batch | -| `create-batch-extracted-data.js` | Shallow-clones extracted data with sliced issues + overridden metadata | +| `backdate-changesets.js` | Core: sorts issues, groups files into batches, backdates all lines per file | +| `should-batch.js` | Exports `ISSUE_BATCH_SIZE` constant; `shouldBatch()` always returns false (multi-analysis disabled) | +| `compute-batch-date.js` | Computes backdated ISO date per batch (30-day spacing) | +| `compute-batch-plan.js` | Returns batch descriptors (legacy, used by disabled multi-analysis path) | +| `create-batch-extracted-data.js` | Shallow-clones extracted data (legacy, used by disabled multi-analysis path) | ### Pipeline Integration -All 4 pipeline versions (sq-9.9, sq-10.0, sq-10.4, sq-2025) integrate batching via a `shouldBatch` gate in the `transferBranch` entry point. Each pipeline has a `*-batched.js` sibling file containing the batch loop. The batching is transparent to callers ā€” `transferBranch` automatically routes to the batched path when issues exceed 5,000. +All 6 pipeline `transfer-branch` entry points call `backdateChangesets(extractedData)` before `buildProtobufMessages()`. The function mutates `extractedData.changesets` in place and is a no-op for projects with ā‰¤5K issues. ## šŸ” Search Slicing for 10K+ Issues diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ca13f71..88d12de 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -590,7 +590,7 @@ The modern statuses (`FALSE_POSITIVE`, `ACCEPTED`, `FIXED`) do **not** exist in SonarQube's `/api/issues/search` endpoint caps results at 10,000 due to an Elasticsearch hard limit. -> **Note (v1.3+):** Large-project issue handling now has two complementary mechanisms. The **search slicer** handles retrieval of >10K issues from SonarQube by splitting the date range into windows that each stay under the 10K API limit. The **batch distributor** handles uploading >5K issues to SonarCloud by splitting them into multiple scanner reports with distinct `analysis_date` values, preventing the Elasticsearch visualization limit (10K per date bucket) from hiding issues in the UI. Together, these two features ensure that projects of any size are fully extracted from SonarQube and fully visible in SonarCloud. +> **Note (v1.3+):** Large-project issue handling now has two complementary mechanisms. The **search slicer** handles retrieval of >10K issues from SonarQube by splitting the date range into windows that each stay under the 10K API limit. The **SCM date-bucket distribution** (`backdateChangesets()`) handles uploading >5K issues to SonarCloud by modifying SCM changeset blame dates within a single analysis, so the CE assigns different creation dates to different groups of issues. This prevents the Elasticsearch visualization limit (10K per date bucket) from hiding issues in the UI. Together, these two features ensure that projects of any size are fully extracted from SonarQube and fully visible in SonarCloud. ### Error: `10001th result asked` @@ -618,31 +618,29 @@ The search slicer algorithm: --- - -## šŸ“¦ Issue Batching (Multiple Scanner Reports Per Branch) + +## šŸ“¦ SCM Date-Bucket Distribution (Large Issue Sets) + +When a branch has more than 5,000 issues, CloudVoyager automatically modifies SCM changeset blame dates so the CE assigns different creation dates to different groups of issues. This spreads issues across multiple date buckets within a **single analysis**, preventing SonarCloud's Elasticsearch visualization limit (10K per date bucket) from hiding issues. -When a branch has more than 5,000 issues, CloudVoyager automatically splits them into batches of 5,000 and submits each batch as a separate scanner report with a distinct `analysis_date`. This prevents SonarCloud's Elasticsearch visualization limit (10K per date bucket) from hiding issues. +> **This is automatic and transparent.** `backdateChangesets()` is called before every protobuf build and is a no-op for branches with ā‰¤5K issues. Users do not need to enable it or take any special action. -> **This is automatic and transparent.** Batching is a built-in feature of the upload pipeline, not a workaround. Users do not need to enable it, configure it, or take any special action. The batch distributor detects when a branch exceeds 5,000 issues and handles the splitting, date assignment, and sequential upload internally. +> **Note:** An earlier version used multi-analysis batching (separate scanner report uploads per batch). This was abandoned because SonarCloud's CE issue tracker treats each analysis as a complete snapshot ā€” issues from prior analyses not in the current one are **closed**, causing silent data loss. -### Why am I seeing multiple uploads for one branch? +### Why do I see issues spread across multiple dates? -**Expected behavior (v1.3+).** If a branch has more than 5,000 issues, CloudVoyager logs: +**Expected behavior.** If a branch has more than 5,000 issues, CloudVoyager logs: ``` -Splitting 20590 issues into 5 batches of up to 5,000 +Backdating SCM data: 31641 issues ā†’ 7 date buckets of ā‰¤5000 +Modified SCM data for 2847 files across 7 date buckets ``` -Each batch is uploaded sequentially (the tool waits for CE completion before sending the next batch). The final batch carries the original analysis date and full project data (sources, changesets, duplications). Earlier batches are backdated by one day each and carry only issues + components + active rules. - - -### Upload interrupted mid-batch - -If a transfer is interrupted between batches, re-run the same command. The checkpoint journal tracks batch progress and the branch status stays `started`, so the migration can be safely resumed. All batches for that branch will be re-uploaded from the beginning. This is safe -- each batch uses a unique `scmRevisionId`, so SonarCloud's CE will not reject them as duplicates. No data is lost or corrupted by the interruption. +Issues are grouped by file into ā‰¤5K batches. Each batch's files have their SCM blame dates set to a different month (30 days apart). The CE uses SCM blame dates to assign issue creation dates, so issues land in separate date buckets. The last batch keeps the original dates. ### Issue counts look correct in the API but not in the UI -This is the exact problem batching solves. Without batching, all issues share one `analysis_date` and SonarCloud's Elasticsearch only visualizes the first 10K per date bucket. If you migrated before v1.3 and have this problem, re-transfer the affected projects ā€” the batch distributor will split the issues automatically. +This is the exact problem SCM date-bucket distribution solves. Without it, all issues share one creation date and SonarCloud's Elasticsearch only visualizes the first 10K per date bucket. Re-transfer the affected projects ā€” `backdateChangesets()` will spread the issues automatically. ### Can I change the batch size? From d751b0c91f9734df837236976c0472db9ba514d8 Mon Sep 17 00:00:00 2001 From: Joshua Quek Date: Sat, 25 Apr 2026 17:52:36 +0800 Subject: [PATCH 2/2] feat: add regression testing system for all fixed issues (Phase 1) Matrix-based GitHub Actions workflow that tests 5 PRIORITY bug scenarios (#53, #56, #57, #70, #88, #91, #94, #98) across all 4 SonarQube versions (9.9, 10.0, 10.4, 2025.1) using ephemeral Docker containers. Architecture: - Ephemeral SQ Enterprise containers with PostgreSQL per job - SQC target keys namespaced (cv-regression-*) for isolation - Shared assertion helpers with retry/backoff for SQC API - Data enrichment scripts (comments, status changes, hotspots) - Integrated into existing regression.yml orchestrator New files: - .github/workflows/regression-bug-fixes.yml (20 matrix jobs) - .github/actions/setup-sonarqube/action.yml (ephemeral SQ setup) - test/regression/helpers/ (sqc-client, assert-utils, config tools) - test/regression/enrichment/ (3 data enrichment scripts) - test/regression/assert-*.js (5 Phase 1 assertion scripts) - test/regression/sample-projects/ (minimal JS project for small scenarios) Reviewed by 5 parallel code review agents. 10 P1s, 8 P2s, 4 P3s found and fixed before commit. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/actions/setup-sonarqube/action.yml | 120 ++++++++++++ .github/workflows/regression-bug-fixes.yml | 131 +++++++++++++ .github/workflows/regression.yml | 7 +- test/regression/assert-external-analyzers.js | 35 ++++ .../assert-github-actions-language.js | 36 ++++ .../assert-issue-sync-first-migration.js | 33 ++++ test/regression/assert-kill-and-continue.js | 41 ++++ .../assert-large-project-10k-plus.js | 55 ++++++ .../regression/enrichment/add-hotspot-data.js | 89 +++++++++ .../enrichment/add-issue-comments.js | 84 +++++++++ .../enrichment/change-issue-status.js | 86 +++++++++ test/regression/helpers/assert-utils.js | 53 ++++++ test/regression/helpers/assert-utils.test.js | 45 +++++ test/regression/helpers/cleanup-project.js | 9 + test/regression/helpers/config-reader.js | 28 +++ test/regression/helpers/patch-config.js | 13 ++ test/regression/helpers/sqc-client.js | 153 +++++++++++++++ test/regression/helpers/sqc-client.test.js | 176 ++++++++++++++++++ test/regression/sample-projects/README.md | 14 ++ .../small-js/sonar-project.properties | 5 + .../sample-projects/small-js/src/auth.js | 39 ++++ .../sample-projects/small-js/src/utils.js | 58 ++++++ 22 files changed, 1309 insertions(+), 1 deletion(-) create mode 100644 .github/actions/setup-sonarqube/action.yml create mode 100644 .github/workflows/regression-bug-fixes.yml create mode 100644 test/regression/assert-external-analyzers.js create mode 100644 test/regression/assert-github-actions-language.js create mode 100644 test/regression/assert-issue-sync-first-migration.js create mode 100644 test/regression/assert-kill-and-continue.js create mode 100644 test/regression/assert-large-project-10k-plus.js create mode 100644 test/regression/enrichment/add-hotspot-data.js create mode 100644 test/regression/enrichment/add-issue-comments.js create mode 100644 test/regression/enrichment/change-issue-status.js create mode 100644 test/regression/helpers/assert-utils.js create mode 100644 test/regression/helpers/assert-utils.test.js create mode 100644 test/regression/helpers/cleanup-project.js create mode 100644 test/regression/helpers/config-reader.js create mode 100644 test/regression/helpers/patch-config.js create mode 100644 test/regression/helpers/sqc-client.js create mode 100644 test/regression/helpers/sqc-client.test.js create mode 100644 test/regression/sample-projects/README.md create mode 100644 test/regression/sample-projects/small-js/sonar-project.properties create mode 100644 test/regression/sample-projects/small-js/src/auth.js create mode 100644 test/regression/sample-projects/small-js/src/utils.js diff --git a/.github/actions/setup-sonarqube/action.yml b/.github/actions/setup-sonarqube/action.yml new file mode 100644 index 0000000..b69fb50 --- /dev/null +++ b/.github/actions/setup-sonarqube/action.yml @@ -0,0 +1,120 @@ +name: 'Setup Ephemeral SonarQube' +description: 'Starts a SonarQube Enterprise container with PostgreSQL, restores a DB dump if available, or runs a fresh scan.' + +inputs: + sq-version: + description: 'SonarQube version (9.9, 10.0, 10.4, 2025.1)' + required: true + sq-license-key: + description: 'SonarQube Enterprise license key' + required: true + db-dump-artifact: + description: 'Name of the DB dump artifact to restore (empty = fresh scan)' + required: false + default: '' + +outputs: + sq-url: + description: 'URL of the running SonarQube instance' + value: ${{ steps.wait-healthy.outputs.sq-url }} + sq-token: + description: 'Admin token for the SonarQube instance' + value: ${{ steps.create-token.outputs.sq-token }} + +runs: + using: 'composite' + steps: + - name: Create Docker network + shell: bash + run: docker network create sq-net 2>/dev/null || true + + - name: Start PostgreSQL + shell: bash + run: | + docker run -d --name sq-postgres --network sq-net \ + -e POSTGRES_USER=sonar \ + -e POSTGRES_PASSWORD=sonar \ + -e POSTGRES_DB=sonarqube \ + postgres:15-alpine + echo "Waiting for PostgreSQL..." + for i in $(seq 1 30); do + docker exec sq-postgres pg_isready -U sonar && break + sleep 1 + done + + - name: Restore DB dump (if provided) + if: inputs.db-dump-artifact != '' + shell: bash + run: | + echo "Restoring DB dump from artifact..." + docker exec -i sq-postgres psql -U sonar -d sonarqube < /tmp/sq-dump-${{ inputs.sq-version }}.sql + echo "Verifying restore..." + ROW_COUNT=$(docker exec sq-postgres psql -U sonar -d sonarqube -t -c "SELECT count(*) FROM projects;" 2>/dev/null | tr -d ' ' || echo "0") + echo "Projects in DB: $ROW_COUNT" + if [ "$ROW_COUNT" = "0" ]; then + echo "ERROR: DB restore failed (0 projects found)" + exit 1 + fi + + - name: Start SonarQube Enterprise + shell: bash + run: | + docker run -d --name sonarqube --network sq-net \ + -e SONAR_JDBC_URL=jdbc:postgresql://sq-postgres:5432/sonarqube \ + -e SONAR_JDBC_USERNAME=sonar \ + -e SONAR_JDBC_PASSWORD=sonar \ + -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true \ + -e SONAR_WEB_JAVAADDITIONALOPTS="-Xmx2g" \ + -p 9000:9000 \ + sonarqube:${{ inputs.sq-version }}-enterprise + echo "SonarQube ${{ inputs.sq-version }} Enterprise starting..." + + - name: Wait for healthy + id: wait-healthy + shell: bash + run: | + for i in $(seq 1 120); do + HEALTH=$(curl -s http://localhost:9000/api/system/status 2>/dev/null | grep -o '"status":"[^"]*"' | head -1 || echo "") + if echo "$HEALTH" | grep -q "UP"; then + echo "SonarQube is UP" + echo "sq-url=http://localhost:9000" >> "$GITHUB_OUTPUT" + break + fi + echo " Waiting for UP... ($i/120) status: $HEALTH" + sleep 2 + done + if ! echo "$HEALTH" | grep -q "UP"; then + echo "ERROR: SonarQube did not become healthy within 240 seconds" + docker logs sonarqube --tail 50 + exit 1 + fi + + - name: Inject license key + shell: bash + run: | + echo "Injecting Enterprise license..." + RESULT=$(curl -s -u admin:admin -X POST \ + "http://localhost:9000/api/editions/set_license" \ + -d "license=${{ inputs.sq-license-key }}") + if echo "$RESULT" | grep -q '"errors"'; then + echo "WARNING: License injection may have failed: $RESULT" + else + echo "License injected successfully" + fi + + - name: Create admin token + id: create-token + shell: bash + run: | + TOKEN_RESPONSE=$(curl -s -u admin:admin \ + -X POST "http://localhost:9000/api/user_tokens/generate" \ + -d "name=regression-test-$(date +%s)" \ + -d "type=GLOBAL_ANALYSIS_TOKEN") + TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"token":"[^"]*"' | cut -d'"' -f4) + if [ -z "$TOKEN" ]; then + echo "ERROR: Failed to create SQ token" + echo "$TOKEN_RESPONSE" + exit 1 + fi + echo "sq-token=$TOKEN" >> "$GITHUB_OUTPUT" + echo "SQ admin token created" diff --git a/.github/workflows/regression-bug-fixes.yml b/.github/workflows/regression-bug-fixes.yml new file mode 100644 index 0000000..1b1480d --- /dev/null +++ b/.github/workflows/regression-bug-fixes.yml @@ -0,0 +1,131 @@ +name: Regression - Bug Fix Scenarios + +on: + workflow_call: + +concurrency: + group: regression-bug-fixes-${{ github.ref }} + cancel-in-progress: false + +jobs: + bug-fix-scenarios: + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 4 + matrix: + scenario: + - name: large-project-10k-plus + issues: '#53, #94, #98' + command: migrate + - name: github-actions-language + issues: '#88' + command: migrate + - name: external-analyzers + issues: '#56, #82' + command: migrate + - name: issue-sync-first-migration + issues: '#91' + command: migrate + - name: kill-and-continue + issues: '#15, #57' + command: migrate + sq_version: + - id: '9.9' + secret_key: '9_9' + - id: '10.0' + secret_key: '10_0' + - id: '10.4' + secret_key: '10_4' + - id: '2025.1' + secret_key: '2025_1' + + env: + SCENARIO: ${{ matrix.scenario.name }} + SQ_VERSION: ${{ matrix.sq_version.id }} + SQC_TARGET_KEY: cv-regression-${{ matrix.scenario.name }}-sq${{ matrix.sq_version.secret_key }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Restore dependencies + uses: ./.github/actions/restore-deps + + - name: Setup ephemeral SonarQube + id: sq-setup + uses: ./.github/actions/setup-sonarqube + with: + sq-version: ${{ matrix.sq_version.id }} + sq-license-key: ${{ secrets[format('SONARQUBE_{0}_LICENSE_KEY', matrix.sq_version.secret_key)] }} + + - name: Generate config + uses: ./.github/actions/generate-config + env: + SONARQUBE_URL: ${{ steps.sq-setup.outputs.sq-url }} + SONARQUBE_TOKEN: ${{ steps.sq-setup.outputs.sq-token }} + SONARCLOUD_URL: ${{ secrets.SONARCLOUD_URL }} + SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + SONARCLOUD_ORG_KEY: ${{ secrets.SONARCLOUD_ORG_KEY }} + SONARCLOUD_ENTERPRISE_KEY: ${{ secrets.SONARCLOUD_ENTERPRISE_KEY }} + + - name: Patch config with target key + shell: bash + run: node test/regression/helpers/patch-config.js "$SQC_TARGET_KEY" "${{ steps.sq-setup.outputs.sq-url }}" "${{ steps.sq-setup.outputs.sq-token }}" + + - name: Cleanup SQC target project + shell: bash + run: node test/regression/helpers/cleanup-project.js "$SQC_TARGET_KEY" + + - name: Enrich test data - Add issue comments + if: success() + run: node test/regression/enrichment/add-issue-comments.js + env: + SQ_PROJECT_KEY: angular + + - name: Enrich test data - Change issue statuses + if: success() + run: node test/regression/enrichment/change-issue-status.js + env: + SQ_PROJECT_KEY: angular + + - name: Enrich test data - Add hotspot data + if: success() + run: node test/regression/enrichment/add-hotspot-data.js + env: + SQ_PROJECT_KEY: angular + + - name: Run migration + if: matrix.scenario.name != 'kill-and-continue' + run: npm run ${{ matrix.scenario.command }} -- -c migrate-config.json --verbose + + - name: Run migration (kill-and-continue) + if: matrix.scenario.name == 'kill-and-continue' + shell: bash + run: | + timeout --signal=SIGTERM 60s npm run migrate -- -c migrate-config.json --verbose || true + echo "Resuming after SIGTERM..." + npm run migrate -- -c migrate-config.json --verbose + + - name: Assert results + run: node test/regression/assert-${{ matrix.scenario.name }}.js --config migrate-config.json + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-${{ matrix.scenario.name }}-sq${{ matrix.sq_version.secret_key }} + path: migration-output/logs/ + retention-days: 7 + + - name: Teardown containers + if: always() + shell: bash + run: | + docker stop sonarqube sq-postgres 2>/dev/null || true + docker rm sonarqube sq-postgres 2>/dev/null || true diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index fb94dfe..f864e41 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -27,6 +27,11 @@ jobs: uses: ./.github/workflows/regression-verify.yml secrets: inherit + bug-fixes: + needs: [setup, lint] + uses: ./.github/workflows/regression-bug-fixes.yml + secrets: inherit + summary: - needs: [migrate, sync-metadata, verify] + needs: [migrate, sync-metadata, verify, bug-fixes] uses: ./.github/workflows/regression-summary.yml diff --git a/test/regression/assert-external-analyzers.js b/test/regression/assert-external-analyzers.js new file mode 100644 index 0000000..23b7214 --- /dev/null +++ b/test/regression/assert-external-analyzers.js @@ -0,0 +1,35 @@ +#!/usr/bin/env node +import { SqcClient } from './helpers/sqc-client.js'; +import { pass, fail, exitWithResults, parseArgs } from './helpers/assert-utils.js'; + +const args = parseArgs(); +const client = await SqcClient.fromConfig(args.config); +const targetKey = process.env.SQC_TARGET_KEY; + +const projectExists = await client.getProjectExists(targetKey); +if (!projectExists) { + fail('#56', `Project ${targetKey} does not exist in SonarCloud.`); + exitWithResults(); +} +pass('#56', `Project ${targetKey} exists in SonarCloud.`); + +const issueCount = await client.getIssueCount(targetKey); +if (issueCount === 0) { + fail('#56', `No issues found in ${targetKey}. External analyzer issues may not have migrated.`); + exitWithResults(); +} +pass('#56', `Total issues: ${issueCount}.`); + +const rulesFacet = await client.searchIssues(targetKey, { ps: 1, facets: 'rules' }); +const ruleFacetValues = rulesFacet?.facets?.find(f => f.property === 'rules')?.values ?? []; +const externalRules = ruleFacetValues.filter(v => v.val.startsWith('external_')); + +if (externalRules.length === 0) { + const allRuleNames = ruleFacetValues.slice(0, 10).map(v => v.val).join(', '); + fail('#82', `No external analyzer rules (external_*) found in migrated issues. Top rules: ${allRuleNames}. Pylint/Ruff issues did not migrate.`); +} else { + const ruleList = externalRules.map(r => `${r.val}(${r.count})`).join(', '); + pass('#82', `External analyzer rules found: ${externalRules.length} rules. ${ruleList}.`); +} + +exitWithResults(); diff --git a/test/regression/assert-github-actions-language.js b/test/regression/assert-github-actions-language.js new file mode 100644 index 0000000..78d1716 --- /dev/null +++ b/test/regression/assert-github-actions-language.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node +import { SqcClient } from './helpers/sqc-client.js'; +import { pass, fail, exitWithResults, parseArgs } from './helpers/assert-utils.js'; + +const args = parseArgs(); +const client = await SqcClient.fromConfig(args.config); +const targetKey = process.env.SQC_TARGET_KEY; + +const projectExists = await client.getProjectExists(targetKey); +if (!projectExists) { + fail('#88', `Project ${targetKey} does not exist in SonarCloud.`); + exitWithResults(); +} +pass('#88', `Project ${targetKey} exists in SonarCloud.`); + +const issueCount = await client.getIssueCount(targetKey); +if (issueCount === 0) { + fail('#88', `No issues found in ${targetKey}. GitHub Actions language issues may not have migrated.`); + exitWithResults(); +} +pass('#88', `Total issues: ${issueCount}.`); + +const ghActionsIssues = await client.searchIssues(targetKey, { + languages: 'github-actions', + ps: 1 +}); +if (ghActionsIssues.total === 0) { + const allLanguages = await client.searchIssues(targetKey, { ps: 1, facets: 'languages' }); + const langFacet = allLanguages.facets?.find(f => f.property === 'languages'); + const languages = langFacet?.values?.map(v => `${v.val}(${v.count})`).join(', ') || 'unknown'; + fail('#88', `No GitHub Actions language issues found. Available languages: ${languages}. The language filter may be broken or the source project has no GHA issues.`); +} else { + pass('#88', `GitHub Actions language issues found: ${ghActionsIssues.total}.`); +} + +exitWithResults(); diff --git a/test/regression/assert-issue-sync-first-migration.js b/test/regression/assert-issue-sync-first-migration.js new file mode 100644 index 0000000..7c7bead --- /dev/null +++ b/test/regression/assert-issue-sync-first-migration.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { SqcClient } from './helpers/sqc-client.js'; +import { pass, fail, exitWithResults, parseArgs } from './helpers/assert-utils.js'; + +const args = parseArgs(); +const client = await SqcClient.fromConfig(args.config); +const targetKey = process.env.SQC_TARGET_KEY; + +const projectExists = await client.getProjectExists(targetKey); +if (!projectExists) { + fail('#91', `Project ${targetKey} does not exist in SonarCloud. First migration did not create it.`); + exitWithResults(); +} +pass('#91', `Project ${targetKey} exists in SonarCloud after first migration.`); + +const issueCount = await client.getIssueCount(targetKey); +if (issueCount === 0) { + fail('#91', `No issues found in ${targetKey} after first migration. Issue sync did not trigger on first run.`); +} else { + pass('#91', `Issue count after first migration: ${issueCount}. Issue sync triggered correctly.`); +} + +const hotspotCount = await client.getHotspotCount(targetKey); +pass('#91', `Hotspot count after first migration: ${hotspotCount}.`); + +const profiles = await client.getQualityProfiles(targetKey); +if (profiles.length === 0) { + fail('#91', `No quality profiles associated with ${targetKey}. Profile migration may have failed.`); +} else { + pass('#91', `Quality profiles: ${profiles.length} associated.`); +} + +exitWithResults(); diff --git a/test/regression/assert-kill-and-continue.js b/test/regression/assert-kill-and-continue.js new file mode 100644 index 0000000..e74e82f --- /dev/null +++ b/test/regression/assert-kill-and-continue.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node +import { SqcClient } from './helpers/sqc-client.js'; +import { pass, fail, exitWithResults, parseArgs } from './helpers/assert-utils.js'; + +const args = parseArgs(); +const client = await SqcClient.fromConfig(args.config); +const targetKey = process.env.SQC_TARGET_KEY; + +const projectExists = await client.getProjectExists(targetKey); +if (!projectExists) { + fail('#15', `Project ${targetKey} does not exist in SonarCloud after kill-and-resume.`); + exitWithResults(); +} +pass('#15', `Project ${targetKey} exists in SonarCloud after kill-and-resume.`); + +const totalIssues = await client.getIssueCount(targetKey); +if (totalIssues === 0) { + fail('#57', `No issues found in ${targetKey} after kill-and-resume. Resume may not have completed the migration.`); + exitWithResults(); +} +pass('#57', `Issue count after kill-and-resume: ${totalIssues}. Migration completed after resume.`); + +let allKeys = []; +let page = 1; +const pageSize = 500; +while (allKeys.length < totalIssues && page <= 100) { + const batch = await client.searchIssues(targetKey, { ps: pageSize, p: page }); + if (!batch.issues?.length) break; + allKeys.push(...batch.issues.map(i => i.key)); + page++; +} + +const uniqueKeys = new Set(allKeys); +if (allKeys.length !== uniqueKeys.size) { + const dupeCount = allKeys.length - uniqueKeys.size; + fail('#15', `Duplicate issues detected: ${allKeys.length} total keys, ${uniqueKeys.size} unique. ${dupeCount} duplicates from kill-resume.`); +} else { + pass('#15', `No duplicate issues across all ${allKeys.length} fetched keys (of ${totalIssues} total).`); +} + +exitWithResults(); diff --git a/test/regression/assert-large-project-10k-plus.js b/test/regression/assert-large-project-10k-plus.js new file mode 100644 index 0000000..df20b6d --- /dev/null +++ b/test/regression/assert-large-project-10k-plus.js @@ -0,0 +1,55 @@ +#!/usr/bin/env node +import { SqcClient } from './helpers/sqc-client.js'; +import { pass, fail, exitWithResults, parseArgs } from './helpers/assert-utils.js'; + +const MIN_EXPECTED_BUCKETS = 3; + +const args = parseArgs(); +const client = await SqcClient.fromConfig(args.config); +const targetKey = process.env.SQC_TARGET_KEY; + +const projectExists = await client.getProjectExists(targetKey); +if (!projectExists) { + fail('#53', `Project ${targetKey} does not exist in SonarCloud. Migration did not create it.`); + exitWithResults(); +} +pass('#53', `Project ${targetKey} exists in SonarCloud.`); + +const issueCount = await client.getIssueCount(targetKey); +if (issueCount === undefined || issueCount === null) { + fail('#94', `Issue count returned undefined/null. API response may be malformed.`); + exitWithResults(); +} +if (issueCount < 10000) { + fail('#94', `Expected >10,000 issues, found ${issueCount}. The 10K issue limit may have regressed.`); +} else { + pass('#94', `Issue count: ${issueCount} (>10,000 threshold met).`); +} + +const { total, dateBuckets } = await client.getIssuesByCreationDate(targetKey); +if (!dateBuckets || dateBuckets.length === 0) { + fail('#98', `No date bucket data returned from SQC API. The createdAt facet may not be available.`); + exitWithResults(); +} + +const nonEmptyBuckets = dateBuckets.filter(b => b.count > 0); +if (nonEmptyBuckets.length < MIN_EXPECTED_BUCKETS) { + fail('#98', `Issues distributed across only ${nonEmptyBuckets.length} date bucket(s). Expected at least ${MIN_EXPECTED_BUCKETS} (SCM date-bucket distribution). Total: ${total}.`); +} else { + const maxBucket = Math.max(...nonEmptyBuckets.map(b => b.count)); + if (maxBucket > 10000) { + fail('#98', `Largest date bucket has ${maxBucket} issues (>10K). SonarCloud UI will cap display. Buckets: ${nonEmptyBuckets.length}.`); + } else { + const bucketSummary = nonEmptyBuckets.map(b => `${b.val}:${b.count}`).join(', '); + pass('#98', `Issues spread across ${nonEmptyBuckets.length} date buckets. Largest: ${maxBucket}. All within 10K cap. [${bucketSummary}]`); + } +} + +const hotspotCount = await client.getHotspotCount(targetKey); +if (hotspotCount === 0) { + fail('#53-hotspots', `No hotspots found in ${targetKey}. Expected hotspot migration.`); +} else { + pass('#53-hotspots', `Hotspot count: ${hotspotCount}.`); +} + +exitWithResults(); diff --git a/test/regression/enrichment/add-hotspot-data.js b/test/regression/enrichment/add-hotspot-data.js new file mode 100644 index 0000000..d9a6091 --- /dev/null +++ b/test/regression/enrichment/add-hotspot-data.js @@ -0,0 +1,89 @@ +#!/usr/bin/env node +import { readConfig, getSqUrl, getSqToken } from '../helpers/config-reader.js'; + +async function sqApi(baseUrl, token, path, params) { + const url = new URL(`${baseUrl}${path}`); + const body = new URLSearchParams(params); + const response = await fetch(url.toString(), { + method: 'POST', + headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/x-www-form-urlencoded' }, + body + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API ${response.status} on ${path}: ${text}`); + } + return response.json().catch(() => ({})); +} + +async function sqGet(baseUrl, token, path, params = {}) { + const url = new URL(`${baseUrl}${path}`); + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)); + const response = await fetch(url.toString(), { + headers: { 'Authorization': `Bearer ${token}` } + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API GET ${response.status} on ${path}: ${text}`); + } + return response.json(); +} + +async function main() { + const configPath = process.argv[2] || undefined; + const config = await readConfig(configPath); + const baseUrl = getSqUrl(config).replace(/\/$/, ''); + const token = getSqToken(config); + + const projectKey = process.env.SQ_PROJECT_KEY || 'angular'; + + console.log(`Adding hotspot data to project ${projectKey} on ${baseUrl}`); + + const hotspots = await sqGet(baseUrl, token, '/api/hotspots/search', { + projectKey, + ps: '10' + }); + + const requireHotspots = process.env.REQUIRE_HOTSPOTS === 'true'; + if (!hotspots.hotspots?.length) { + if (requireHotspots) { + throw new Error(`No hotspots found in project ${projectKey} but REQUIRE_HOTSPOTS=true. Enrichment cannot proceed.`); + } + console.log('No hotspots found. Skipping hotspot enrichment (set REQUIRE_HOTSPOTS=true to fail on this).'); + return; + } + + let enriched = 0; + for (const hotspot of hotspots.hotspots.slice(0, 3)) { + await sqApi(baseUrl, token, '/api/hotspots/add_comment', { + hotspot: hotspot.key, + comment: 'Migration test: reviewed by security team, safe to use.' + }); + + await sqApi(baseUrl, token, '/api/hotspots/change_status', { + hotspot: hotspot.key, + status: 'REVIEWED', + resolution: 'SAFE' + }); + enriched++; + + const verify = await sqGet(baseUrl, token, '/api/hotspots/show', { hotspot: hotspot.key }); + if (verify.status !== 'REVIEWED') { + throw new Error(`Verification failed: hotspot ${hotspot.key} expected status REVIEWED, got ${verify.status}`); + } + const hasComment = verify.comment?.some?.(c => c.markdown?.includes('Migration test')) || + verify.comments?.some?.(c => c.htmlText?.includes('Migration test')) || + verify.comment?.length > 0; + if (!hasComment) { + console.warn(` WARNING: Could not verify comment on hotspot ${hotspot.key} (comment field format may vary by SQ version)`); + } + console.log(` Enriched hotspot ${hotspot.key}: status REVIEWED (verified), comment added`); + } + + console.log(`Done: ${enriched} hotspots enriched and verified.`); +} + +main().catch(err => { + console.error(`ENRICHMENT FAILED: ${err.message}`); + process.exit(1); +}); diff --git a/test/regression/enrichment/add-issue-comments.js b/test/regression/enrichment/add-issue-comments.js new file mode 100644 index 0000000..d218a8f --- /dev/null +++ b/test/regression/enrichment/add-issue-comments.js @@ -0,0 +1,84 @@ +#!/usr/bin/env node +import { readConfig, getSqUrl, getSqToken } from '../helpers/config-reader.js'; + +const SQ_COMMENTS = [ + 'Migration test: this issue was reviewed by the security team.', + 'Migration test: false positive confirmed, marking as won\'t fix.', + 'Migration test: assigned to platform-team for Q2 sprint.' +]; + +async function sqApi(baseUrl, token, path, params) { + const url = new URL(`${baseUrl}${path}`); + const body = new URLSearchParams(params); + const response = await fetch(url.toString(), { + method: 'POST', + headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/x-www-form-urlencoded' }, + body + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API ${response.status} on ${path}: ${text}`); + } + return response.json().catch(() => ({})); +} + +async function sqGet(baseUrl, token, path, params = {}) { + const url = new URL(`${baseUrl}${path}`); + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)); + const response = await fetch(url.toString(), { + headers: { 'Authorization': `Bearer ${token}` } + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API GET ${response.status} on ${path}: ${text}`); + } + return response.json(); +} + +async function main() { + const configPath = process.argv[2] || undefined; + const config = await readConfig(configPath); + const baseUrl = getSqUrl(config).replace(/\/$/, ''); + const token = getSqToken(config); + + const projectKey = process.env.SQ_PROJECT_KEY || 'angular'; + + console.log(`Adding issue comments to project ${projectKey} on ${baseUrl}`); + + const issues = await sqGet(baseUrl, token, '/api/issues/search', { + componentKeys: projectKey, + ps: '10', + statuses: 'OPEN,CONFIRMED' + }); + + if (!issues.issues?.length) { + throw new Error(`No issues found in project ${projectKey}. Enrichment cannot proceed.`); + } + + let added = 0; + for (const issue of issues.issues.slice(0, 5)) { + const comment = SQ_COMMENTS[added % SQ_COMMENTS.length]; + await sqApi(baseUrl, token, '/api/issues/add_comment', { + issue: issue.key, + text: comment + }); + added++; + + const verify = await sqGet(baseUrl, token, '/api/issues/search', { + issues: issue.key, + additionalFields: 'comments' + }); + const hasComment = verify.issues?.[0]?.comments?.some(c => c.markdown === comment); + if (!hasComment) { + throw new Error(`Verification failed: comment not found on issue ${issue.key}`); + } + console.log(` Added comment to issue ${issue.key} (verified)`); + } + + console.log(`Done: ${added} comments added and verified.`); +} + +main().catch(err => { + console.error(`ENRICHMENT FAILED: ${err.message}`); + process.exit(1); +}); diff --git a/test/regression/enrichment/change-issue-status.js b/test/regression/enrichment/change-issue-status.js new file mode 100644 index 0000000..904d8f3 --- /dev/null +++ b/test/regression/enrichment/change-issue-status.js @@ -0,0 +1,86 @@ +#!/usr/bin/env node +import { readConfig, getSqUrl, getSqToken } from '../helpers/config-reader.js'; + +const TRANSITIONS = [ + { transition: 'confirm', expectedStatus: 'CONFIRMED', expectedResolution: null }, + { transition: 'resolve', expectedStatus: 'RESOLVED', expectedResolution: 'FIXED' }, + { transition: 'falsepositive', expectedStatus: 'RESOLVED', expectedResolution: 'FALSE-POSITIVE' } +]; + +async function sqApi(baseUrl, token, path, params) { + const url = new URL(`${baseUrl}${path}`); + const body = new URLSearchParams(params); + const response = await fetch(url.toString(), { + method: 'POST', + headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/x-www-form-urlencoded' }, + body + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API ${response.status} on ${path}: ${text}`); + } + return response.json().catch(() => ({})); +} + +async function sqGet(baseUrl, token, path, params = {}) { + const url = new URL(`${baseUrl}${path}`); + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)); + const response = await fetch(url.toString(), { + headers: { 'Authorization': `Bearer ${token}` } + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`SQ API GET ${response.status} on ${path}: ${text}`); + } + return response.json(); +} + +async function main() { + const configPath = process.argv[2] || undefined; + const config = await readConfig(configPath); + const baseUrl = getSqUrl(config).replace(/\/$/, ''); + const token = getSqToken(config); + + const projectKey = process.env.SQ_PROJECT_KEY || 'angular'; + + console.log(`Changing issue statuses in project ${projectKey} on ${baseUrl}`); + + const issues = await sqGet(baseUrl, token, '/api/issues/search', { + componentKeys: projectKey, + ps: '10', + statuses: 'OPEN' + }); + + if (!issues.issues?.length) { + throw new Error(`No OPEN issues found in project ${projectKey}. Enrichment cannot proceed.`); + } + + let changed = 0; + for (let i = 0; i < Math.min(issues.issues.length, TRANSITIONS.length); i++) { + const issue = issues.issues[i]; + const { transition, expectedStatus, expectedResolution } = TRANSITIONS[i]; + + await sqApi(baseUrl, token, '/api/issues/do_transition', { + issue: issue.key, + transition + }); + changed++; + + const verify = await sqGet(baseUrl, token, '/api/issues/search', { issues: issue.key }); + const actual = verify.issues?.[0]; + if (actual?.status !== expectedStatus) { + throw new Error(`Verification failed: issue ${issue.key} expected status ${expectedStatus}, got ${actual?.status}`); + } + if (expectedResolution && actual?.resolution !== expectedResolution) { + throw new Error(`Verification failed: issue ${issue.key} expected resolution ${expectedResolution}, got ${actual?.resolution}`); + } + console.log(` Transitioned issue ${issue.key}: ${transition} ā†’ ${actual.status}/${actual.resolution || 'none'} (verified)`); + } + + console.log(`Done: ${changed} issue status changes applied and verified.`); +} + +main().catch(err => { + console.error(`ENRICHMENT FAILED: ${err.message}`); + process.exit(1); +}); diff --git a/test/regression/helpers/assert-utils.js b/test/regression/helpers/assert-utils.js new file mode 100644 index 0000000..5bda5c7 --- /dev/null +++ b/test/regression/helpers/assert-utils.js @@ -0,0 +1,53 @@ +let results = []; +let hasFailure = false; + +export function reset() { + results = []; + hasFailure = false; +} + +export function pass(issueRef, message) { + const line = `PASS: [${issueRef}] ${message}`; + console.log(line); + results.push({ status: 'PASS', issueRef, message }); +} + +export function fail(issueRef, message) { + const line = `FAIL: [${issueRef}] ${message}`; + console.error(line); + results.push({ status: 'FAIL', issueRef, message }); + hasFailure = true; +} + +export function summary() { + const passed = results.filter(r => r.status === 'PASS').length; + const failed = results.filter(r => r.status === 'FAIL').length; + console.log(`\n--- ASSERTION SUMMARY ---`); + console.log(`PASSED: ${passed} FAILED: ${failed} TOTAL: ${results.length}`); + if (hasFailure) { + console.error('\nFailed assertions:'); + results.filter(r => r.status === 'FAIL').forEach(r => { + console.error(` [${r.issueRef}] ${r.message}`); + }); + } + return { passed, failed, total: results.length, hasFailure }; +} + +export function exitWithResults() { + const { hasFailure: failed } = summary(); + process.exit(failed ? 1 : 0); +} + +export function parseArgs() { + const args = process.argv.slice(2); + const parsed = {}; + for (let i = 0; i < args.length; i++) { + if (args[i].startsWith('--') && i + 1 < args.length && !args[i + 1].startsWith('--')) { + parsed[args[i].slice(2)] = args[i + 1]; + i++; + } else if (args[i].startsWith('--')) { + parsed[args[i].slice(2)] = true; + } + } + return parsed; +} diff --git a/test/regression/helpers/assert-utils.test.js b/test/regression/helpers/assert-utils.test.js new file mode 100644 index 0000000..7ff3b2e --- /dev/null +++ b/test/regression/helpers/assert-utils.test.js @@ -0,0 +1,45 @@ +import test from 'ava'; + +test.serial('pass() and fail() record results, summary() reports them', async t => { + const mod = await import('./assert-utils.js'); + mod.reset(); + + mod.pass('#98', 'Test pass message'); + mod.fail('#53', 'Test fail message'); + + const result = mod.summary(); + t.is(result.passed, 1); + t.is(result.failed, 1); + t.is(result.total, 2); + t.true(result.hasFailure); + + mod.reset(); +}); + +test.serial('reset() clears all state', async t => { + const mod = await import('./assert-utils.js'); + mod.reset(); + + mod.pass('#1', 'first'); + mod.fail('#2', 'second'); + mod.reset(); + + const result = mod.summary(); + t.is(result.total, 0); + t.false(result.hasFailure); + + mod.reset(); +}); + +test.serial('parseArgs() extracts --key value pairs and boolean flags', async t => { + const originalArgv = process.argv; + try { + process.argv = ['node', 'script.js', '--config', 'migrate-config.json', '--verbose']; + const mod = await import('./assert-utils.js'); + const args = mod.parseArgs(); + t.is(args.config, 'migrate-config.json'); + t.is(args.verbose, true); + } finally { + process.argv = originalArgv; + } +}); diff --git a/test/regression/helpers/cleanup-project.js b/test/regression/helpers/cleanup-project.js new file mode 100644 index 0000000..923f29e --- /dev/null +++ b/test/regression/helpers/cleanup-project.js @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import { SqcClient } from './sqc-client.js'; + +const targetKey = process.argv[2]; +if (!targetKey) { console.error('Usage: cleanup-project.js '); process.exit(1); } + +const client = await SqcClient.fromConfig(); +const result = await client.deleteProject(targetKey); +console.log('Cleanup:', result.deleted ? 'deleted' : 'not found (OK for first run)'); diff --git a/test/regression/helpers/config-reader.js b/test/regression/helpers/config-reader.js new file mode 100644 index 0000000..53118ff --- /dev/null +++ b/test/regression/helpers/config-reader.js @@ -0,0 +1,28 @@ +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +export async function readConfig(configPath) { + const fullPath = configPath || resolve(process.cwd(), 'migrate-config.json'); + const raw = await readFile(fullPath, 'utf-8'); + return JSON.parse(raw); +} + +export function getSqcUrl(config) { + return config.sonarcloud?.organizations?.[0]?.url || 'https://sonarcloud.io'; +} + +export function getSqcToken(config) { + return config.sonarcloud?.organizations?.[0]?.token; +} + +export function getSqcOrgKey(config) { + return config.sonarcloud?.organizations?.[0]?.key; +} + +export function getSqUrl(config) { + return config.sonarqube?.url; +} + +export function getSqToken(config) { + return config.sonarqube?.token; +} diff --git a/test/regression/helpers/patch-config.js b/test/regression/helpers/patch-config.js new file mode 100644 index 0000000..263cb1d --- /dev/null +++ b/test/regression/helpers/patch-config.js @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import { readFile, writeFile } from 'node:fs/promises'; + +const [targetKey, sqUrl, sqToken] = process.argv.slice(2); +if (!targetKey) { console.error('Usage: patch-config.js [sqUrl] [sqToken]'); process.exit(1); } + +const config = JSON.parse(await readFile('migrate-config.json', 'utf-8')); +config.transfer = config.transfer || {}; +config.transfer.targetProjectKey = targetKey; +if (sqUrl) config.sonarqube.url = sqUrl; +if (sqToken) config.sonarqube.token = sqToken; +await writeFile('migrate-config.json', JSON.stringify(config, null, 2)); +console.log(`Config patched: target=${targetKey} sq=${config.sonarqube.url}`); diff --git a/test/regression/helpers/sqc-client.js b/test/regression/helpers/sqc-client.js new file mode 100644 index 0000000..c17eb24 --- /dev/null +++ b/test/regression/helpers/sqc-client.js @@ -0,0 +1,153 @@ +import { readConfig, getSqcUrl, getSqcToken, getSqcOrgKey } from './config-reader.js'; + +const MAX_RETRIES = 3; +const BASE_DELAY_MS = 1000; + +async function sleep(ms) { + return new Promise(r => setTimeout(r, ms)); +} + +async function fetchWithRetry(url, options, retries = MAX_RETRIES) { + for (let attempt = 0; attempt <= retries; attempt++) { + let response; + try { + response = await fetch(url, options); + } catch (networkErr) { + if (attempt < retries) { + const delay = BASE_DELAY_MS * Math.pow(2, attempt); + console.error(` Retry ${attempt + 1}/${retries}: network error on ${url} (waiting ${delay}ms): ${networkErr.message}`); + await sleep(delay); + continue; + } + throw new Error(`Network error after ${retries} retries. URL: ${url}. Last error: ${networkErr.message}`); + } + + if (response.status === 401) { + throw new Error(`SQC auth failed (401). Check SONARCLOUD_TOKEN. URL: ${url}`); + } + + if (response.status === 429 || response.status >= 500) { + if (attempt < retries) { + const delay = BASE_DELAY_MS * Math.pow(2, attempt); + console.error(` Retry ${attempt + 1}/${retries}: ${response.status} on ${url} (waiting ${delay}ms)`); + await sleep(delay); + continue; + } + throw new Error(`SQC API error ${response.status} after ${retries} retries. URL: ${url}`); + } + + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`SQC API error ${response.status}: ${body.slice(0, 500)}. URL: ${url}`); + } + + try { + return await response.json(); + } catch { + return {}; + } + } +} + +export class SqcClient { + constructor(baseUrl, token, orgKey) { + this.baseUrl = baseUrl.replace(/\/$/, ''); + this.token = token; + this.orgKey = orgKey; + this.headers = { 'Authorization': `Bearer ${token}` }; + } + + static async fromConfig(configPath) { + const config = await readConfig(configPath); + const token = getSqcToken(config); + if (!token) throw new Error('SonarCloud token not found in config. Check sonarcloud.organizations[0].token.'); + const url = getSqcUrl(config); + if (!url) throw new Error('SonarCloud URL not found in config.'); + return new SqcClient(url, token, getSqcOrgKey(config)); + } + + async get(path, params = {}) { + const url = new URL(`${this.baseUrl}${path}`); + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)); + return fetchWithRetry(url.toString(), { headers: this.headers }); + } + + async deleteProject(projectKey) { + const url = `${this.baseUrl}/api/projects/delete`; + const body = `project=${encodeURIComponent(projectKey)}`; + try { + const result = await fetchWithRetry(url, { + method: 'POST', + headers: { ...this.headers, 'Content-Type': 'application/x-www-form-urlencoded' }, + body + }); + return { deleted: true }; + } catch (err) { + if (err.message.includes('404')) return { deleted: false, reason: 'not_found' }; + throw err; + } + } + + async getIssueCount(projectKey) { + const data = await this.get('/api/issues/search', { + componentKeys: projectKey, + organization: this.orgKey, + ps: 1 + }); + return data?.total ?? 0; + } + + async getIssuesByCreationDate(projectKey) { + const data = await this.get('/api/issues/search', { + componentKeys: projectKey, + organization: this.orgKey, + ps: 1, + facets: 'createdAt' + }); + const facet = data?.facets?.find(f => f.property === 'createdAt'); + return { + total: data?.total ?? 0, + dateBuckets: facet?.values ?? [] + }; + } + + async getHotspotCount(projectKey) { + const data = await this.get('/api/hotspots/search', { + projectKey, + ps: 1 + }); + return data?.paging?.total ?? 0; + } + + async getProjectExists(projectKey) { + try { + await this.get('/api/components/show', { component: projectKey }); + return true; + } catch (e) { + if (e.message.includes('404')) return false; + throw e; + } + } + + async getProjectSettings(projectKey) { + const data = await this.get('/api/settings/values', { component: projectKey }); + return data?.settings ?? []; + } + + async getQualityProfiles(projectKey) { + const data = await this.get('/api/qualityprofiles/search', { + project: projectKey, + organization: this.orgKey + }); + return data?.profiles ?? []; + } + + async searchIssues(projectKey, params = {}) { + return this.get('/api/issues/search', { + componentKeys: projectKey, + organization: this.orgKey, + ps: 500, + ...params + }); + } +} diff --git a/test/regression/helpers/sqc-client.test.js b/test/regression/helpers/sqc-client.test.js new file mode 100644 index 0000000..02d6bf1 --- /dev/null +++ b/test/regression/helpers/sqc-client.test.js @@ -0,0 +1,176 @@ +import test from 'ava'; +import esmock from 'esmock'; +import sinon from 'sinon'; + +let originalFetch; + +test.before(() => { + originalFetch = global.fetch; +}); + +test.afterEach(() => { + sinon.restore(); + global.fetch = originalFetch; +}); + +test.serial('SqcClient.getIssueCount returns total from API response', async t => { + global.fetch = sinon.stub().resolves({ + ok: true, + status: 200, + json: async () => ({ total: 15234, issues: [] }) + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'test-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'test-token', 'test-org'); + const count = await client.getIssueCount('my-project'); + t.is(count, 15234); +}); + +test.serial('SqcClient retries on 429 then succeeds', async t => { + let callCount = 0; + global.fetch = sinon.stub().callsFake(async () => { + callCount++; + if (callCount <= 2) { + return { ok: false, status: 429, text: async () => 'rate limited' }; + } + return { ok: true, status: 200, json: async () => ({ total: 100 }) }; + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'test-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'test-token', 'test-org'); + const count = await client.getIssueCount('my-project'); + t.is(count, 100); + t.is(callCount, 3); +}); + +test.serial('SqcClient retries on 500 then succeeds', async t => { + let callCount = 0; + global.fetch = sinon.stub().callsFake(async () => { + callCount++; + if (callCount === 1) { + return { ok: false, status: 500, text: async () => 'server error' }; + } + return { ok: true, status: 200, json: async () => ({ total: 50 }) }; + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'test-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'test-token', 'test-org'); + const count = await client.getIssueCount('my-project'); + t.is(count, 50); + t.is(callCount, 2); +}); + +test.serial('SqcClient throws immediately on 401', async t => { + global.fetch = sinon.stub().resolves({ + ok: false, + status: 401, + text: async () => 'unauthorized' + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'bad-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'bad-token', 'test-org'); + const error = await t.throwsAsync(() => client.getIssueCount('my-project')); + t.regex(error.message, /401/); + t.is(global.fetch.callCount, 1); +}); + +test.serial('SqcClient retries on network error', async t => { + let callCount = 0; + global.fetch = sinon.stub().callsFake(async () => { + callCount++; + if (callCount === 1) { + throw new TypeError('fetch failed'); + } + return { ok: true, status: 200, json: async () => ({ total: 25 }) }; + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'test-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'test-token', 'test-org'); + const count = await client.getIssueCount('my-project'); + t.is(count, 25); + t.is(callCount, 2); +}); + +test.serial('SqcClient.fromConfig throws on missing token', async t => { + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => undefined, + getSqcOrgKey: () => 'test-org' + } + }); + + const error = await t.throwsAsync(() => SqcClient.fromConfig()); + t.regex(error.message, /token not found/); +}); + +test.serial('SqcClient.getIssuesByCreationDate returns buckets', async t => { + global.fetch = sinon.stub().resolves({ + ok: true, + status: 200, + json: async () => ({ + total: 31642, + facets: [{ property: 'createdAt', values: [ + { val: '2025-10-01', count: 4854 }, + { val: '2025-11-01', count: 4816 }, + { val: '2025-12-01', count: 4959 } + ]}] + }) + }); + + const { SqcClient } = await esmock('./sqc-client.js', { + './config-reader.js': { + readConfig: async () => ({}), + getSqcUrl: () => 'https://sonarcloud.io', + getSqcToken: () => 'test-token', + getSqcOrgKey: () => 'test-org' + } + }); + + const client = new SqcClient('https://sonarcloud.io', 'test-token', 'test-org'); + const { total, dateBuckets } = await client.getIssuesByCreationDate('my-project'); + t.is(total, 31642); + t.is(dateBuckets.length, 3); + t.is(dateBuckets[0].count, 4854); +}); diff --git a/test/regression/sample-projects/README.md b/test/regression/sample-projects/README.md new file mode 100644 index 0000000..b2d821a --- /dev/null +++ b/test/regression/sample-projects/README.md @@ -0,0 +1,14 @@ +# Sample Projects for Regression Testing + +These minimal codebases are scanned by sonar-scanner inside ephemeral SQ containers +to seed test data for regression scenarios that don't need the full Angular codebase. + +## small-js/ +A tiny JavaScript project (~50 files) that generates a manageable number of issues. +Used for: issue-sync-first-migration, kill-and-continue, and other scenarios where +the test is about CloudVoyager behavior, not issue volume. + +## How it works +The regression workflow's setup phase scans these projects into the ephemeral SQ +container, then enrichment scripts add comments, status changes, and hotspot data +before the migration runs. diff --git a/test/regression/sample-projects/small-js/sonar-project.properties b/test/regression/sample-projects/small-js/sonar-project.properties new file mode 100644 index 0000000..f5c092e --- /dev/null +++ b/test/regression/sample-projects/small-js/sonar-project.properties @@ -0,0 +1,5 @@ +sonar.projectKey=cv-test-small-js +sonar.projectName=CloudVoyager Test - Small JS +sonar.sources=src +sonar.language=js +sonar.sourceEncoding=UTF-8 diff --git a/test/regression/sample-projects/small-js/src/auth.js b/test/regression/sample-projects/small-js/src/auth.js new file mode 100644 index 0000000..31a4310 --- /dev/null +++ b/test/regression/sample-projects/small-js/src/auth.js @@ -0,0 +1,39 @@ +// More intentional code smells for issue generation +var crypto = require('crypto'); + +function hashPassword(password) { + return crypto.createHash('md5').update(password).digest('hex'); // weak hash +} + +function generateToken() { + return Math.random().toString(36); // insecure random +} + +function validateInput(input) { + if (input == undefined) return false; // == instead of === + if (input == '') return false; + if (input.length > 0) { + return true; + } +} + +function hardcodedSecret() { + var apiKey = 'sk-1234567890abcdef'; // hardcoded secret + var dbPassword = 'admin123'; // hardcoded password + return { apiKey, dbPassword }; +} + +function xssVulnerable(userHtml) { + document.innerHTML = userHtml; // XSS vulnerability +} + +function noReturnPath(x) { + if (x > 0) { + return 'positive'; + } else if (x < 0) { + return 'negative'; + } + // missing return for x === 0 +} + +module.exports = { hashPassword, generateToken, validateInput, hardcodedSecret, xssVulnerable, noReturnPath }; diff --git a/test/regression/sample-projects/small-js/src/utils.js b/test/regression/sample-projects/small-js/src/utils.js new file mode 100644 index 0000000..33eb3bc --- /dev/null +++ b/test/regression/sample-projects/small-js/src/utils.js @@ -0,0 +1,58 @@ +// Intentional code smells for SonarQube to flag as issues +function processData(data) { + var result = []; // var instead of let/const + for (var i = 0; i < data.length; i++) { + if (data[i] != null) { // == instead of === + result.push(data[i]); + } + } + return result; +} + +function handleError(err) { + console.log(err); // console.log in production code + return null; // swallowing error +} + +function duplicateLogic(items) { + var output = []; + for (var i = 0; i < items.length; i++) { + if (items[i].active == true) { // == instead of === + output.push(items[i].name); + } + } + return output; +} + +function unusedFunction() { + var x = 1; + var y = 2; + return x; + return y; // dead code +} + +function complexFunction(a, b, c, d, e) { + if (a) { + if (b) { + if (c) { + if (d) { + if (e) { + return true; + } + } + } + } + } + return false; +} + +function sqlBuilder(userInput) { + var query = "SELECT * FROM users WHERE name = '" + userInput + "'"; // SQL injection + return query; +} + +function evalUsage(code) { + return eval(code); // eval usage +} + +module.exports = { processData, handleError, duplicateLogic, unusedFunction, complexFunction, sqlBuilder, evalUsage };