Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export function buildMetadata(ctx) {
branchName, branchType: 1,
referenceBranchName: referenceBranch,
scmRevisionId: ctx.data.metadata.scmRevisionId || generateFakeCommitHash(),
projectVersion: '1.0.0',
projectVersion: ctx.sourceProjectVersion || '1.0.0',
analyzedIndexedFileCountPerType: ctx.buildFileCountsByType(),
};
logger.debug(`Metadata built: projectKey=${metadata.projectKey}, branch=${metadata.branchName}, scmRevisionId=${metadata.scmRevisionId}`);
Expand Down
1 change: 1 addition & 0 deletions src/pipelines/sq-10.0/protobuf/builder/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ export function createProtobufBuilder(extractedData, scConfig = {}, scProfiles =
referenceBranchName: options.referenceBranchName || null,
sonarCloudRepos: options.sonarCloudRepos || new Set(),
ruleEnrichmentMap: options.ruleEnrichmentMap || new Map(),
sourceProjectVersion: options.sourceProjectVersion || null,
};
bindBuilderMethods(ctx);
return ctx;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export async function finalizeTransfer({ mainResult, sonarCloudMainBranch, syncA
if (nonMainBranches.length > 0) {
await waitForMainAnalysis(sonarCloudClient, mainResult.ceTask?.id, wait);
logger.info(`Syncing ${nonMainBranches.length} additional branch(es): ${nonMainBranches.map(b => b.name).join(', ')}`);
await transferNonMainBranches({ nonMainBranches, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, journal, cache, stateTracker, isIncremental, shutdownCheck, performanceConfig, sonarCloudRepos, ruleEnrichmentMap, aggregatedStats });
await transferNonMainBranches({ nonMainBranches, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, cache, stateTracker, isIncremental, shutdownCheck, performanceConfig, sonarCloudRepos, ruleEnrichmentMap, aggregatedStats });
} else {
logger.info('No additional branches to sync (only the main branch exists)');
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import logger from '../../../../shared/utils/logger.js';
export async function transferBranchBatched(opts) {
const { extractedData, sonarcloudConfig, sonarCloudProfiles, branchName,
referenceBranchName, sonarCloudClient, label, isMainBranch,
sonarCloudRepos, ruleEnrichmentMap } = opts;
sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion } = opts;

extractedData.issues.sort((a, b) => (a.component || '').localeCompare(b.component || ''));

Expand All @@ -29,7 +29,7 @@ export async function transferBranchBatched(opts) {
logger.info(`[${batchLabel}] Issues ${batch.startIndex + 1}-${batch.endIndex} | date=${batchDate}`);

const builder = new ProtobufBuilder(batchData, sonarcloudConfig, sonarCloudProfiles, {
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap,
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
const messages = builder.buildAll();
const encoder = new ProtobufEncoder();
Expand All @@ -38,7 +38,7 @@ export async function transferBranchBatched(opts) {
const uploader = new ReportUploader(sonarCloudClient);
const metadata = {
projectKey: sonarcloudConfig.projectKey, organization: sonarcloudConfig.organization,
version: '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
version: sourceProjectVersion || '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
};

lastCeTask = await uploader.uploadAndWait(encodedReport, metadata);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,21 @@ import { ReportUploader } from '../../sonarcloud/uploader.js';
import { computeBranchStats } from './compute-branch-stats.js';
import { transferBranchBatched } from './transfer-branch-batched.js';
import { shouldBatch, backdateChangesets } from '../../../../shared/utils/batch-distributor.js';
import { resolveSourceProjectVersion } from '../../../../shared/utils/source-version/resolve-source-project-version.js';

// -------- Transfer Single Branch --------

/**
* Build, encode, and upload a single branch report to SonarCloud.
*/
export async function transferBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, referenceBranchName, wait, sonarCloudClient, label, isMainBranch = false, sonarCloudRepos = new Set(), ruleEnrichmentMap = new Map() }) {
export async function transferBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, referenceBranchName, wait, sonarCloudClient, sonarQubeClient, label, isMainBranch = false, sonarCloudRepos = new Set(), ruleEnrichmentMap = new Map() }) {
const sourceProjectVersion = await resolveSourceProjectVersion(sonarQubeClient, sonarQubeClient?.projectKey, isMainBranch ? null : branchName);

if (shouldBatch(extractedData)) {
const ceTask = await transferBranchBatched({
extractedData, sonarcloudConfig, sonarCloudProfiles, branchName,
referenceBranchName, sonarCloudClient, label, isMainBranch,
sonarCloudRepos, ruleEnrichmentMap,
sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
return { stats: computeBranchStats(extractedData), ceTask };
}
Expand All @@ -25,7 +28,7 @@ export async function transferBranch({ extractedData, sonarcloudConfig, sonarClo

logger.info(`[${label}] Building protobuf messages...`);
const builder = new ProtobufBuilder(extractedData, sonarcloudConfig, sonarCloudProfiles, {
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap,
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
const messages = builder.buildAll();

Expand All @@ -39,7 +42,7 @@ export async function transferBranch({ extractedData, sonarcloudConfig, sonarClo
const metadata = {
projectKey: sonarcloudConfig.projectKey,
organization: sonarcloudConfig.organization,
version: '1.0.0',
version: sourceProjectVersion || '1.0.0',
...(!isMainBranch && branchName ? { branchName } : {}),
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { transferBranch } from './transfer-branch.js';

const ZERO_STATS = { issuesTransferred: 0, hotspotsTransferred: 0, componentsTransferred: 0, sourcesTransferred: 0, linesOfCode: 0 };

export async function transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, journal, sonarCloudRepos, ruleEnrichmentMap }) {
export async function transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, sonarCloudRepos, ruleEnrichmentMap }) {
const alreadyCompleted = journal?.getBranchStatus(sonarCloudMainBranch) === 'completed';

if (alreadyCompleted) {
Expand All @@ -26,7 +26,7 @@ export async function transferMainBranch({ extractedData, sonarcloudConfig, sona
const result = await transferBranch({
extractedData, sonarcloudConfig, sonarCloudProfiles,
branchName: sonarCloudMainBranch, referenceBranchName: sonarCloudMainBranch,
wait, sonarCloudClient, label: 'main', isMainBranch: true,
wait, sonarCloudClient, sonarQubeClient, label: 'main', isMainBranch: true,
sonarCloudRepos, ruleEnrichmentMap,
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import { aggregateBranchStats } from './aggregate-branch-stats.js';

// -------- Transfer Non-Main Branches --------

export async function transferNonMainBranches({ nonMainBranches, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, journal, cache, stateTracker, isIncremental, shutdownCheck, performanceConfig, sonarCloudRepos, ruleEnrichmentMap, aggregatedStats }) {
export async function transferNonMainBranches({ nonMainBranches, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, cache, stateTracker, isIncremental, shutdownCheck, performanceConfig, sonarCloudRepos, ruleEnrichmentMap, aggregatedStats }) {
const branchResults = await mapConcurrent(
nonMainBranches,
async (branch) => transferSingleNonMainBranch({ branch, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, journal, cache, stateTracker, isIncremental, shutdownCheck, sonarCloudRepos, ruleEnrichmentMap }),
async (branch) => transferSingleNonMainBranch({ branch, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, cache, stateTracker, isIncremental, shutdownCheck, sonarCloudRepos, ruleEnrichmentMap }),
{ concurrency: performanceConfig?.maxConcurrency || 4, settled: true }
);
aggregateBranchStats(aggregatedStats, branchResults);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export async function transferProject(opts) {
const sonarCloudRepos = await sonarCloudClient.getRuleRepositories();
const ruleEnrichmentMap = p.prebuiltEnrichmentMap || new Map();
checkShutdown(p.shutdownCheck);
const mainResult = await transferMainBranch({ extractedData, sonarcloudConfig: p.sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait: p.wait, sonarCloudClient, journal, sonarCloudRepos, ruleEnrichmentMap });
const mainResult = await transferMainBranch({ extractedData, sonarcloudConfig: p.sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait: p.wait, sonarCloudClient, sonarQubeClient, journal, sonarCloudRepos, ruleEnrichmentMap });
return await finalizeTransfer({ mainResult, sonarCloudMainBranch, ...p, extractedData, extractor, sonarCloudProfiles, sonarCloudClient, sonarQubeClient, journal, cache, stateTracker, sonarCloudRepos, ruleEnrichmentMap, lockFile });
} catch (error) {
if (!(error instanceof GracefulShutdownError) && journal) { await journal.markInterrupted().catch(() => {}); }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { transferBranch } from './transfer-branch.js';

// -------- Transfer Single Non-Main Branch --------

export async function transferSingleNonMainBranch({ branch, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, journal, cache, stateTracker, isIncremental, shutdownCheck, sonarCloudRepos, ruleEnrichmentMap }) {
export async function transferSingleNonMainBranch({ branch, extractedData, extractor, sonarcloudConfig, sonarCloudProfiles, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, cache, stateTracker, isIncremental, shutdownCheck, sonarCloudRepos, ruleEnrichmentMap }) {
const branchName = branch.name;
if (shutdownCheck()) return { skipped: true, branchName, reason: 'shutdown' };
if (isIncremental && stateTracker.isBranchCompleted(branchName)) {
Expand All @@ -24,7 +24,7 @@ export async function transferSingleNonMainBranch({ branch, extractedData, extra
const branchResult = await transferBranch({
extractedData: branchData, sonarcloudConfig, sonarCloudProfiles,
branchName, referenceBranchName: sonarCloudMainBranch, wait,
sonarCloudClient, label: branchName, sonarCloudRepos, ruleEnrichmentMap,
sonarCloudClient, sonarQubeClient, label: branchName, sonarCloudRepos, ruleEnrichmentMap,
});
if (journal) {
await journal.recordUpload(branchName, branchResult.ceTask?.id);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export function buildMetadata(instance) {
qprofilesPerLanguage: instance.buildQProfiles(),
branchName, branchType: 1, referenceBranchName: referenceBranch,
scmRevisionId: instance.data.metadata.scmRevisionId || instance.generateFakeCommitHash(),
projectVersion: '1.0.0',
projectVersion: instance.sourceProjectVersion || '1.0.0',
analyzedIndexedFileCountPerType: instance.buildFileCountsByType(),
};
logger.debug(`Metadata built: projectKey=${metadata.projectKey}, branch=${metadata.branchName}, scmRevisionId=${metadata.scmRevisionId}`);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export function createProtobufBuilder(extractedData, sonarCloudConfig = {}, sona
referenceBranchName: options.referenceBranchName || null,
sonarCloudRepos: options.sonarCloudRepos || new Set(),
ruleEnrichmentMap: options.ruleEnrichmentMap || new Map(),
sourceProjectVersion: options.sourceProjectVersion || null,
};

attachUtilityMethods(instance);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ export async function executeTransfer(opts) {

checkShutdown(shutdownCheck);

const mainBranchResult = await transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName: sonarCloudMainBranch, wait, sonarCloudClient, journal, sonarCloudRepos, ruleEnrichmentMap });
const mainBranchResult = await transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName: sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, journal, sonarCloudRepos, ruleEnrichmentMap });
const aggregatedStats = { issuesTransferred: mainBranchResult.stats.issuesTransferred || 0, hotspotsTransferred: mainBranchResult.stats.hotspotsTransferred || 0, componentsTransferred: mainBranchResult.stats.componentsTransferred || 0, sourcesTransferred: mainBranchResult.stats.sourcesTransferred || 0, linesOfCode: mainBranchResult.stats.linesOfCode || 0, branchesTransferred: [sonarCloudMainBranch] };

if (isIncremental) { stateTracker.markBranchCompleted(sonarCloudMainBranch); await stateTracker.save(); }
checkShutdown(shutdownCheck);

if (syncAllBranches) {
await transferNonMainBranches({ extractedData, sonarcloudConfig, sonarCloudProfiles, mainBranchResult, sonarCloudMainBranch, wait, sonarCloudClient, extractor, journal, cache, stateTracker, isIncremental, shutdownCheck, excludeBranches, includeBranches, performanceConfig, aggregatedStats, sonarCloudRepos, ruleEnrichmentMap });
await transferNonMainBranches({ extractedData, sonarcloudConfig, sonarCloudProfiles, mainBranchResult, sonarCloudMainBranch, wait, sonarCloudClient, sonarQubeClient, extractor, journal, cache, stateTracker, isIncremental, shutdownCheck, excludeBranches, includeBranches, performanceConfig, aggregatedStats, sonarCloudRepos, ruleEnrichmentMap });
}

// -------- Phase 2: Metadata Sync --------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import logger from '../../../../shared/utils/logger.js';
export async function transferBranchBatched(opts) {
const { extractedData, sonarcloudConfig, sonarCloudProfiles, branchName,
referenceBranchName, sonarCloudClient, label, isMainBranch,
sonarCloudRepos, ruleEnrichmentMap } = opts;
sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion } = opts;

extractedData.issues.sort((a, b) => (a.component || '').localeCompare(b.component || ''));

Expand All @@ -29,7 +29,7 @@ export async function transferBranchBatched(opts) {
logger.info(`[${batchLabel}] Issues ${batch.startIndex + 1}-${batch.endIndex} | date=${batchDate}`);

const builder = new ProtobufBuilder(batchData, sonarcloudConfig, sonarCloudProfiles, {
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap,
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
const messages = builder.buildAll();
const encoder = new ProtobufEncoder();
Expand All @@ -38,7 +38,7 @@ export async function transferBranchBatched(opts) {
const uploader = new ReportUploader(sonarCloudClient);
const metadata = {
projectKey: sonarcloudConfig.projectKey, organization: sonarcloudConfig.organization,
version: '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
version: sourceProjectVersion || '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
};

lastCeTask = await uploader.uploadAndWait(encodedReport, metadata);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,22 @@ import { ReportUploader } from '../../sonarcloud/uploader.js';
import { computeBranchStats } from './compute-branch-stats.js';
import { transferBranchBatched } from './transfer-branch-batched.js';
import { shouldBatch, backdateChangesets } from '../../../../shared/utils/batch-distributor.js';
import { resolveSourceProjectVersion } from '../../../../shared/utils/source-version/resolve-source-project-version.js';
import logger from '../../../../shared/utils/logger.js';

// -------- Main Logic --------

/**
* Build, encode, and upload a single branch report to SonarCloud.
*/
export async function transferBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, referenceBranchName, wait, sonarCloudClient, label, isMainBranch = false, sonarCloudRepos = new Set(), ruleEnrichmentMap = new Map() }) {
export async function transferBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, referenceBranchName, wait, sonarCloudClient, sonarQubeClient, label, isMainBranch = false, sonarCloudRepos = new Set(), ruleEnrichmentMap = new Map() }) {
const sourceProjectVersion = await resolveSourceProjectVersion(sonarQubeClient, sonarQubeClient?.projectKey, isMainBranch ? null : branchName);

if (shouldBatch(extractedData)) {
const ceTask = await transferBranchBatched({
extractedData, sonarcloudConfig, sonarCloudProfiles, branchName,
referenceBranchName, sonarCloudClient, label, isMainBranch,
sonarCloudRepos, ruleEnrichmentMap,
sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
return { stats: computeBranchStats(extractedData), ceTask };
}
Expand All @@ -25,7 +28,7 @@ export async function transferBranch({ extractedData, sonarcloudConfig, sonarClo

logger.info(`[${label}] Building protobuf messages...`);
const builder = new ProtobufBuilder(extractedData, sonarcloudConfig, sonarCloudProfiles, {
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap,
sonarCloudBranchName: branchName, referenceBranchName, sonarCloudRepos, ruleEnrichmentMap, sourceProjectVersion,
});
const messages = builder.buildAll();

Expand All @@ -38,7 +41,7 @@ export async function transferBranch({ extractedData, sonarcloudConfig, sonarClo
const uploader = new ReportUploader(sonarCloudClient);
const metadata = {
projectKey: sonarcloudConfig.projectKey, organization: sonarcloudConfig.organization,
version: '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
version: sourceProjectVersion || '1.0.0', ...(!isMainBranch && branchName ? { branchName } : {}),
};

let ceTask;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import logger from '../../../../shared/utils/logger.js';
* @param {object} opts - Transfer options
* @returns {Promise<object>} { stats, ceTask }
*/
export async function transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, wait, sonarCloudClient, journal, sonarCloudRepos, ruleEnrichmentMap }) {
export async function transferMainBranch({ extractedData, sonarcloudConfig, sonarCloudProfiles, branchName, wait, sonarCloudClient, sonarQubeClient, journal, sonarCloudRepos, ruleEnrichmentMap }) {
const mainBranchCompleted = journal?.getBranchStatus(branchName) === 'completed';
const emptyStats = { issuesTransferred: 0, hotspotsTransferred: 0, componentsTransferred: 0, sourcesTransferred: 0, linesOfCode: 0 };

Expand All @@ -33,7 +33,7 @@ export async function transferMainBranch({ extractedData, sonarcloudConfig, sona

const result = await transferBranch({
extractedData, sonarcloudConfig, sonarCloudProfiles, branchName,
referenceBranchName: branchName, wait, sonarCloudClient, label: 'main',
referenceBranchName: branchName, wait, sonarCloudClient, sonarQubeClient, label: 'main',
isMainBranch: true, sonarCloudRepos, ruleEnrichmentMap,
});

Expand Down
Loading