From b20364b503b0422e9aaa00e50733b72cbfc0ea69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Thu, 16 Jul 2026 01:12:42 +0200 Subject: [PATCH 1/5] chore(pipeline): cleanup before splits introduction --- CONTRIBUTING.md | 25 +++ Jenkinsfile | 444 +++++++++++++++++++++++++----------------------- 2 files changed, 260 insertions(+), 209 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed88c86f9..7c2296194 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,6 +106,11 @@ if you have switched the version in `bom-weekly/pom.xml` to a `*-SNAPSHOT`. To minimize cloud resources, PCT is not run at all by default on pull requests, only some basic sanity checks. +> [!TIP] +> If you need to restart a build without any change and if you can (ex: you're a maintainer), +> prefer using replays and reruns from GitHub over empty commits to benefit from the prep archive built and saved in the previous builds. +> It saves about half an hour of total build time, and resources. + ### Running weekly tests Add the label `weekly-test` to run the tests against the latest weekly Jenkins version - This is what you want most of the time. @@ -158,6 +163,26 @@ git commit -m 'Run limited plugin set' Keep the PR in draft until tests pass and this file can be deleted. +### Replay any specific case + +If you want to test specific cases without ever changing labels or marker files, you can update the `flags` map on top of the pipeline in replays. + +Ex, to simulate a `full-test` marker with a `weekly-test` & `limited-plugin-set` labels, update this at the top of the pipeline: + +```diff +// Test flags depending on the presence of corresponding labels or marker files +// Can be modified to test specific cases independently of the current PR labels or markers +// Possible value(s): 'label', 'marker' +Map flags = [ + - 'weekly-test': [] as Set, ++ 'weekly-test': ['label'] as Set, +- 'full-test': [] as Set, ++ 'full-test': ['marker'] as Set, +- 'limited-plugin-set': [] as Set, ++ 'limited-plugin-set': ['label', 'marker'] as Set, +] +``` + ## LTS lines A separate BOM artifact is available for the latest weekly, current LTS line and a few historical lines. diff --git a/Jenkinsfile b/Jenkinsfile index f6b1b20a0..ae89ca4cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,18 +4,18 @@ if(env.BRANCH_NAME == "master") { cronTrigger = '0 15 * * 5' } +// === Actionable in replay env.MAVEN_NTP = true - -def fullTestLabel -def weeklyTestLabel -def limitedPluginSetLabel -if (env.CHANGE_ID) { - fullTestLabel = pullRequest.labels.contains('full-test') - weeklyTestLabel = pullRequest.labels.contains('weekly-test') - limitedPluginSetLabel = pullRequest.labels.contains('limited-plugin-set') -} - -def fixedPrepArchiveName = '' // can be set to a specific prep archive name in case last commits aren't impacting it +// Can be set to a specific prep archive name in case last commits aren't impacting it +final String fixedPrepArchiveName = '' +// Test flags depending on the presence of corresponding labels or marker files +// Can be modified to test specific cases independently of the current PR labels or markers +// Possible value(s): 'label', 'marker' +Map flags = [ + 'weekly-test': [] as Set, + 'full-test': [] as Set, + 'limited-plugin-set': [] as Set, +] properties([ disableConcurrentBuilds(abortPrevious: true), @@ -28,11 +28,20 @@ if (env.BRANCH_NAME == 'master' && currentBuild.buildCauses*._class == ['jenkins error 'No longer running builds on response to master branch pushes. If you wish to cut a release, use “Re-run checks” from this failing check in https://github.com/jenkinsci/bom/commits/master' } -def mavenEnv(Map params = [:], Closure body) { - def attempt = 0 - def attempts = 6 +// Collect flags from labels +if (env.CHANGE_ID) { + flags.each { name, sources -> + if (pullRequest.labels.contains(name)) { + sources << 'label' + } + } +} + +void mavenEnv(Map params = [:], Closure body) { + int attempt = 0 + final int attempts = 6 retry(count: attempts, conditions: [kubernetesAgent(handleNonKubernetes: true), nonresumable()]) { - echo 'Attempt ' + ++attempt + ' of ' + attempts + echo '[INFO] Attempt ' + ++attempt + ' of ' + attempts // no Dockerized tests; https://github.com/jenkins-infra/documentation/blob/master/ci.adoc#container-agents node('maven-bom') { timeout(120) { @@ -45,7 +54,6 @@ def mavenEnv(Map params = [:], Closure body) { "CURRENT_ATTEMPT=${attempt}", ]) { infra.loadMavenLocalCacheIfAny(env.MVN_LOCAL_REPO) - body() } } @@ -54,23 +62,14 @@ def mavenEnv(Map params = [:], Closure body) { } } -@NonCPS -def parsePlugins(plugins) { - def pluginsByRepository = [:] - plugins.each { plugin -> - def splits = plugin.split('\t') - pluginsByRepository[splits[0].split('/')[1]] = splits[1].split(',') - } - pluginsByRepository -} +String commitId +int prepFoundInBuildNumber = 0 +Map pluginsByRepository = [:] +List lines = [] +List newestAndOldestLines = [] +Map results = [:] -def commitId -def pluginsByRepository -def lines -def fullTestMarkerFile -def weeklyTestMarkerFile -def limitedPluginSetMarkerFile -def limitedPluginSet = [ +final String[] limitedPluginSet = [ 'jenkinsci/aws-credentials-plugin aws-credentials', 'jenkinsci/aws-global-configuration-plugin aws-global-configuration', 'jenkinsci/azure-credentials-plugin azure-credentials', @@ -80,32 +79,40 @@ def limitedPluginSet = [ 'jenkinsci/badge-plugin badge', 'jenkinsci/basic-branch-build-strategies-plugin basic-branch-build-strategies', 'jenkinsci/cron_column-plugin cron_column', - 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', + 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', // longer than the others, multiple plugins ] -def results = [:] mavenEnv(jdk: 21) { - def scmVars = checkout scm - commitId = scmVars.GIT_COMMIT - - fullTestMarkerFile = fileExists 'full-test' - weeklyTestMarkerFile = fileExists 'weekly-test' - limitedPluginSetMarkerFile = fileExists 'limited-plugin-set' + String prepArchiveName + stage('init') { + Map scmVars = checkout scm - // Ensure prep archive corresponds to the current state - def prepArchiveName = "bom-prep-${commitId}.tar.gz" - def prepFoundInBuildNumber = 0 - - stage('retrieve prep archive') { + // Ensure prep archive corresponds to the current state + commitId = scmVars.GIT_COMMIT.substring(0, 7) + prepArchiveName = "bom-prep-${commitId}.tar.gz" if (fixedPrepArchiveName) { + echo "[WARNING] Using fixed prep archive name ${fixedPrepArchiveName} instead of ${prepArchiveName}" prepArchiveName = fixedPrepArchiveName - echo "[WARNING] Using fixed prep archive name ${fixedPrepArchiveName} instead of bom-prep-${commitId}.tar.gz" + } else { + echo "[INFO] Using prep archive name ${prepArchiveName}" } - prepFoundInBuildNumber = copyArtifactsFromAnyPreviousBuild(prepArchiveName, env.JOB_NAME) - if (prepFoundInBuildNumber == 0) { - catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { - error("[INFO] ${prepArchiveName} not found") + + // Collect flags from marker files + flags.each { name, sources -> + if (fileExists(name)) { + sources << 'marker' } + } + echo '[INFO] Flags:\n' + flags.collect { name, conditions -> + final String desc = conditions ? conditions.join(' & ') : 'none' + " - ${name.padRight(20)} : ${desc}" + }.join('\n') + } + + stage('retrieve prep archive') { + prepFoundInBuildNumber = retrieveArtifactsFromPreviousBuilds(prepArchiveName, env.JOB_NAME) + if (prepFoundInBuildNumber == 0) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error("[SKIP] ${prepArchiveName} not found") } return } } @@ -138,93 +145,96 @@ mavenEnv(jdk: 21) { stage('parse prep') { dir('target') { - def plugins = [] - def allLines = [] - def from = 'plugins.txt' - if (limitedPluginSetLabel || limitedPluginSetMarkerFile) { + String[] plugins = [] + String[] allLines = [] + String from = 'plugins.txt' + + if (flagEnabled(flags, 'limited-plugin-set')) { from = 'a limited set of plugins' echo('[WARNING] Running on a limited set of plugins') - // Limited set - plugins = limitedPluginSet - if (limitedPluginSetMarkerFile) { - plugins = readFile('../limited-plugin-set').split('\n') - } + // Limited set from marker file if it exists + plugins = fileExists('../limited-plugin-set') ? readFile('../limited-plugin-set').readLines() : limitedPluginSet // Lines from sample-plugin - allLines = sh ( - script: ''' - echo "weekly $(grep -F '.x' ../sample-plugin/pom.xml | sed -E 's, *(.+),\\1,g' | sort -rn | xargs)" - ''', - returnStdout: true - ).trim().split(' ') + allLines = sh (returnStdout: true, script: ''' + echo "weekly $(grep -F '.x' ../sample-plugin/pom.xml | sed -E 's, *(.+),\\1,g' | sort -rn | xargs)" + ''').trim().split(' ') } else { - plugins = readFile('plugins.txt').split('\n') - allLines = readFile('lines.txt').split('\n') + plugins = readFile('plugins.txt').readLines() + allLines = readFile('lines.txt').readLines() } + pluginsByRepository = parsePlugins(plugins) echo "[INFO] ${pluginsByRepository.size()} repositories retrieved from ${from}" echo "[INFO] List of repositories and their plugins:\n${plugins.join('\n')}" - newestAndOldestLines = [allLines[0], allLines[-1]] // Save resources by running PCT only on newest and oldest lines echo "[INFO] ${allLines.size()} lines retrieved from lines.txt: ${allLines.join(' ')} " // For archival, keeping track of newest and oldest lines as PR labels may change accross builds - // For stashes, we only care about the lines of the current build + // For stashes, we only care about the final lines of the current build + newestAndOldestLines = [allLines.first(), allLines.last()] // Save resources by running PCT only on newest and oldest lines lines = newestAndOldestLines - if (weeklyTestMarkerFile || weeklyTestLabel) { - echo "[INFO] Keeping only 'weekly' line as there is a 'weekly-test' label or marker file" + echo "[INFO] Keeping only newest and oldest lines to save resources: ${lines.join(' ')} " + if (flagEnabled(flags, 'weekly-test')) { lines = ['weekly'] - } else { - echo "[INFO] Keeping only newest and oldest lines to save resources: ${lines.join(' ')} " + echo "[WARNING] Keeping only 'weekly' line as there is a 'weekly-test' label or marker file" + } + if (BRANCH_NAME != 'master' && !(flagEnabled(flags, 'full-test') || flagEnabled(flags, 'weekly-test'))) { + lines = [] + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { + error('[SKIP] Removing all lines, build not from master or running without any "weekly-test" / "full-test" flags') + } + return } } } stage('archive new prep') { - if (prepFoundInBuildNumber == 0) { - def prepArchiveGlob = 'pct.sh excludes.txt bom-*/excludes.txt target/pct.jar target/plugins.txt target/lines.txt' - // Both newest and oldest lines in the prep archive, in case labels change on PR accross builds - // ex: from weekly-test to full-test - newestAndOldestLines.each { line -> - prepArchiveGlob += " target/megawar-${line}.war" - } - withEnv(["ARCHIVE_NAME=${prepArchiveName}", "ARCHIVE_GLOB=${prepArchiveGlob}",]) { - sh 'tar czfv "${ARCHIVE_NAME}" ${ARCHIVE_GLOB}' - archiveArtifacts artifacts: prepArchiveName, fingerprint: true - echo "[INFO] New ${prepArchiveName} archived" - } - } else { - catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { - error("[INFO] No new prep to archive") - } + if (prepFoundInBuildNumber > 0) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error("[SKIP] No new prep to archive") } return } + + String prepArchiveGlob = 'pct.sh excludes.txt bom-*/excludes.txt target/pct.jar target/plugins.txt target/lines.txt' + // Both newest and oldest lines in the prep archive, in case labels change on PR accross builds + // ex: from weekly-test to full-test + newestAndOldestLines.each { line -> + prepArchiveGlob += " target/megawar-${line}.war" + } + withEnv(["ARCHIVE_NAME=${prepArchiveName}", "ARCHIVE_GLOB=${prepArchiveGlob}",]) { + sh 'tar czfv "${ARCHIVE_NAME}" ${ARCHIVE_GLOB}' + archiveArtifacts artifacts: prepArchiveName, fingerprint: true + echo "[INFO] New ${prepArchiveName} archived" + } } stage('stash prep lines') { - if (lines.size() > 0) { - lines.each { line -> - stash name: line, includes: "pct.sh,excludes.txt,bom-*/excludes.txt,target/pct.jar,target/megawar-${line}.war" - } - } else { - catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { - error('[INFO] No line to stash') - } + if (lines.isEmpty()) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No line to stash') } return } + + echo "[INFO] Stashing ${lines.join(' & ')}" + lines.each { line -> + stash name: line, includes: "pct.sh,excludes.txt,bom-*/excludes.txt,target/pct.jar,target/megawar-${line}.war" + } } } -if (BRANCH_NAME == 'master' || fullTestMarkerFile || weeklyTestMarkerFile || env.CHANGE_ID && (fullTestLabel || weeklyTestLabel)) { - def branches = [failFast: false] - lines.each {line -> - if (line != 'weekly' && (weeklyTestMarkerFile || env.CHANGE_ID && weeklyTestLabel)) { - return +stage('run pct') { + if (lines.isEmpty()) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { + error('[SKIP] No line to run') } + return + } + + Map branches = [failFast: false] + lines.each {line -> pluginsByRepository.each { repository, plugins -> - def branchName = "${repository}:${line}" + final String branchName = "${repository}:${line}" branches[branchName] = { - def jdk = line == 'weekly' || line == '2.555.x' ? 21 : 17 + final int jdk = line == 'weekly' ? 21 : 17 withChecks(name: 'Tests', includeStage: true) { mavenEnv(jdk: jdk) { unstash line @@ -233,8 +243,8 @@ if (BRANCH_NAME == 'master' || fullTestMarkerFile || weeklyTestMarkerFile || env "LINE=$line", 'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1' ]) { - def start = System.currentTimeMillis() - def currentAttempt = env.CURRENT_ATTEMPT.toInteger() + final long start = System.currentTimeMillis() + int currentAttempt = env.CURRENT_ATTEMPT.toInteger() echo "[INFO] Current attempt: ${currentAttempt}" try { @@ -249,22 +259,30 @@ if (BRANCH_NAME == 'master' || fullTestMarkerFile || weeklyTestMarkerFile || env throw e } } finally { - def elapsed = System.currentTimeMillis() - start + final double elapsed = (System.currentTimeMillis() - start) / 1000.0 def junitResults try { junitResults = junit(testResults: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml') } catch(e) { - echo "error junitResult: ${e}" + echo "[WARNING] Error junitResult: ${e}" } - results[branchName] = getResultFromJunit(junitResults) - results[branchName]['elapsed'] = (elapsed / 1000.0) - results[branchName]['plugins'] = plugins - results[branchName]['pluginCount'] = plugins.size() - results[branchName]['attempt'] = currentAttempt - results[branchName]['build_id'] = env.BUILD_ID - results[branchName]['job_base_name'] = env.JOB_BASE_NAME - results[branchName]['short_commit_id'] = commitId.substring(0, 7) - echo "[INFO] results[${branchName}]: ${results[branchName]}" + Map result = [ + failCount : junitResults?.failCount ?: 0, + skipCount : junitResults?.skipCount ?: 0, + passCount : junitResults?.passCount ?: 0, + totalCount: junitResults?.totalCount ?: 0, + duration : junitResults?.duration ?: 0, + ] + result.elapsed = elapsed + result.plugins = plugins.join(',') + result.pluginCount = plugins.size() + result.attempt = currentAttempt + result.build_id = env.BUILD_ID + result.job_base_name = env.JOB_BASE_NAME + result.short_commit_id = commitId.substring(0, 7) + + results[branchName] = result + echo "[INFO] results for ${branchName}: ${result}" } } } @@ -276,117 +294,125 @@ if (BRANCH_NAME == 'master' || fullTestMarkerFile || weeklyTestMarkerFile || env } stage('report results') { - if (results.size() == 0) { - catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { - error('[INFO] No result to report') - } + if (results.isEmpty()) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No result to report') } return - } else { - node('maven-bom') { - Double totalElapsed = 0 - Double totalCount = 0 - Double totalSkipCount = 0 - Double totalFailCount = 0 - def reportLines = '' - results.each { combination, result -> - totalElapsed += result['elapsed'] - totalCount += result['totalCount'] - totalSkipCount += result['skipCount'] - totalFailCount += result['failCount'] - def normalizedCombination = combination.replaceAll('-', '_').replaceAll(':', '_').replaceAll('\\.', '_') - reportLines += '\n' - } - if (reportLines) { - def xmlReport = """ - - ${reportLines} - - """ - - def txtReport = results.collect { combination, result -> - 'name=' + combination + ';' + result.collect { key, value -> key + '=' + value }.join(';') - }.sort().join('\n') - - writeFile file: 'bom-report.xml', text: xmlReport - writeFile file: 'bom-report.txt', text: txtReport - archiveArtifacts artifacts: 'bom-report.*' - junit allowEmptyResults: true, testResults: 'bom-report.xml' - } + } + + node('maven-bom') { + Map totals = results.values().inject([ + elapsed : 0d, + totalCount: 0, + skipCount : 0, + failCount : 0 + ]) { acc, r -> + acc.elapsed += r.elapsed + acc.totalCount += r.totalCount + acc.skipCount += r.skipCount + acc.failCount += r.failCount + acc } + totals.resultsCount = results.size() + + final String reportLines = results.collect { combination, result -> + final String normalized = combination.replaceAll('[-:.]', '_') + """""" + }.join('\n') + + final String xmlReport = """ + + ${reportLines} + + """ + + final String txtReport = results.collect { combination, result -> + "name=${combination};" + result.collect { k, v -> "${k}=${v}" }.join(';') + }.sort().join('\n') + + writeFile file: 'bom-report.xml', text: xmlReport + writeFile file: 'bom-report.txt', text: txtReport + archiveArtifacts artifacts: 'bom-report.*' + junit allowEmptyResults: true, testResults: 'bom-report.xml' + + echo "[INFO] Aggregates from ${totals.resultsCount} result(s):\n${totals}" } } -stage('checks') { - if (fullTestMarkerFile) { - error 'Remember to `git rm full-test` before taking out of draft' +stage('flag checks') { + // Mark build as failed on any marker file + def markerErrors = flags.findAll { flag, sources -> 'marker' in sources }.keySet() + if (!markerErrors.isEmpty()) { + error "Remember to `git rm ${markerErrors.join(' ')}` before taking out of draft" } - if (limitedPluginSetMarkerFile) { - error 'Remember to `git rm limited-plugin-set` before taking out of draft' + + // Mark build as unstable on PR with limited-plugin-set + if (flagEnabled(flags, 'limited-plugin-set', 'label')) { + unstable 'Remember to remove `limited-plugin-set` label before taking out of draft' } - if (limitedPluginSetLabel) { - error 'Remember to remove `limited-plugin-set` label before taking out of draft' +} + +stage('publish incrementals') { + if (prepFoundInBuildNumber > 0) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No new prep to publish to incrementals') } + return } + infra.maybePublishIncrementals() } -infra.maybePublishIncrementals() +// === Helper functions +@NonCPS +Map parsePlugins(plugins) { + Map pluginsByRepository = [:] + plugins.each { plugin -> + String[] splits = plugin.split('\t') + pluginsByRepository[splits[0].split('/')[1]] = splits[1].split(',') + } + pluginsByRepository +} -// === Helper functions +// Return if a test flag is set +// If only a flag is passed, return true if there is a corresponding label and/or marker file +// If a flag and a source like 'marker' or 'label' are passed, return true if there is that source +boolean flagEnabled(Map flags, String flag, String source = null) { + source ? source in flags[flag] : !flags[flag].isEmpty() +} // Search and copy an artifact from builds of a job // Returns the build number where it has been found, zero otherwise -def copyArtifactsFromAnyPreviousBuild(archiveName, jobName) { - def foundInBuildNumber = 0 - def archiveExists = false - def buildNumber = env.BUILD_NUMBER.toInteger() +int retrieveArtifactsFromPreviousBuilds(String archiveName, String jobName) { + int foundInBuildNumber = 0 + boolean archiveExists = false + final int buildNumber = env.BUILD_NUMBER.toInteger() if (buildNumber == 1) { echo "[INFO] First build of ${jobName}, no ${archiveName} available yet" - } else { - // Loop over builds to retrieve the prep archive as previous build can have (only) other archive(s) - def checkBuildNumber = buildNumber - 1 - // Don't loop until the first build of master '^^ - def limit = jobName.endsWith('master') ? buildNumber - 50 : 0 - while (!archiveExists && checkBuildNumber > limit) { - echo "[INFO] Trying to retrieve ${archiveName} from ${jobName}#${checkBuildNumber}..." - try { - copyArtifacts(projectName: jobName, - selector: specific("${checkBuildNumber}"), - filter: archiveName, - fingerprintArtifacts: true, - optional: false, - ) - archiveExists = true - } catch(e) {} - if (!archiveExists) { - checkBuildNumber = checkBuildNumber - 1 - } - } + return 0 + } + + // Loop over builds to retrieve the prep archive as previous build can have (only) other archive(s) + int checkBuildNumber = buildNumber - 1 + // Don't loop until the first build of master '^^ + final int limit = jobName.endsWith('master') ? buildNumber - 50 : 0 + while (!archiveExists && checkBuildNumber > limit) { + echo "[INFO] Trying to retrieve ${archiveName} from ${jobName}#${checkBuildNumber}..." + try { + copyArtifacts(projectName: jobName, + selector: specific("${checkBuildNumber}"), + filter: archiveName, + fingerprintArtifacts: true, + optional: false, + ) + archiveExists = true + } catch(e) {} if (!archiveExists) { - echo "[INFO] No ${archiveName} found in any build of ${jobName}" - } else { - foundInBuildNumber = checkBuildNumber - echo "[INFO] ${archiveName} found in ${jobName}#${checkBuildNumber}" + checkBuildNumber = checkBuildNumber - 1 } } - return foundInBuildNumber -} - -@NonCPS -def getResultFromJunit(junitResults) { - if (!junitResults) { - return [ - failCount: 0, - skipCount: 0, - passCount: 0, - totalCount: 0, - duration: 0, - ] + if (!archiveExists) { + echo "[INFO] No ${archiveName} found in any build of ${jobName}" + } else { + foundInBuildNumber = checkBuildNumber + echo "[INFO] ${archiveName} found in ${jobName}#${checkBuildNumber}" } - return [ - failCount: junitResults.failCount ?: 0, - skipCount: junitResults.skipCount ?: 0, - passCount: junitResults.passCount ?: 0, - totalCount: junitResults.totalCount ?: 0, - duration: junitResults.duration ?: 0, - ] + return foundInBuildNumber } From 140d489d806ed29560884c01be30acdf96069fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Jul 2026 03:08:27 +0200 Subject: [PATCH 2/5] debug: test https://github.com/jenkins-infra/pipeline-library/pull/1034 --- Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae89ca4cc..9686ea76e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,5 @@ +@Library('pipeline-library@pull/1034/head') _ + // Do not trigger build regularly on change requests as it costs a lot String cronTrigger = '' if(env.BRANCH_NAME == "master") { @@ -7,7 +9,7 @@ if(env.BRANCH_NAME == "master") { // === Actionable in replay env.MAVEN_NTP = true // Can be set to a specific prep archive name in case last commits aren't impacting it -final String fixedPrepArchiveName = '' +final String fixedPrepArchiveName = 'bom-prep-dc9067a4dd575925e2d4a7d0c3b4ceb166d4798c.tar.gz' // Test flags depending on the presence of corresponding labels or marker files // Can be modified to test specific cases independently of the current PR labels or markers // Possible value(s): 'label', 'marker' From d0c6c0aa29b82f06a19626574215e282ef5e2114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Jul 2026 03:12:58 +0200 Subject: [PATCH 3/5] debug: use `infra.retrieveArtifactsFromPreviousBuilds` --- Jenkinsfile | 56 +++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9686ea76e..4dec02822 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,7 +112,10 @@ mavenEnv(jdk: 21) { } stage('retrieve prep archive') { - prepFoundInBuildNumber = retrieveArtifactsFromPreviousBuilds(prepArchiveName, env.JOB_NAME) + prepFoundInBuildNumber = infra.retrieveArtifactsFromPreviousBuilds([ + archiveName: prepArchiveName, + jobName: env.JOB_NAME + ]) if (prepFoundInBuildNumber == 0) { catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error("[SKIP] ${prepArchiveName} not found") } return @@ -380,41 +383,26 @@ boolean flagEnabled(Map flags, String flag, String source = null) { source ? source in flags[flag] : !flags[flag].isEmpty() } -// Search and copy an artifact from builds of a job -// Returns the build number where it has been found, zero otherwise -int retrieveArtifactsFromPreviousBuilds(String archiveName, String jobName) { - int foundInBuildNumber = 0 - boolean archiveExists = false - final int buildNumber = env.BUILD_NUMBER.toInteger() - if (buildNumber == 1) { - echo "[INFO] First build of ${jobName}, no ${archiveName} available yet" - return 0 +// Return a map combinations split into maxSplits from a map of reports containing elapsed time per combination +Map splitReports(Map matrix, int maxSplits, String reportType = 'unknown') { + List splits = (0.. + matrix[b].results.initial.elapsed <=> matrix[a].results.initial.elapsed } - // Loop over builds to retrieve the prep archive as previous build can have (only) other archive(s) - int checkBuildNumber = buildNumber - 1 - // Don't loop until the first build of master '^^ - final int limit = jobName.endsWith('master') ? buildNumber - 50 : 0 - while (!archiveExists && checkBuildNumber > limit) { - echo "[INFO] Trying to retrieve ${archiveName} from ${jobName}#${checkBuildNumber}..." - try { - copyArtifacts(projectName: jobName, - selector: specific("${checkBuildNumber}"), - filter: archiveName, - fingerprintArtifacts: true, - optional: false, - ) - archiveExists = true - } catch(e) {} - if (!archiveExists) { - checkBuildNumber = checkBuildNumber - 1 - } + for (combination in combinations) { + Map target = splits.min { it.total } + target.combinations << combination + target.total += matrix[combination].results.initial.elapsed } - if (!archiveExists) { - echo "[INFO] No ${archiveName} found in any build of ${jobName}" - } else { - foundInBuildNumber = checkBuildNumber - echo "[INFO] ${archiveName} found in ${jobName}#${checkBuildNumber}" + + Map result = [:] + splits.findAll { it.combinations }.eachWithIndex { split, idx -> + result["${reportType}-${idx + 1}"] = split } - return foundInBuildNumber + + return result } From e227b2cfe26cc22dcde2c29f0751e90b0803ab21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Jul 2026 03:16:25 +0200 Subject: [PATCH 4/5] feat(pipeline): split test combinations --- Jenkinsfile | 212 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 145 insertions(+), 67 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4dec02822..226815b80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,7 @@ if(env.BRANCH_NAME == "master") { // === Actionable in replay env.MAVEN_NTP = true +int maxSplits = 20 // Can be set to a specific prep archive name in case last commits aren't impacting it final String fixedPrepArchiveName = 'bom-prep-dc9067a4dd575925e2d4a7d0c3b4ceb166d4798c.tar.gz' // Test flags depending on the presence of corresponding labels or marker files @@ -39,7 +40,7 @@ if (env.CHANGE_ID) { } } -void mavenEnv(Map params = [:], Closure body) { +void mavenNode(Map params = [:], Closure body) { int attempt = 0 final int attempts = 6 retry(count: attempts, conditions: [kubernetesAgent(handleNonKubernetes: true), nonresumable()]) { @@ -47,16 +48,14 @@ void mavenEnv(Map params = [:], Closure body) { // no Dockerized tests; https://github.com/jenkins-infra/documentation/blob/master/ci.adoc#container-agents node('maven-bom') { timeout(120) { - infra.withArtifactCachingProxy { - withEnv([ - 'JAVA_HOME=/opt/jdk-' + params['jdk'], - 'PATH+JDK=/opt/jdk-' + params['jdk'] + '/bin', - "MAVEN_ARGS=${env.MAVEN_ARGS != null ? MAVEN_ARGS : ''} -B ${env.MAVEN_NTP != null ? '-ntp' : ''} -Dmaven.repo.local=${WORKSPACE_TMP}/m2repo", - "MVN_LOCAL_REPO=${WORKSPACE_TMP}/m2repo", - "CURRENT_ATTEMPT=${attempt}", - ]) { - infra.loadMavenLocalCacheIfAny(env.MVN_LOCAL_REPO) - body() + withEnv([ + "MAVEN_ARGS=${env.MAVEN_ARGS != null ? MAVEN_ARGS : ''} -B ${env.MAVEN_NTP != null ? '-ntp' : ''} -Dmaven.repo.local=${WORKSPACE_TMP}/m2repo", + "MVN_LOCAL_REPO=${WORKSPACE_TMP}/m2repo", + "CURRENT_ATTEMPT=${attempt}", + ]) { + infra.loadMavenLocalCacheIfAny(env.MVN_LOCAL_REPO) + infra.withArtifactCachingProxy { + mavenEnv(params, body) } } } @@ -64,13 +63,21 @@ void mavenEnv(Map params = [:], Closure body) { } } +void mavenEnv(Map params = [:], Closure body) { + withEnv(['JAVA_HOME=/opt/jdk-' + params['jdk'], 'PATH+JDK=/opt/jdk-' + params['jdk'] + '/bin',]) { + body() + } +} + String commitId int prepFoundInBuildNumber = 0 Map pluginsByRepository = [:] List lines = [] List newestAndOldestLines = [] -Map results = [:] +Map testMatrix = [:] +Map splitPlan = [:] +final int limitedMaxSplits = 3 final String[] limitedPluginSet = [ 'jenkinsci/aws-credentials-plugin aws-credentials', 'jenkinsci/aws-global-configuration-plugin aws-global-configuration', @@ -84,7 +91,7 @@ final String[] limitedPluginSet = [ 'jenkinsci/pipeline-maven-plugin pipeline-maven,pipeline-maven-api,pipeline-maven-database', // longer than the others, multiple plugins ] -mavenEnv(jdk: 21) { +mavenNode(jdk: 21) { String prepArchiveName stage('init') { Map scmVars = checkout scm @@ -160,6 +167,7 @@ mavenEnv(jdk: 21) { // Limited set from marker file if it exists plugins = fileExists('../limited-plugin-set') ? readFile('../limited-plugin-set').readLines() : limitedPluginSet + maxSplits = limitedMaxSplits // Lines from sample-plugin allLines = sh (returnStdout: true, script: ''' echo "weekly $(grep -F '.x' ../sample-plugin/pom.xml | sed -E 's, *(.+),\\1,g' | sort -rn | xargs)" @@ -191,6 +199,24 @@ mavenEnv(jdk: 21) { } return } + + // Generating all combinations of repository x lines + lines.each { line -> + pluginsByRepository.each { repository, repoPlugins -> + testMatrix["${repository}:${line}"] = [ + plugins: repoPlugins.join(','), + results: [ + initial: [ + elapsed: 0.0001, + failures: 0, + count: 0 + ] + ] + ] + } + } + final String allCombinationNames = testMatrix.keySet().join('\n') + echo "[INFO] ${testMatrix.size()} resulting combinations:\n${allCombinationNames}" } } @@ -213,9 +239,28 @@ mavenEnv(jdk: 21) { } } + stage('generate splits') { + if (testMatrix.isEmpty()) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No combination to split') } + return + } + + int splitCount = maxSplits + + if (flagEnabled(flags, 'full-test')) { + splitCount = maxSplits * lines.size() + echo "[INFO] 'full-test' build, increasing maxSplits of ${maxSplits} by ${lines.size()}x (lines)" + } + + splitPlan = splitReports(testMatrix, splitCount, 'blank') + splitPlan.each { name, split -> + echo "[INFO] '${name}' estimated ${split.total}s (${split.combinations.size()} combinations):\n${split.combinations.join('\n')}" + } + } + stage('stash prep lines') { - if (lines.isEmpty()) { - catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No line to stash') } + if (splitPlan.isEmpty()) { + catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No split, no need to stash any line') } return } @@ -235,62 +280,88 @@ stage('run pct') { } Map branches = [failFast: false] - lines.each {line -> - pluginsByRepository.each { repository, plugins -> - final String branchName = "${repository}:${line}" - branches[branchName] = { - final int jdk = line == 'weekly' ? 21 : 17 - withChecks(name: 'Tests', includeStage: true) { - mavenEnv(jdk: jdk) { - unstash line - withEnv([ - "PLUGINS=${plugins.join(',')}", - "LINE=$line", - 'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1' - ]) { - final long start = System.currentTimeMillis() - int currentAttempt = env.CURRENT_ATTEMPT.toInteger() - echo "[INFO] Current attempt: ${currentAttempt}" - - try { - sh ''' - mvn -v - bash pct.sh - ''' - } catch (e) { - if (!(e instanceof InterruptedException) && !(e instanceof org.jenkinsci.plugins.workflow.support.steps.AgentOfflineException)) { - unstable('PCT failed in ' + repository + ' - line ' + line) - } else { - throw e - } - } finally { - final double elapsed = (System.currentTimeMillis() - start) / 1000.0 - def junitResults - try { - junitResults = junit(testResults: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml') - } catch(e) { - echo "[WARNING] Error junitResult: ${e}" + + splitPlan.each { splitName, split -> + final String estimated = String.format("%.0fs", split.total) + final String stageName = (estimated == '0s') ? splitName : "${splitName} (~${estimated})" + branches[stageName] = { + mavenNode() { + final int totalCombination = split.combinations.size() + final String splitCombinationNames = split.combinations.join('\n') + int combinationCount = 1 + int currentAttempt = env.CURRENT_ATTEMPT.toInteger() + + echo "[INFO] Current split combinations, attempt n°${currentAttempt}:\n${splitCombinationNames}" + + // Unstash all lines used in this split + stage('unstash') { + final List unstashLines = split.combinations.collect { it.split(':')[1] }.unique() + echo "[INFO] Unstashing ${unstashLines.join(' & ')}" + unstashLines.each { unstash it } + } + + split.combinations.each { combination -> + Map data = testMatrix[combination] + final String combinationPlugins = data.plugins + final String[] parts = combination.split(':') + final String repository = parts[0] + final String line = parts[1] + final int jdk = line == 'weekly' ? 21 : 17 + + echo "[INFO] Combination ${combinationCount}/${totalCombination} \"${combination}\", plugin(s): ${combinationPlugins}" + + stage("${combination} (${combinationCount}/${totalCombination})") { + withChecks(name: "Tests ${combination}") { + mavenEnv(jdk: jdk) { + withEnv([ + "PLUGINS=${combinationPlugins}", + "LINE=${line}", + 'EXTRA_MAVEN_PROPERTIES=maven.test.failure.ignore=true:surefire.rerunFailingTestsCount=1' + ]) { + final long start = System.currentTimeMillis() + try { + sh ''' + mvn -v + bash pct.sh + ''' + } catch (e) { + if (!(e instanceof InterruptedException) && !(e instanceof org.jenkinsci.plugins.workflow.support.steps.AgentOfflineException)) { + unstable('PCT failed in ' + repository + ' - line ' + line) + } else { + throw e + } + } finally { + final double elapsed = (System.currentTimeMillis() - start) / 1000.0 + def junitResults + try { + junitResults = junit(testResults: '**/target/surefire-reports/TEST-*.xml,**/target/failsafe-reports/TEST-*.xml') + } catch(e) { + echo "[WARNING] Error junitResult: ${e}" + } + Map result = [ + failCount : junitResults?.failCount ?: 0, + skipCount : junitResults?.skipCount ?: 0, + passCount : junitResults?.passCount ?: 0, + totalCount: junitResults?.totalCount ?: 0, + duration : junitResults?.duration ?: 0, + ] + result.elapsed = elapsed + result.plugins = combinationPlugins + result.pluginCount = combinationPlugins.count(',') + result.attempt = currentAttempt + result.build_id = env.BUILD_ID + result.job_base_name = env.JOB_BASE_NAME + result.short_commit_id = commitId + + testMatrix[combination].results.current = result + + echo "[INFO] results for ${combination}: ${result}" + } } - Map result = [ - failCount : junitResults?.failCount ?: 0, - skipCount : junitResults?.skipCount ?: 0, - passCount : junitResults?.passCount ?: 0, - totalCount: junitResults?.totalCount ?: 0, - duration : junitResults?.duration ?: 0, - ] - result.elapsed = elapsed - result.plugins = plugins.join(',') - result.pluginCount = plugins.size() - result.attempt = currentAttempt - result.build_id = env.BUILD_ID - result.job_base_name = env.JOB_BASE_NAME - result.short_commit_id = commitId.substring(0, 7) - - results[branchName] = result - echo "[INFO] results for ${branchName}: ${result}" } } } + combinationCount++ } } } @@ -299,6 +370,13 @@ stage('run pct') { } stage('report results') { + // Extract only combinations that have a result + Map results = testMatrix.findAll { key, data -> + data.results?.current + }.collectEntries { k, v -> + [k, v.results.current] + } + if (results.isEmpty()) { catchError(buildResult: 'SUCCESS', stageResult: 'NOT_BUILT') { error('[SKIP] No result to report') } return From 65554d4dcbb51a88c2a1df4e7ae8c535eb2d8719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Jul 2026 03:24:11 +0200 Subject: [PATCH 5/5] debug: remove fixed prep archive name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 226815b80..22ff4ac93 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ if(env.BRANCH_NAME == "master") { env.MAVEN_NTP = true int maxSplits = 20 // Can be set to a specific prep archive name in case last commits aren't impacting it -final String fixedPrepArchiveName = 'bom-prep-dc9067a4dd575925e2d4a7d0c3b4ceb166d4798c.tar.gz' +final String fixedPrepArchiveName = '' // Test flags depending on the presence of corresponding labels or marker files // Can be modified to test specific cases independently of the current PR labels or markers // Possible value(s): 'label', 'marker'