diff --git a/cypress/jenkins/Jenkinsfile b/cypress/jenkins/Jenkinsfile index 1e751f6c34a..64d431390f2 100644 --- a/cypress/jenkins/Jenkinsfile +++ b/cypress/jenkins/Jenkinsfile @@ -1,7 +1,7 @@ #!groovy // Jenkinsfile — CI pipeline for Rancher Dashboard E2E tests // -// Stages: Pre-Clean → Preflight → Checkout → Run Tests → Grab Results +// Stages: Pre-Clean → Preflight → Checkout → Disk → Run Tests → Grab Results // finally: Clean Test Environment → Cleanup Executor → Test Report def branch = "${env.BRANCH}" != 'null' && "${env.BRANCH}" != '' ? "${env.BRANCH}" : 'master' @@ -68,6 +68,10 @@ node('harvester-vpn-1') { ]) // init.sh clones qa-infra-automation at runtime } + // In the repo, so it cannot run before Checkout. + stage('Disk') { + sh 'cypress/jenkins/disk-report.sh before' + } stage('Run Tests') { timeout(time: 180, unit: 'MINUTES') { testExitCode = sh(script: 'cypress/jenkins/init.sh', returnStatus: true) @@ -112,6 +116,9 @@ node('harvester-vpn-1') { rm -f "${WORKSPACE}/qa-infra-automation/ansible/testing/dashboard-e2e/vars.yaml" 2>/dev/null || true rm -f "${WORKSPACE}/qa-infra-automation/ansible/testing/dashboard-e2e/.env" 2>/dev/null || true """ + if (fileExists('cypress/jenkins/disk-report.sh')) { + sh 'cypress/jenkins/disk-report.sh after' + } } try { stage('Test Report') { diff --git a/cypress/jenkins/disk-report.sh b/cypress/jenkins/disk-report.sh new file mode 100755 index 00000000000..0d7b21c0b26 --- /dev/null +++ b/cypress/jenkins/disk-report.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# disk-report.sh - report agent disk usage around a build. +# +# The agent's disk is shared by every executor and by other jobs, and nothing +# reports what a build costs it. This prints a few lines at each end of a build +# so growth is attributable to the build that caused it. +# +# Read only, apart from a counter under /tmp used to print the delta. +# +set -uo pipefail + +DISK_PHASE="${1:?usage: disk-report.sh }" + +ws="${WORKSPACE:-$PWD}" +tag="[disk:${DISK_PHASE}]" +stamp="/tmp/jenkins-disk-${EXECUTOR_NUMBER:-0}.before" + +avail_kb=$(df -P "$ws" 2>/dev/null | awk 'NR==2 {print $4}') +avail_kb=${avail_kb:-0} + +# The preflight guard measures the workspace, which may not be where images and +# cache actually live. +root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null) +if [ -n "$root" ]; then + df -P "$root" 2>/dev/null | + awk -v t="${tag}" -v r="$root" 'NR==2 {print t " docker root " r ": " int($4/1024/1024) "GB free of " int($2/1024/1024) "GB"}' +else + df -P "$ws" 2>/dev/null | + awk -v t="${tag}" 'NR==2 {print t " workspace: " int($4/1024/1024) "GB free of " int($2/1024/1024) "GB"}' +fi + +# Totals rather than a catalogue. A build that reuses cached layers leaves these +# unchanged, so a jump between the two phases is this build's doing. +# +# Counted from the end because the type name is one word or two, and the +# percentage suffix on RECLAIMABLE is stripped first so every row has the same +# shape. +docker system df 2>/dev/null | sed 's/ ([0-9]*%)//' | + awk -v t="${tag}" ' + $1 == "Images" || $1 == "Containers" { printf "%s %-11s %9s total, %9s reclaimable\n", t, $1, $(NF-1), $NF } + $1 == "Build" { printf "%s %-11s %9s total, %9s reclaimable\n", t, "BuildCache", $(NF-1), $NF }' || + echo "${tag} docker unavailable" + +if [ "${DISK_PHASE}" = "before" ]; then + echo "${avail_kb}" > "${stamp}" 2>/dev/null || true +elif [ -f "${stamp}" ]; then + before_kb=$(cat "${stamp}" 2>/dev/null || echo 0) + if [ "${before_kb}" -gt 0 ] 2>/dev/null; then + # Other executors share the disk, so this is net movement rather than this + # build's own footprint, and may be negative. + delta_mb=$(( (before_kb - avail_kb) / 1024 )) + if [ "${delta_mb}" -ge 0 ]; then + echo "${tag} net change over this build: ${delta_mb}MB consumed" + else + echo "${tag} net change over this build: $(( -delta_mb ))MB released" + fi + fi + rm -f "${stamp}" 2>/dev/null || true +fi +exit 0 diff --git a/cypress/jenkins/init.sh b/cypress/jenkins/init.sh index ee08e2d38bb..37bddf62df1 100755 --- a/cypress/jenkins/init.sh +++ b/cypress/jenkins/init.sh @@ -38,10 +38,14 @@ clone_qa_infra() { echo "[init] qa-infra-automation (${QA_INFRA_BRANCH}) at $(git -C "${QA_INFRA_DIR}" rev-parse --short HEAD)" } -# Build the runner image from Dockerfile.quickstart +# Build the runner image from Dockerfile.quickstart. +# +# Cached rather than --no-cache: Pre-Clean removes this tag every build, so the +# image is rebuilt regardless, and --no-cache only added a cold build plus a +# BuildKit cache record per layer that nothing on the agent reclaims. build_runner_image() { echo "[init] Building ${RUNNER_IMAGE} image..." - docker build -q --no-cache -f "${PLAYBOOK_DIR}/Dockerfile.quickstart" \ + docker build -q -f "${PLAYBOOK_DIR}/Dockerfile.quickstart" \ -t "${RUNNER_IMAGE}" "${PLAYBOOK_DIR}" }