diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dd5d6dfa..99887d5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v5 - + - name: Set up Ginkgo CLI run: | go install github.com/onsi/ginkgo/v2/ginkgo@v2.19.1 @@ -99,6 +99,25 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v5 + - name: Move Docker data directory to /mnt + # The default storage device on GitHub-hosted runners is running low during e2e tests. + # Moving Docker data directory to /mnt which has more space and is backed by a larger + # storage device. + # Upstream kubefleet repo has a much bigger default storage device and no secondary + # storage device to use. + run: | + echo "=== Moving Docker to /mnt for more disk space ===" + df -h + sudo systemctl stop docker + echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json + sudo mkdir -p /mnt/docker + if [ -d "/var/lib/docker" ]; then + sudo mv /var/lib/docker/* /mnt/docker/ || true + fi + sudo systemctl start docker + echo "=== Docker moved to /mnt, verifying ===" + docker info | grep "Docker Root Dir" || true + - name: Install Ginkgo CLI run: | go install github.com/onsi/ginkgo/v2/ginkgo@v2.23.4 diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index b58487cb1..b4642bb8c 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -97,10 +97,86 @@ fi # Build the Fleet agent images echo "Building and the Fleet agent images..." +echo "=== Cleaning any existing BuildKit cache ===" +docker buildx prune -f || true +echo "=== Disk usage BEFORE docker builds ===" +df -h +echo "=== Docker system usage BEFORE builds ===" +docker system df +echo "=== Docker volumes BEFORE builds ===" +docker volume ls +echo "=== Docker volumes sizes BEFORE builds ===" +docker system df -v | grep -A 20 "Local Volumes" +echo "=========================================" + +echo "Building hub-agent..." make -C "../.." docker-build-hub-agent +echo "=== Cleaning BuildKit cache after hub-agent ===" +docker buildx prune -f +echo "=== Disk usage after hub-agent build ===" +df -h +echo "=== Docker system usage after hub-agent ===" +docker system df +echo "=== Docker volumes after hub-agent ===" +docker volume ls +echo "=== New volumes since last check ===" +docker system df -v | grep -A 20 "Local Volumes" +echo "=========================================" + +echo "Building member-agent..." make -C "../.." docker-build-member-agent +echo "=== Cleaning BuildKit cache after member-agent ===" +docker buildx prune -f +echo "=== Disk usage after member-agent build ===" +df -h +echo "=== Docker system usage after member-agent ===" +docker system df +echo "=== Docker volumes after member-agent ===" +docker volume ls +echo "=== Volume details after member-agent ===" +docker system df -v | grep -A 20 "Local Volumes" +echo "=========================================" + +echo "Building refresh-token..." make -C "../.." docker-build-refresh-token +echo "=== Cleaning BuildKit cache after refresh-token ===" +docker buildx prune -f +echo "=== Disk usage after refresh-token build ===" +df -h +echo "=== Docker system usage after refresh-token ===" +docker system df +echo "=== Docker volumes after refresh-token ===" +docker volume ls +echo "=== Volume details after refresh-token ===" +docker system df -v | grep -A 20 "Local Volumes" +echo "=========================================" + +echo "Building crd-installer..." make -C "../.." docker-build-crd-installer +echo "=== Cleaning BuildKit cache after crd-installer ===" +docker buildx prune -f +echo "=== Disk usage after crd-installer build ===" +df -h +echo "=== Docker system usage after crd-installer ===" +docker system df +echo "=== Docker volumes after crd-installer ===" +docker volume ls +echo "=== FINAL Docker system usage breakdown ===" +docker system df -v +echo "=== Detailed volume analysis ===" +echo "Volume names and their mount points:" +docker volume ls --format "table {{.Name}}\t{{.Driver}}\t{{.Size}}" +echo "=== Inspect largest volumes ===" +for vol in $(docker volume ls -q | head -10); do + echo "Volume: $vol" + docker volume inspect "$vol" | grep -E "(Mountpoint|CreatedAt|Name)" | head -3 + echo "---" +done +echo "=== Built Fleet images sizes ===" +docker images | grep -E "(hub-agent|member-agent|refresh-token|crd-installer)" | grep "$TAG" +echo "=== All images (for debugging) ===" +docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | head -20 +echo "=============================================" # Load the Fleet agent images into the kind clusters @@ -116,6 +192,10 @@ do kind load docker-image --name "$i" $REGISTRY/$REFRESH_TOKEN_IMAGE:$TAG done +echo "=== FINAL disk usage after all Docker builds and image loading ===" +df -h +echo "==================================================================" + # Install the helm charts # Install the hub agent to the hub cluster