test: analyze setup script docker build disk usage

.github/workflows/ci.yml

@@ -41,7 +41,7 @@ jobs:
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v5
-              
+
       - name: Set up Ginkgo CLI
         run: |
           go install github.com/onsi/ginkgo/v2/ginkgo@v2.19.1
@@ -99,6 +99,25 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v5
 
+      - name: Move Docker data directory to /mnt
+        # The default storage device on GitHub-hosted runners is running low during e2e tests.
+        # Moving Docker data directory to /mnt which has more space and is backed by a larger
+        # storage device.
+        # Upstream kubefleet repo has a much bigger default storage device and no secondary
+        # storage device to use.
+        run: |
+          echo "=== Moving Docker to /mnt for more disk space ==="
+          df -h
+          sudo systemctl stop docker
+          echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+          sudo mkdir -p /mnt/docker
+          if [ -d "/var/lib/docker" ]; then
+            sudo mv /var/lib/docker/* /mnt/docker/ || true
+          fi
+          sudo systemctl start docker
+          echo "=== Docker moved to /mnt, verifying ==="
+          docker info | grep "Docker Root Dir" || true
+
       - name: Install Ginkgo CLI
         run: |
           go install github.com/onsi/ginkgo/v2/ginkgo@v2.23.4

test/e2e/setup.sh

@@ -97,10 +97,86 @@ fi
 # Build the Fleet agent images
 echo "Building and the Fleet agent images..."
 
+echo "=== Cleaning any existing BuildKit cache ==="
+docker buildx prune -f || true
+echo "=== Disk usage BEFORE docker builds ==="
+df -h
+echo "=== Docker system usage BEFORE builds ==="
+docker system df
+echo "=== Docker volumes BEFORE builds ==="
+docker volume ls
+echo "=== Docker volumes sizes BEFORE builds ==="
+docker system df -v | grep -A 20 "Local Volumes"
+echo "========================================="
+
+echo "Building hub-agent..."
 make -C "../.." docker-build-hub-agent
+echo "=== Cleaning BuildKit cache after hub-agent ==="
+docker buildx prune -f
+echo "=== Disk usage after hub-agent build ==="
+df -h
+echo "=== Docker system usage after hub-agent ==="
+docker system df
+echo "=== Docker volumes after hub-agent ==="
+docker volume ls
+echo "=== New volumes since last check ==="
+docker system df -v | grep -A 20 "Local Volumes"
+echo "========================================="
+
+echo "Building member-agent..."
 make -C "../.." docker-build-member-agent
+echo "=== Cleaning BuildKit cache after member-agent ==="
+docker buildx prune -f
+echo "=== Disk usage after member-agent build ==="
+df -h
+echo "=== Docker system usage after member-agent ==="
+docker system df
+echo "=== Docker volumes after member-agent ==="
+docker volume ls
+echo "=== Volume details after member-agent ==="
+docker system df -v | grep -A 20 "Local Volumes"
+echo "========================================="
+
+echo "Building refresh-token..."
 make -C "../.." docker-build-refresh-token
+echo "=== Cleaning BuildKit cache after refresh-token ==="
+docker buildx prune -f
+echo "=== Disk usage after refresh-token build ==="
+df -h
+echo "=== Docker system usage after refresh-token ==="
+docker system df
+echo "=== Docker volumes after refresh-token ==="
+docker volume ls
+echo "=== Volume details after refresh-token ==="
+docker system df -v | grep -A 20 "Local Volumes"
+echo "========================================="
+
+echo "Building crd-installer..."
 make -C "../.." docker-build-crd-installer
+echo "=== Cleaning BuildKit cache after crd-installer ==="
+docker buildx prune -f
+echo "=== Disk usage after crd-installer build ==="
+df -h
+echo "=== Docker system usage after crd-installer ==="
+docker system df
+echo "=== Docker volumes after crd-installer ==="
+docker volume ls
+echo "=== FINAL Docker system usage breakdown ==="
+docker system df -v
+echo "=== Detailed volume analysis ==="
+echo "Volume names and their mount points:"
+docker volume ls --format "table {{.Name}}\t{{.Driver}}\t{{.Size}}"
+echo "=== Inspect largest volumes ==="
+for vol in $(docker volume ls -q | head -10); do
+    echo "Volume: $vol"
+    docker volume inspect "$vol" | grep -E "(Mountpoint|CreatedAt|Name)" | head -3
+    echo "---"
+done
+echo "=== Built Fleet images sizes ==="
+docker images | grep -E "(hub-agent|member-agent|refresh-token|crd-installer)" | grep "$TAG"
+echo "=== All images (for debugging) ==="
+docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | head -20
+echo "============================================="
 
 # Load the Fleet agent images into the kind clusters
 
@@ -116,6 +192,10 @@ do
     kind load docker-image --name "$i" $REGISTRY/$REFRESH_TOKEN_IMAGE:$TAG
 done
 
+echo "=== FINAL disk usage after all Docker builds and image loading ==="
+df -h
+echo "=================================================================="
+
 # Install the helm charts
 
 # Install the hub agent to the hub cluster