diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index f33116855..393099b46 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -27,6 +27,7 @@ jobs:
           docker buildx build \
             --load \
             --file Dockerfile \
+            --build-arg SKIP_UPGRADE \
             --cache-from=type=local,src=/tmp/docker-cache \
             --cache-to=type=local,dest=/tmp/docker-cache,mode=max \
             .
diff --git a/Dockerfile b/Dockerfile
index 3e1d98f57..d992f7a3d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,8 +4,11 @@ FROM ubuntu:resolute AS builder
 LABEL description="Development build environment"
 
 # Update the distro and install our tools
-RUN apt-get -y update && apt-get -y upgrade \
- && apt-get -y install software-properties-common \
+RUN apt-get -y update
+
+RUN if [ -z "SKIP_UPGRADE" ]; then apt-get -y upgrade; fi
+
+RUN apt-get -y install software-properties-common \
  && apt-get -y install wget \
  # GCC stuff
  && apt-get -y install build-essential gcc-15 g++-15 \