fix(ci): preserve system CA trust

Build a combined runner certificate bundle instead of replacing public roots with the generated proxy CA. Centralize additive container installation in the shared proxy CA helper.

Assisted-by: Codex:gpt-5
Signed-off-by: Richard Palethorpe <io@richiejp.com>

Author: richiejp

.docker/apt-mirror.sh

@@ -17,22 +17,8 @@
 
 set -e
 
-# BuildKit exposes the ephemeral interception CA at this dedicated path. Copy
-# it into the image trust bundle only when the proxy-enabled workflows supply
-# it; ordinary local and test builds retain their base-image trust unchanged.
-proxy_ca=/run/secrets/build_proxy_ca
-if [ -s "$proxy_ca" ]; then
-    # Keep the generated CA in the distribution-managed local certificate
-    # directory. Installing or upgrading ca-certificates later in this layer
-    # regenerates the bundle, so appending directly to it would be lost.
-    mkdir -p /usr/local/share/ca-certificates
-    cp "$proxy_ca" /usr/local/share/ca-certificates/localai-build-proxy.crt
-    if command -v update-ca-certificates >/dev/null 2>&1; then
-        update-ca-certificates
-    fi
-    cat > /etc/apt/apt.conf.d/99localai-build-proxy-ca <<EOF
-Acquire::https::CaInfo "$proxy_ca";
-EOF
+if [ -f /usr/local/sbin/install-build-proxy-ca ]; then
+    sh /usr/local/sbin/install-build-proxy-ca
 fi
 
 # Ubuntu 24.04 (noble) ships DEB822 sources at /etc/apt/sources.list.d/ubuntu.sources;

.docker/install-base-deps.sh

@@ -35,22 +35,6 @@
 
 set -eux
 
-# Some callers use this script without apt-mirror.sh. Ensure those first-stage
-# downloads also trust the proxy without masking the normal system CA bundle.
-proxy_ca=/run/secrets/build_proxy_ca
-if [ -s "$proxy_ca" ]; then
-    # Persist the generated CA as a local certificate so a subsequent
-    # ca-certificates package install cannot regenerate the bundle without it.
-    mkdir -p /usr/local/share/ca-certificates
-    cp "$proxy_ca" /usr/local/share/ca-certificates/localai-build-proxy.crt
-    if command -v update-ca-certificates >/dev/null 2>&1; then
-        update-ca-certificates
-    fi
-    cat > /etc/apt/apt.conf.d/99localai-build-proxy-ca <<EOF
-Acquire::https::CaInfo "$proxy_ca";
-EOF
-fi
-
 # --- 0. apt mirror rewrite (no-op when APT_MIRROR / APT_PORTS_MIRROR unset) ---
 if [ -x /usr/local/sbin/apt-mirror ]; then
     APT_MIRROR="${APT_MIRROR:-}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR:-}" \

.docker/install-build-proxy-ca.sh

@@ -10,4 +10,17 @@ fi
 
 mkdir -p /usr/local/share/ca-certificates
 cp "$proxy_ca" /usr/local/share/ca-certificates/localai-build-proxy.crt
-update-ca-certificates
+if command -v update-ca-certificates >/dev/null 2>&1; then
+    update-ca-certificates
+elif [ -f /etc/ssl/certs/ca-certificates.crt ]; then
+    cat "$proxy_ca" >>/etc/ssl/certs/ca-certificates.crt
+else
+    echo 'build proxy: no system CA bundle found' >&2
+    exit 1
+fi
+
+if [ -d /etc/apt/apt.conf.d ]; then
+    cat > /etc/apt/apt.conf.d/99localai-build-proxy-ca <<EOF
+Acquire::https::CaInfo "$proxy_ca";
+EOF
+fi

.github/scripts/start-build-proxy.sh

@@ -11,19 +11,38 @@ for _ in $(seq 1 50); do
 done
 grep '^proxy=' "$output/proxy.log"
 grep '^ca=' "$output/proxy.log"
+proxy_ca="$output/ca/ca.crt"
+ca_bundle="$output/ca/ca-bundle.crt"
+system_ca=""
+for candidate in \
+  /etc/ssl/certs/ca-certificates.crt \
+  /etc/ssl/cert.pem \
+  /etc/pki/tls/certs/ca-bundle.crt \
+  /etc/openssl/certs/ca-certificates.crt; do
+  if test -s "$candidate"; then
+    system_ca="$candidate"
+    break
+  fi
+done
+if test -z "$system_ca"; then
+  echo 'build proxy: unable to find the runner system CA bundle' >&2
+  exit 1
+fi
+cat "$system_ca" "$proxy_ca" >"$ca_bundle"
 {
   echo "LOCALAI_BUILD_PROXY=http://127.0.0.1:18080"
   echo "LOCALAI_BUILD_PROXY_OUTPUT=$output"
-  echo "LOCALAI_BUILD_PROXY_CA=$output/ca/ca.crt"
+  echo "LOCALAI_BUILD_PROXY_CA=$proxy_ca"
+  echo "LOCALAI_BUILD_PROXY_CA_BUNDLE=$ca_bundle"
   echo "HTTP_PROXY=http://127.0.0.1:18080"
   echo "HTTPS_PROXY=http://127.0.0.1:18080"
   echo "http_proxy=http://127.0.0.1:18080"
   echo "https_proxy=http://127.0.0.1:18080"
-  echo "SSL_CERT_FILE=$output/ca/ca.crt"
-  echo "CURL_CA_BUNDLE=$output/ca/ca.crt"
-  echo "REQUESTS_CA_BUNDLE=$output/ca/ca.crt"
-  echo "GIT_SSL_CAINFO=$output/ca/ca.crt"
-  echo "NODE_EXTRA_CA_CERTS=$output/ca/ca.crt"
+  echo "SSL_CERT_FILE=$ca_bundle"
+  echo "CURL_CA_BUNDLE=$ca_bundle"
+  echo "REQUESTS_CA_BUNDLE=$ca_bundle"
+  echo "GIT_SSL_CAINFO=$ca_bundle"
+  echo "NODE_EXTRA_CA_CERTS=$proxy_ca"
   echo "NO_PROXY=localhost,127.0.0.1"
   echo "no_proxy=localhost,127.0.0.1"
 } >>"$GITHUB_ENV"

Dockerfile

@@ -23,7 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 # hwdata ships /usr/share/hwdata/pci.ids. Without it, the ghw library we use
 # for hardware detection cannot resolve PCI vendor IDs and fails to enumerate
 # GPUs at all, so the image reports "No GPU detected" (see issue #10941).
-RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -278,7 +278,7 @@ ARG APT_PORTS_MIRROR
 RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \
 gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
 RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu ${UBUNTU_CODENAME}/lts/2350 unified" > /etc/apt/sources.list.d/intel-graphics.list
-RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
     apt-get install -y --no-install-recommends \

backend/Dockerfile.audio-cpp

@@ -77,7 +77,7 @@ WORKDIR /build
 #
 # BUILD_TYPE=vulkan additionally needs the loader headers and glslc; both are in
 # Noble. The CUDA toolkit for BUILD_TYPE=cublas comes from BASE_IMAGE.
-RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
     apt-get install -y --no-install-recommends \

backend/Dockerfile.base-grpc-builder

@@ -96,6 +96,7 @@ WORKDIR /build
 # script the variant Dockerfiles' builder-fromsource stage runs.
 RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \
     --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+    --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \
     bash /usr/local/sbin/install-base-deps
 
 WORKDIR /

backend/Dockerfile.bonsai

@@ -78,6 +78,7 @@ WORKDIR /build
 # this from-source path are bit-equivalent.
 RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \
     --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+    --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca \
     bash /usr/local/sbin/install-base-deps
 
 # Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so

backend/Dockerfile.ds4

@@ -27,7 +27,7 @@ WORKDIR /build
 #   - gRPC/Protobuf: system apt packages are sufficient; ds4's wrapper only links
 #     against them, it doesn't ship the gRPC source tree.
 #   - nlohmann-json: dsml_renderer's only third-party dep.
-RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
     apt-get install -y --no-install-recommends \

backend/Dockerfile.golang

@@ -30,7 +30,7 @@ ARG APT_PORTS_MIRROR
 # build-essential. So: try gcc-14 from the configured repos, fall back
 # gracefully when it's not available so jammy-based builds don't fail
 # at the apt step.
-RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+RUN --mount=type=secret,id=build_proxy_ca,target=/run/secrets/build_proxy_ca,mode=0444 --mount=type=bind,source=.docker/install-build-proxy-ca.sh,target=/usr/local/sbin/install-build-proxy-ca --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     APT_MIRROR="${APT_MIRROR}" APT_PORTS_MIRROR="${APT_PORTS_MIRROR}" sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
     apt-get install -y --no-install-recommends \