test: migrate to openstack

.github/workflows/spread-docs.yaml

@@ -42,7 +42,7 @@ jobs:
           path: ${{ steps.snapcraft.outputs.snap }}
 
   docs-spread-tests:
-    runs-on: [self-hosted, spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     needs: [docs-snap-build]
     timeout-minutes: 90
 
@@ -65,7 +65,7 @@ jobs:
 
       - name: Run docs spread tests
         run: |
-          spread google:...docs...
+          spread openstack:...docs...
 
       - name: Discard spread workers
         if: always()

.github/workflows/spread-manual.yaml

@@ -52,7 +52,7 @@ jobs:
   kernel-plugins:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -70,15 +70,15 @@ jobs:
           path: tests
       - name: Kernel plugin test
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel       \
-                 google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-bin   \
-                 google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-cross \
-                 google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-deb
+          spread openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel       \
+                 openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-bin   \
+                 openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-cross \
+                 openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/kernel-deb
 
   initrd-plugins:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -96,7 +96,7 @@ jobs:
           path: tests
       - name: Initrd plugin test
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/initrd
+          spread openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/initrd
 
       - name: Discard spread workers
         if: always()
@@ -109,7 +109,7 @@ jobs:
   remote-build:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -129,9 +129,8 @@ jobs:
         env:
           LAUNCHPAD_TOKEN: "${{ secrets.LAUNCHPAD_TOKEN }}"
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/core22/remote-build \
-            google:ubuntu-24.04-64:tests/spread/core24/remote-build \
-            google:fedora-39-64:tests/spread/core24/remote-build:no_platforms
+          spread openstack:ubuntu-22.04-64:tests/spread/core22/remote-build \
+            openstack:ubuntu-24.04-64:tests/spread/core24/remote-build
 
       - name: Discard spread workers
         if: always()
@@ -144,7 +143,7 @@ jobs:
   matter-sdk:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -163,8 +162,8 @@ jobs:
       - name: matter-sdk test
         run: |
           spread \
-              google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/matter-sdk \
-              google:ubuntu-22.04-64:tests/spread/core24-suites/plugins/matter-sdk
+              openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/matter-sdk \
+              openstack:ubuntu-22.04-64:tests/spread/core24-suites/plugins/matter-sdk
 
       - name: Discard spread workers
         if: always()
@@ -177,7 +176,7 @@ jobs:
   colcon-plugins:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -195,11 +194,11 @@ jobs:
           path: tests
       - name: colcon plugin test
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-hello \
-            google:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-msg-package \
-            google:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-packages-ignore \
-            google:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-talker-listener \
-            google:ubuntu-22.04-64:tests/spread/plugins/craft-parts/colcon-hello
+          spread openstack:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-hello \
+            openstack:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-msg-package \
+            openstack:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-packages-ignore \
+            openstack:ubuntu-22.04-64:tests/spread/core24-suites/plugins/colcon-talker-listener \
+            openstack:ubuntu-22.04-64:tests/spread/plugins/craft-parts/colcon-hello
 
       - name: Discard spread workers
         if: always()
@@ -212,7 +211,7 @@ jobs:
   extensions:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -230,7 +229,7 @@ jobs:
           path: tests
       - name: extensions test
         run: |
-          spread google.*:tests/spread/extensions/
+          spread openstack.*:tests/spread/extensions/
 
       - name: Discard spread workers
         if: always()
@@ -243,7 +242,7 @@ jobs:
   snapd-bases:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Cleanup job workspace
         run: |
@@ -261,7 +260,7 @@ jobs:
           path: tests
       - name: Spread test
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/snaps/snapd
+          spread openstack:ubuntu-22.04-64:tests/spread/snaps/snapd
 
       - name: Discard spread workers
         if: always()
@@ -274,7 +273,7 @@ jobs:
   squid-proxy:
     needs: [predicate, snap-build]
     if: needs.predicate.outputs.should_run == 'true'
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     steps:
       - name: Clean up job workspace
         run: |
@@ -292,8 +291,7 @@ jobs:
           path: tests
       - name: Spread test
         run: |
-          spread google:ubuntu-24.04-64:tests/spread/core24/squid-proxy google:ubuntu-22.04-64:tests/spread/core22/squid-proxy
-
+          spread openstack:ubuntu-24.04-64:tests/spread/core24/squid-proxy openstack:ubuntu-22.04-64:tests/spread/core22/squid-proxy
       - name: Discard spread workers
         if: always()
         run: |

.github/workflows/spread.yml

@@ -53,7 +53,7 @@ jobs:
           sudo snap install --dangerous --classic ${{ steps.build-snapcraft.outputs.snap }}
 
   integration-spread-tests:
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     needs: build
     if: ${{ needs.build.outputs.source-files == 'true' }}
     strategy:
@@ -65,9 +65,9 @@ jobs:
       fail-fast: false
       matrix:
         spread-jobs:
-          - google:ubuntu-22.04-64
-          - google:ubuntu-24.04-64
-          - google:ubuntu-26.04-64
+          - openstack:ubuntu-22.04-64
+          - openstack:ubuntu-24.04-64
+          - openstack:ubuntu-26.04-64
 
     steps:
       - name: Checkout snapcraft
@@ -83,9 +83,8 @@ jobs:
           path: tests
 
       - name: Run Ubuntu Pro spread tests
-        if: matrix.spread-jobs != 'google:ubuntu-22.04-64'
+        if: matrix.spread-jobs != 'openstack:ubuntu-22.04-64'
         env:
-          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
           UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }}
         run: |
           if [ -z "$UBUNTU_PRO_TOKEN" ]; then
@@ -101,7 +100,6 @@ jobs:
 
       - name: Run spread
         env:
-          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
           UA_TOKEN: ${{ secrets.UA_TOKEN }}
           RUN_UA_TESTS: ${{ ! github.event.pull_request.head.repo.fork }}
         run: spread ${{ matrix.spread-jobs }}
@@ -115,7 +113,7 @@ jobs:
           done
 
   integration-spread-tests-store:
-    runs-on: [spread-installed]
+    runs-on: [self-hosted, spread-enabled]
     needs: build
     if: ${{ needs.build.outputs.source-files == 'true' }}
     strategy:
@@ -155,13 +153,12 @@ jobs:
       - if: steps.decisions.outputs.RUN == 'true'
         name: Run spread
         env:
-          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
           SNAPCRAFT_ASSERTION_KEY: "${{ secrets.SNAPCRAFT_ASSERTION_KEY }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING_CANDID: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING_CANDID }}"
           SNAPCRAFT_STORE_CREDENTIALS_STAGING_LEGACY: "${{ secrets.SNAPCRAFT_STORE_CREDENTIALS_STAGING_LEGACY }}"
         run: |
-          spread google:ubuntu-22.04-64:tests/spread/store/
+          spread openstack:ubuntu-22.04-64:tests/spread/store/
 
       - name: Discard spread workers
         if: always()

pyproject.toml

@@ -6,7 +6,7 @@ dependencies = [
     "catkin-pkg==1.1.0; sys_platform == 'linux'",
     "click>=8.3.3",
     "craft-application[remote]>=7.1.0",
-    "craft-archives>=2.2.0",
+    "craft-archives>=2.2.1",
     "craft-cli>=3.4.0",
     "craft-grammar>=2.3.0",
     "craft-parts>=2.33.0",

snapcraft/parts/plugins/initrd_build.sh

@@ -130,9 +130,16 @@ setup_ppa() {
   chroot_run "mkdir -p --mode 700 /root/.gnupg"
   chroot_run "mkdir -p            ${gpg_file%/*}"
   chroot_run "echo keyserver hkp://keyserver.ubuntu.com > ${dirmngr_conf}"
+
+  keyserver_options=""
+  if [ -n "${http_proxy:-}" ]; then
+    keyserver_options="--keyserver-options http-proxy=${http_proxy}"
+  fi
+
   chroot_run "gpg --homedir /root/.gnupg         \
                   --no-default-keyring           \
                   --keyring \"${snappy_key}\"    \
+                  ${keyserver_options}           \
                   --recv-keys \"${fingerprint}\""
 
   chroot_run "gpg --homedir /root/.gnupg         \

spread.yaml

@@ -66,9 +66,53 @@ backends:
           storage: 20G
           workers: 6
 
-      - fedora-39-64:
-          workers: 2
-          manual: true
+      # - fedora-39-64:
+      #     workers: 2
+      #     manual: true
+
+  openstack:
+    key: '$(HOST: echo "$OS_CREDENTIALS_AMD64_PS7")'
+    plan: shared.medium
+    halt-timeout: 2h
+    wait-timeout: 10m
+    proxy: ingress-haproxy.ps7.canonical.com
+    cidr-port-rel: ["10.151.96.0/21:5000"]
+    volume-auto-delete: true
+    environment: &openstack-env
+      SNAPD_USE_PROXY: "true"
+      HTTP_PROXY: "http://egress.ps7.internal:3128"
+      http_proxy: "http://egress.ps7.internal:3128"
+      HTTPS_PROXY: "http://egress.ps7.internal:3128"
+      https_proxy: "http://egress.ps7.internal:3128"
+      no_proxy: "127.0.0.1,127.0.0.53,localhost"
+      NO_PROXY: "127.0.0.1,127.0.0.53,localhost"
+      NTP_SERVER: "ntp.ps7.internal"
+    systems: &openstack-systems
+      - ubuntu-22.04-64:
+          image: snapd-spread/ubuntu-22.04-64
+          workers: 10
+          storage: 20G
+      - ubuntu-24.04-64:
+          image: snapd-spread/ubuntu-24.04-64
+          workers: 6
+          storage: 20G
+      - ubuntu-26.04-64:
+          image: snapd-spread/ubuntu-26.04-64
+          workers: 6
+          storage: 20G
+
+  openstack-dev:
+    type: openstack
+    key: '$(HOST: echo "$OS_CREDENTIALS_STG_AMD64_PS7")'
+    plan: shared.xsmall
+    halt-timeout: 2h
+    wait-timeout: 10m
+    groups: [default]
+    proxy: ingress-haproxy.ps7.canonical.com
+    cidr-port-rel: ["10.151.54.0/24:4000"]
+    volume-auto-delete: true
+    environment: *openstack-env
+    systems: *openstack-systems
 
   multipass:
     type: adhoc
@@ -94,8 +138,8 @@ backends:
         sudo sh -c \
         "if [ -d /etc/ssh/sshd_config.d/ ]
         then
-          echo 'PasswordAuthentication yes' > /etc/ssh/sshd_config.d/10-custom.conf
-          echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config.d/10-custom.conf
+          echo 'PasswordAuthentication yes' > /etc/ssh/sshd_config.d/10-spread.conf
+          echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config.d/10-spread.conf
         else
           sed -i /etc/ssh/sshd_config -E -e 's/^#?PasswordAuthentication.*/PasswordAuthentication yes/' -e 's/^#?PermitRootLogin.*/PermitRootLogin yes/'
         fi"
@@ -136,13 +180,25 @@ prepare: |
   # Now run 'apt-get update', but don't show output unless it fails. The logs are
   # long enough as it is.
   if os.query is-ubuntu; then
+    if [ -n "${HTTP_PROXY:-}" ]; then
+      printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' "${HTTP_PROXY}" "${HTTPS_PROXY}" | tee /etc/apt/apt.conf.d/proxy.conf
+    fi
     tempfile="$(mktemp)"
     if ! apt-get update > "$tempfile" 2>&1; then
         cat "$tempfile"
         exit 1
     fi
   fi
 
+  #shellcheck source=tests/spread/tools/proxy.sh
+  . tests/spread/tools/proxy.sh
+  if [ -n "${HTTP_PROXY:-}" ]; then
+    # The snapd daemon doesn't see the environment variables, so we must modify its config
+    # to use a proxy with it. Came from:
+    # https://github.com/canonical/snapd/blob/619cc925793591348db21bb45fa4af65288d2c42/tests/lib/prepare.sh#L90
+    tools_setup_snapd_proxy
+  fi
+
   tests.pkgs install snapd
 
   snap wait system seed.loaded
@@ -166,6 +222,14 @@ prepare: |
   /snap/bin/lxd waitready --timeout=30
   /snap/bin/lxd init --auto
 
+  if [ -n "${HTTP_PROXY:-}" ]; then
+    # Lxd either can't access or doesn't fallback to the environment variables,
+    # so we must set them. This is needed to download remote images.
+    lxc config set core.proxy_http "$HTTP_PROXY"
+    lxc config set core.proxy_https "$HTTPS_PROXY"
+    lxc config set core.proxy_ignore_hosts "$NO_PROXY"
+  fi
+
   install_snapcraft
 
   pushd /snapcraft
@@ -346,6 +410,11 @@ suites:
       SNAPCRAFT_BUILD_ENVIRONMENT: ""
       UBUNTU_PRO_TOKEN: '$(HOST: echo "$UBUNTU_PRO_TOKEN")'
     prepare: |
+      if [ -n "${HTTP_PROXY:-}" ]; then
+        sudo pro config set http_proxy="$http_proxy"
+        sudo pro config set https_proxy="$https_proxy"
+      fi
+
       sudo pro attach --no-auto-enable $UBUNTU_PRO_TOKEN
       sudo pro enable --assume-yes esm-apps
       sudo pro config set lxd_guest_attach=available