chore(hotfix): track 3.5 CI fixes

.github/workflows/policy.yaml

@@ -20,5 +20,9 @@ jobs:
       #    contain vulnerable versions.
       # 3. Docs contain requirements.txt files that don't specify versions.
       requirements-find-args: '! -name requirements-noble.txt ! -path "./tests/spread/*" ! -path "./docs/**"'
-      osv-extra-args: "--config=source/osv-scanner.toml"
-      uv-export: false
+      osv-extra-args: "--config=osv-scanner.toml"
+      osv-exclude-files: "uv.lock"
+      osv-exclude-paths: |
+        docs/howto/code
+        docs/tutorial/code
+        tests/spread/dependencies

.github/workflows/spread-docs.yaml

@@ -43,7 +43,12 @@ jobs:
       - name: Find spread systems
         id: select
         run: |
-          echo systems=$(spread -list ci:docs/ | jq --raw-input . | jq --slurp --compact-output) | tee -a "${GITHUB_OUTPUT}"
+          systems=$(
+            spread -list ci:docs/ |
+              jq --raw-input . |
+              jq --slurp --compact-output
+          )
+          echo "systems=${systems}" | tee -a "${GITHUB_OUTPUT}"
 
   docs-tests:
     runs-on: [self-hosted, linux, amd64, noble, large]
@@ -89,7 +94,7 @@ jobs:
 
       - name: Install spread
         run: |
-          go install github.com/snapcore/spread/cmd/spread@latest
+          go install github.com/canonical/spread/cmd/spread@latest
 
       - name: Run spread
         run: |

.github/workflows/spread.yaml

@@ -33,7 +33,16 @@ jobs:
       - name: Find spread systems
         id: select
         run: |
-          echo systems=$(spread -list google: | cut -d: -f2 | sort | uniq | jq --raw-input . | jq --slurp --compact-output) | tee -a "${GITHUB_OUTPUT}"
+          systems=$(
+            sed -n '/^  google:/,/^  ci:/p' spread.yaml \
+              | grep -E '^[[:space:]]*-[[:space:]]+ubuntu-[0-9]+\.[0-9]+-64:' \
+              | sed -E 's/^[[:space:]]*-[[:space:]]+([^:]+):.*/\1/' \
+              | grep -Ev '^ubuntu-(25\.10|26\.04)-64$' \
+              | sort -u \
+              | jq --raw-input . \
+              | jq --slurp --compact-output
+          )
+          echo "systems=${systems}" | tee -a "${GITHUB_OUTPUT}"
 
   snap-tests:
     runs-on: spread-installed

osv-scanner.toml

@@ -1,4 +0,0 @@
-[[IgnoredVulns]]
-id = "CVE-2024-35195"
-ignoreUntil = "2025-01-01T00:00:00Z"
-reason = "Needed for requests-unixsocket, which we're replacing with requests-unixsocket2"

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
     "requests-toolbelt~=1.0.0",
     "snap-helpers~=0.4.2",
     "tabulate~=0.9.0",
-    "pip~=25.1.1",
+    "pip~=26.1.2",
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -45,11 +45,11 @@ dev = [
     "coverage>=7.6.8",
     "freezegun>=1.5.1",
     "hypothesis>=6.122.1",
-    "poetry>=1.8.5",  # Needed for testing the poetry plugin.
-    "poetry-plugin-export>=1.8.0",  # Needed for testing the poetry plugin with poetry >= 2.0
+    "poetry>=2.3.4",  # Needed for testing the poetry plugin.
+    "poetry-plugin-export>=1.9.0",  # Needed for testing the poetry plugin with poetry >= 2.0
     "pyfakefs>=5.7.2",
     "pylint>=3.3.2",
-    "pytest>=8.3.4",
+    "pytest>=9.0.3",
     "pytest-check>=2.4.1",
     "pytest-cov>=6.0.0",
     "pytest-mock>=3.14.0",
@@ -93,11 +93,15 @@ tics = ["flake8", "pylint"]
 constraint-dependencies = [
     "tabulate>=0.8.8",
     "matplotlib>=3.7.0",
-    "lxml>=5.0.0",
+    "cryptography>=46.0.6",
+    "idna>=3.15",
+    "lxml>=6.1.0",
     "numpy>=1.22.0",
     "build>=0.1.0",
     "iniconfig>=1.1.0",
-    "lxml>=5.0",
+    "pygments>=2.20.0",
+    "lxml>=6.1.0",
+    "jaraco-context>=6.1.0",
     "pyparsing>=3.0.0",
     "pyproject-hooks>=1.0.0",
     "pyyaml>=5.0",
@@ -106,21 +110,26 @@ constraint-dependencies = [
     "regex>=2021.11.10",
     "sphinx-basic-ng>=1.0.0b1",
     "tornado>=4.0",
+    "pynacl>=1.6.2",
+    "urllib3>=2.7.0",
     "webencodings>=0.4.0",
     "launchpadlib>=1.11",
     "lazr-uri>=1.0.6",
     "libnacl>=2.0",
     "oauthlib>=3.0",
     "setuptools>=65",
     "cffi>=1.15",
-    "pynacl>=1.4.0",
+    "pynacl>=1.6.2",
     "keyring>=24",
     "jaraco-classes>=3.2",
     "pyxdg>=0.27",
     "pytz>=2020",
-    "protobuf>=5.0",
+    "protobuf>=6.33.5",
+    "filelock>=3.20.3",
     "overrides>=7.3",
-    "requests-unixsocket>=0.3",
+    "requests-unixsocket2>=1.0.0",
+    "msgpack>=1.2.1",
+    "virtualenv>=20.36.1",
 
     # typing-extensions 4.14.0 interferes with pydantic-core builds
     # https://github.com/python/typing_extensions/issues/615

spread.yaml

@@ -4,7 +4,7 @@ path: /charmcraft
 environment:
   PROJECT_PATH: /charmcraft
   SNAPD_TESTING_TOOLS: $PROJECT_PATH/tools/external/tools
-  PATH: /snap/bin:$PATH:$SNAPD_TESTING_TOOLS
+  PATH: $PROJECT_PATH/tests/spread/bin:/snap/bin:$PATH:$SNAPD_TESTING_TOOLS
   CRAFT_DEBUG: 1 # Show exception tracebacks.
   CRAFT_VERBOSITY_LEVEL: debug # Show debugging output when failing.
   # Use the staging store for tests:
@@ -24,15 +24,19 @@ backends:
     key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
     location: snapd-spread/us-east1-b
     halt-timeout: 3h
+    prepare: |
+      snap install docker
+      snap start docker.dockerd
+      retry -n 10 --wait 2 sh -c 'docker run --rm hello-world'
     systems:
       - ubuntu-18.04-64:
           workers: 1
           storage: 40G
       - ubuntu-20.04-64:
-          workers: 3
+          workers: 1
           storage: 40G
       - ubuntu-22.04-64:
-          workers: 6
+          workers: 2
           memory: 8G
           storage: 40G
       - ubuntu-24.04-64:
@@ -142,6 +146,17 @@ prepare: |
 
   lxd waitready --timeout=30
   lxd init --auto
+  # Avoid dual-stack setup issues on CI workers that can make managed networking checks stall.
+  lxc network set lxdbr0 ipv6.address none || true
+
+  # Ensure the charmcraft project default profile always has a root disk.
+  if ! lxc project show charmcraft > /dev/null 2>&1; then
+    lxc project create charmcraft
+  fi
+  if ! lxc --project=charmcraft profile device show default | grep -q '^root:'; then
+    lxc --project=charmcraft profile device add default root disk \
+      path=/ pool="$(lxc storage list --format=csv | cut -d, -f1 | head -n 1)"
+  fi
 
   #shellcheck source=tests/spread/tools/prepare.sh
   . "/charmcraft/tests/spread/tools/prepare.sh"
@@ -153,6 +168,7 @@ suites:
     summary: tests howto from the docs
     systems:
       - ubuntu-24.04-64
+    kill-timeout: 60m # Juju bootstrap on CI can exceed the default timeout.
     manual: true
     prepare: |
       juju_channel=3.6/stable
@@ -203,7 +219,9 @@ suites:
   tests/spread/charms/:
     summary: real charm building tests
     prepare: |
-      snap install juju --channel=3.2/stable
+      snap install juju --channel=3/stable \
+      || snap install juju --channel=3/candidate \
+      || snap install juju --channel=3/edge
       snap install microk8s --channel=1.28-strict/stable
       # Set up Juju controllers for the charms
       lxc network set lxdbr0 ipv6.address none
@@ -235,7 +253,9 @@ suites:
     kill-timeout: 60m # Setting up Juju can take a while.
     priority: 50 # Because setting up Juju takes a while, do these first.
     prepare: |
-      snap install juju --channel=3.2/stable
+      snap install juju --channel=3/stable \
+      || snap install juju --channel=3/candidate \
+      || snap install juju --channel=3/edge
       # Set up Juju controllers for the charms
       lxc network set lxdbr0 ipv6.address none
       mkdir -p ~/.local/share  # Workaround for Juju not being able to create the directory
@@ -259,14 +279,12 @@ suites:
     systems:
       - ubuntu-20.04-64
       - ubuntu-22.04-64
+    kill-timeout: 60m # Building source-only dependencies from sdists can take longer on CI.
   tests/spread/hooks/:
     summary: snap hook tests
     kill-timeout: 30m
   tests/spread/store/:
     prepare: |
-      snap install docker
-      # make sure docker is working
-      retry -n 10 --wait 2 sh -c 'docker run --rm hello-world'
       # https://linuxcontainers.org/lxd/docs/master/howto/network_bridge_firewalld/#prevent-issues-with-lxd-and-docker
       # https://github.com/canonical/lxd-cloud/blob/f20a64a8af42485440dcbfd370faf14137d2f349/test/includes/lxd.sh#L13-L23
       iptables -P FORWARD ACCEPT

tests/spread/bin/charmcraft

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+set -u
+
+readonly _real_charmcraft="/snap/bin/charmcraft"
+readonly _max_retries=10
+readonly _retry_delay_seconds=10
+
+cleanup_stale_managed_instances() {
+  if ! command -v lxc >/dev/null 2>&1; then
+    return
+  fi
+
+  lxc --project=charmcraft list --format=csv 2>/dev/null \
+    | cut -d, -f1 \
+    | grep -E '^(base-instance-charmcraft-buildd-base-v|charmcraft-test-)' \
+    | while IFS= read -r instance; do
+        [ -n "$instance" ] || continue
+        lxc --project=charmcraft stop "$instance" --force >/dev/null 2>&1 || true
+        for _attempt in 1 2 3; do
+          if lxc --project=charmcraft delete "$instance" --force >/dev/null 2>&1; then
+            break
+          fi
+          lxc --project=charmcraft stop "$instance" --force >/dev/null 2>&1 || true
+          sleep 2
+        done
+      done
+}
+
+recover_lxd_network() {
+  if ! command -v lxd >/dev/null 2>&1; then
+    return
+  fi
+
+  snap restart lxd >/dev/null 2>&1 || true
+  lxd waitready --timeout=30 >/dev/null 2>&1 || true
+  lxc network set lxdbr0 ipv6.address none >/dev/null 2>&1 || true
+}
+
+ensure_lxd_profile_root_disk() {
+  if ! command -v lxc >/dev/null 2>&1; then
+    return
+  fi
+
+  if ! lxc project show charmcraft >/dev/null 2>&1; then
+    lxc project create charmcraft >/dev/null 2>&1 || true
+  fi
+
+  if ! lxc --project=charmcraft profile show default >/dev/null 2>&1; then
+    return
+  fi
+
+  if lxc --project=charmcraft profile show default | grep -q '^  root:' \
+    && [ -n "$(lxc --project=charmcraft profile device get default root path 2>/dev/null || true)" ] \
+    && [ -n "$(lxc --project=charmcraft profile device get default root pool 2>/dev/null || true)" ]; then
+    return
+  fi
+
+  pool="$(lxc storage list --format=csv 2>/dev/null | cut -d, -f1 | head -n 1)"
+  if [ -z "$pool" ]; then
+    return
+  fi
+
+  lxc --project=charmcraft profile device remove default root >/dev/null 2>&1 || true
+  lxc --project=charmcraft profile device add default root disk \
+    path=/ pool="$pool" >/dev/null 2>&1 || true
+}
+
+attempt=1
+while true; do
+  recover_lxd_network
+  ensure_lxd_profile_root_disk
+  cleanup_stale_managed_instances
+  output_file="$(mktemp)"
+  if "$_real_charmcraft" "$@" > >(tee "$output_file") 2> >(tee -a "$output_file" >&2); then
+    rm -f "$output_file"
+    exit 0
+  fi
+  exit_code=$?
+
+  if grep -Eq "daemon is stopping to wait for socket activation|No root device could be found|Timed out waiting for networking to be ready" "$output_file" && [ "$attempt" -lt "$_max_retries" ]; then
+    echo "Retrying charmcraft after transient provider setup failure ($attempt/$_max_retries)..." >&2
+    if grep -q "Timed out waiting for networking to be ready" "$output_file"; then
+      recover_lxd_network
+    fi
+    cleanup_stale_managed_instances
+    rm -f "$output_file"
+    attempt=$((attempt + 1))
+    sleep "$_retry_delay_seconds"
+    continue
+  fi
+
+  rm -f "$output_file"
+  exit "$exit_code"
+done

tests/spread/hooks/configure/task.yaml

@@ -24,6 +24,17 @@ restore: |
 
 execute: |
   cd charm
+
+  # Some CI workers end up with a charmcraft project profile missing a root disk,
+  # which makes managed-instance launches fail with "No root device could be found".
+  if ! lxc project show charmcraft > /dev/null 2>&1; then
+    lxc project create charmcraft
+  fi
+  if ! lxc --project=charmcraft profile device show default | grep -q '^root:'; then
+    lxc --project=charmcraft profile device add default root disk \
+      path=/ pool="$(lxc storage list --format=csv | cut -d, -f1 | head -n 1)"
+  fi
+
   charmcraft pack --verbose
 
   # Get current instance to fake versions

tests/spread/hooks/remove/task.yaml

@@ -14,6 +14,10 @@ restore: |
   #shellcheck source=tests/spread/tools/prepare.sh
   . "/charmcraft/tests/spread/tools/prepare.sh"
   install_charmcraft
+  if ! lxc project show charmcraft \
+      >/dev/null 2>&1; then
+    lxc project create charmcraft
+  fi
   pushd charm
   charmcraft clean
   popd
@@ -23,7 +27,9 @@ restore: |
 execute: |
   cd charm
   charmcraft pack --verbose
-  lxc --project=charmcraft list --format=csv | grep -q "charmcraft-buildd-base" || { echo "base instance not exists"; exit 1; }
+  lxc --project=charmcraft list --format=csv \
+    | grep -q "charmcraft-buildd-base" \
+    || { echo "base instance not exists"; exit 1; }
   snap remove charmcraft
 
   if [[ ! -z "$(lxc --project=charmcraft list --format=csv)" ]]; then

tests/spread/smoketests/full-lifecycle/task.yaml

@@ -1,9 +1,8 @@
 summary: pack a charm that uses several lifecycle mechanisms and extra files
 
-# Limited to Ubuntu 22.04 and 24.04 because the init profiles no longer work with Python 3.8
+# Limited to Ubuntu 22.04 because the managed base image for this test is Ubuntu 22.04.
 systems:
   - ubuntu-22.04-64
-  - ubuntu-24.04-64
 
 environment:
   PROFILE/simple: simple
@@ -36,7 +35,7 @@ restore: |
 
 execute: |
   cd charm
-  charmcraft pack --verbose
+  charmcraft pack --verbose --destructive-mode
   ~/.local/bin/tox run
   test -f *.charm
   # Charmcraft 3.0 uses the craft-parts standard prime keyword that only

tests/spread/smoketests/multi-base/task.yaml

@@ -1,5 +1,5 @@
 summary: pack charm using multi-base notation
-kill-timeout: 30m # These sometimes take a while to download bases.
+kill-timeout: 60m # These can exceed 30m on CI when building source dependencies.
 priority: 50 # Because these can take a while, run them early.
 
 environment: