Add runner update channels for canary field testing on Bencher Cloud

.github/workflows/ci.yml

@@ -122,12 +122,62 @@ jobs:
   runner:
     name: Runner
     needs: changes
-    if: ${{ github.ref != 'refs/heads/cloud' && (needs.changes.outputs.runner == 'true' || github.ref == 'refs/heads/devel' || startsWith(github.ref, 'refs/tags/')) }}
+    # On cloud pushes, the runner is only rebuilt when runner code changed
+    # (avoids pointless canary fleet updates) and skips the scenario tests
+    # (the same commits already ran them on devel).
+    if: ${{ needs.changes.outputs.runner == 'true' || github.ref == 'refs/heads/devel' || startsWith(github.ref, 'refs/tags/') }}
     uses: ./.github/workflows/runner.yml
     with:
       mold-version: "2.34.1"
       zig-version: "0.13.0"
       zig-build-version: "0.19.3"
+      build-only: ${{ github.ref == 'refs/heads/cloud' }}
+
+  # Publish the cloud branch runner build to the rolling `canary` prerelease.
+  # Canary channel runners poll this release's checksums and self-update when
+  # the published binary changes.
+  publish_runner_canary:
+    name: Publish Runner Canary
+    permissions:
+      contents: write
+    needs: runner
+    if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/cloud' && needs.runner.result == 'success' }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download Runner Artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: runner-canary-linux-*
+          merge-multiple: true
+      - name: Publish to canary prerelease
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+          if ! gh release view canary > /dev/null 2>&1; then
+            gh release create canary \
+              --prerelease \
+              --latest=false \
+              --target "$GITHUB_SHA" \
+              --title "Runner Canary" \
+              --notes "Rolling prerelease of the runner, rebuilt on cloud branch deploys."
+          fi
+          # Binaries first, checksums last: the server triggers updates off
+          # the .sha256 files, so a partial upload never advertises a binary
+          # that is not yet in place.
+          gh release upload canary --clobber \
+            runner-canary-linux-x86-64 \
+            runner-canary-linux-arm-64
+          gh release upload canary --clobber \
+            runner-canary-linux-x86-64.sha256 \
+            runner-canary-linux-arm-64.sha256
+          # Keep the canary tag pointing at the commit the assets were built
+          # from. Downloads are by asset, so this is purely for humans
+          # inspecting the release; tag moves do not trigger CI (tags: v**).
+          gh api --method PATCH "repos/{owner}/{repo}/git/refs/tags/canary" \
+            -f "sha=$GITHUB_SHA" -F force=true > /dev/null
+          gh release edit canary --notes "Rolling prerelease of the runner, rebuilt on cloud branch deploys. Currently built from $GITHUB_SHA."
 
   test:
     name: Test
@@ -232,7 +282,7 @@ jobs:
   ci-success:
     name: CI Success
     if: always()
-    needs: [lint, cli, runner, test, build, docker, deploy, release]
+    needs: [lint, cli, runner, publish_runner_canary, test, build, docker, deploy, release]
     runs-on: ubuntu-22.04
     steps:
       - name: Check all jobs passed
@@ -249,6 +299,10 @@ jobs:
             echo "Runner failed"
             exit 1
           fi
+          if [[ "${{ needs.publish_runner_canary.result }}" != "success" && "${{ needs.publish_runner_canary.result }}" != "skipped" ]]; then
+            echo "Publish Runner Canary failed"
+            exit 1
+          fi
           if [[ "${{ needs.test.result }}" != "success" && "${{ needs.test.result }}" != "skipped" ]]; then
             echo "Test failed"
             exit 1

.github/workflows/runner.yml

@@ -12,6 +12,13 @@ on:
       zig-build-version:
         type: string
         required: true
+      # Skip the KVM scenario tests and only build the binaries.
+      # Used on the cloud branch, where the same commits already ran the
+      # scenario tests on devel.
+      build-only:
+        type: boolean
+        required: false
+        default: false
 
 env:
   CARGO_TERM_COLOR: always
@@ -20,6 +27,7 @@ env:
 jobs:
   test_runner_scenarios:
     name: Test Runner Scenarios
+    if: ${{ !inputs.build-only }}
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -71,6 +79,11 @@ jobs:
           HEAD_REF: ${{ github.head_ref || github.ref_name }}
         run: |
           VERSION="$HEAD_REF"
+          # Cloud branch builds are versioned as `canary` for the rolling
+          # canary prerelease that Bencher Cloud runners track.
+          if [[ "$GITHUB_REF" == "refs/heads/cloud" ]]; then
+            VERSION="canary"
+          fi
           echo "RUNNER_VERSION=${VERSION//\//-}" >> "$GITHUB_ENV"
       - name: Setup Rust
         uses: ./.github/actions/setup-rust

lib/bencher_api_tests/Cargo.toml

@@ -45,6 +45,7 @@ sha2 = { workspace = true, optional = true }
 slog.workspace = true
 tempfile.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+url.workspace = true
 
 [lints]
 workspace = true

lib/bencher_api_tests/src/context.rs

@@ -44,13 +44,13 @@ pub struct TestServer {
 impl TestServer {
     /// Create a new test server with default settings.
     pub async fn new() -> Self {
-        Self::build(None, None, None).await
+        Self::build(None, None, None, None).await
     }
 
     /// Create a new test server with custom upload timeout and max body size.
     #[cfg(feature = "plus")]
     pub async fn new_with_limits(upload_timeout: u64, max_body_size: u64) -> Self {
-        Self::build(Some(upload_timeout), Some(max_body_size), None).await
+        Self::build(Some(upload_timeout), Some(max_body_size), None, None).await
     }
 
     /// Create a new test server with custom upload timeout, max body size, and injectable clock.
@@ -60,7 +60,13 @@ impl TestServer {
         max_body_size: u64,
         clock: bencher_json::Clock,
     ) -> Self {
-        Self::build(Some(upload_timeout), Some(max_body_size), Some(clock)).await
+        Self::build(Some(upload_timeout), Some(max_body_size), Some(clock), None).await
+    }
+
+    /// Create a new test server with a custom runner self-update base URL.
+    #[cfg(feature = "plus")]
+    pub async fn new_with_runner_update_base_url(base_url: url::Url) -> Self {
+        Self::build(None, None, None, Some(base_url)).await
     }
 
     #[cfg(feature = "plus")]
@@ -73,6 +79,7 @@ impl TestServer {
         upload_timeout: Option<u64>,
         max_body_size: Option<u64>,
         clock: Option<bencher_json::Clock>,
+        runner_update_base_url: Option<url::Url>,
     ) -> Self {
         // Create logger early so it can be used for OCI storage
         let log_config = ConfigLogging::StderrTerminal {
@@ -147,6 +154,7 @@ impl TestServer {
             heartbeat_timeout: std::time::Duration::from_secs(5),
             job_timeout_grace_period: std::time::Duration::from_mins(1),
             heartbeat_tasks: bencher_schema::context::HeartbeatTasks::new(),
+            runner_update: bencher_schema::context::RunnerUpdate::new(runner_update_base_url),
             shutdown: bencher_schema::context::CancellationToken::new(),
         };
 
@@ -164,6 +172,7 @@ impl TestServer {
         upload_timeout: Option<u64>,
         max_body_size: Option<u64>,
         _clock: Option<()>,
+        _runner_update_base_url: Option<url::Url>,
     ) -> Self {
         // Create logger early so it can be used for OCI storage
         let log_config = ConfigLogging::StderrTerminal {

lib/bencher_config/src/config_tx.rs

@@ -85,6 +85,9 @@ pub enum ConfigTxError {
     #[error("Failed to get server ID: {0}")]
     ServerId(dropshot::HttpError),
     #[cfg(feature = "plus")]
+    #[error("Invalid runner update base URL: {0}")]
+    RunnerUpdateBaseUrl(bencher_json::ValidError),
+    #[cfg(feature = "plus")]
     #[error("Failed to spawn stats: {0}")]
     SpawnStats(dropshot::HttpError),
 }
@@ -299,6 +302,13 @@ async fn into_context(
                 u64::from(u32::from(r.job_timeout_grace_period))
             }),
     );
+    #[cfg(feature = "plus")]
+    let runner_update_base_url = plus
+        .as_ref()
+        .and_then(|p| p.runners.as_ref())
+        .and_then(|r| r.update_base_url.clone())
+        .map(|url| url.try_into().map_err(ConfigTxError::RunnerUpdateBaseUrl))
+        .transpose()?;
 
     info!(log, "Configuring Bencher Plus");
     #[cfg(feature = "plus")]
@@ -369,6 +379,8 @@ async fn into_context(
         #[cfg(feature = "plus")]
         heartbeat_tasks: bencher_schema::context::HeartbeatTasks::new(),
         #[cfg(feature = "plus")]
+        runner_update: bencher_schema::context::RunnerUpdate::new(runner_update_base_url),
+        #[cfg(feature = "plus")]
         shutdown: bencher_schema::context::CancellationToken::new(),
     })
 }

lib/bencher_json/src/lib.rs

@@ -43,7 +43,7 @@ pub mod urlencoded;
 pub mod user;
 
 #[cfg(feature = "plus")]
-pub use bencher_valid::{Architecture, OperatingSystem, Sandbox, Sha256};
+pub use bencher_valid::{Architecture, OperatingSystem, Sandbox, Sha256, UpdateChannel};
 pub use big_int::BigInt;
 pub use clock::Clock;
 pub use organization::{

lib/bencher_json/src/runner/websocket.rs

@@ -1,6 +1,4 @@
-#[cfg(feature = "plus")]
-use bencher_valid::Sha256;
-use bencher_valid::{Architecture, OperatingSystem, PollTimeout};
+use bencher_valid::{Architecture, OperatingSystem, PollTimeout, Sha256, UpdateChannel};
 #[cfg(feature = "schema")]
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -60,6 +58,12 @@ pub struct JsonRunnerMetadata {
     pub arch: Architecture,
     /// Runner binary version
     pub version: String,
+    /// Update channel (absent means stable)
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub channel: Option<UpdateChannel>,
+    /// SHA-256 checksum of the running runner binary
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub checksum: Option<Sha256>,
 }
 
 /// Messages sent from the server to the runner over the WebSocket channel.
@@ -136,6 +140,8 @@ mod tests {
                 os: OperatingSystem::Linux,
                 arch: Architecture::X86_64,
                 version: test_version(),
+                channel: None,
+                checksum: None,
             }),
         };
         let json = serde_json::to_string(&msg).unwrap();
@@ -150,6 +156,8 @@ mod tests {
                 assert_eq!(runner.os, OperatingSystem::Linux);
                 assert_eq!(runner.arch, Architecture::X86_64);
                 assert_eq!(runner.version, test_version());
+                assert!(runner.channel.is_none());
+                assert!(runner.checksum.is_none());
             },
             other => panic!("Expected Ready, got {other:?}"),
         }
@@ -163,6 +171,8 @@ mod tests {
                 os: OperatingSystem::Linux,
                 arch: Architecture::Aarch64,
                 version: test_version(),
+                channel: None,
+                checksum: None,
             }),
         };
         let json = serde_json::to_string(&msg).unwrap();
@@ -177,11 +187,79 @@ mod tests {
                 assert_eq!(runner.os, OperatingSystem::Linux);
                 assert_eq!(runner.arch, Architecture::Aarch64);
                 assert_eq!(runner.version, test_version());
+                assert!(runner.channel.is_none());
+                assert!(runner.checksum.is_none());
+            },
+            other => panic!("Expected Ready, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn ready_canary_channel_roundtrip() {
+        let checksum: Sha256 = "a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3"
+            .parse()
+            .unwrap();
+        let msg = RunnerMessage::Ready {
+            poll_timeout: None,
+            runner: Some(JsonRunnerMetadata {
+                os: OperatingSystem::Linux,
+                arch: Architecture::X86_64,
+                version: test_version(),
+                channel: Some(UpdateChannel::Canary),
+                checksum: Some(checksum.clone()),
+            }),
+        };
+        let json = serde_json::to_string(&msg).unwrap();
+        let deserialized: RunnerMessage = serde_json::from_str(&json).unwrap();
+        match deserialized {
+            RunnerMessage::Ready { runner, .. } => {
+                let runner = runner.unwrap();
+                assert_eq!(runner.channel, Some(UpdateChannel::Canary));
+                assert_eq!(runner.checksum, Some(checksum));
+            },
+            other => panic!("Expected Ready, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn ready_legacy_metadata_deserializes() {
+        // A pre-channel runner sends only os/arch/version; the new fields
+        // must default to the stable channel with no checksum.
+        let json = format!(
+            r#"{{"event":"ready","runner":{{"os":"linux","arch":"x86_64","version":"{version}"}}}}"#,
+            version = test_version(),
+        );
+        let deserialized: RunnerMessage = serde_json::from_str(&json).unwrap();
+        match deserialized {
+            RunnerMessage::Ready { runner, .. } => {
+                let runner = runner.unwrap();
+                assert_eq!(runner.version, test_version());
+                assert!(runner.channel.is_none());
+                assert!(runner.checksum.is_none());
             },
             other => panic!("Expected Ready, got {other:?}"),
         }
     }
 
+    #[test]
+    fn ready_stable_metadata_wire_format_is_legacy() {
+        // A stable runner without a checksum must serialize exactly like a
+        // pre-channel runner: no channel or checksum fields on the wire.
+        let msg = RunnerMessage::Ready {
+            poll_timeout: None,
+            runner: Some(JsonRunnerMetadata {
+                os: OperatingSystem::Linux,
+                arch: Architecture::X86_64,
+                version: test_version(),
+                channel: None,
+                checksum: None,
+            }),
+        };
+        let json = serde_json::to_string(&msg).unwrap();
+        assert!(!json.contains("channel"), "unexpected channel in {json}");
+        assert!(!json.contains("checksum"), "unexpected checksum in {json}");
+    }
+
     #[test]
     fn ready_no_runner_metadata_roundtrip() {
         let msg = RunnerMessage::Ready {

lib/bencher_json/src/system/config/plus/runners.rs

@@ -1,6 +1,6 @@
 use std::sync::LazyLock;
 
-use bencher_valid::{GracePeriod, HeartbeatTimeout};
+use bencher_valid::{GracePeriod, HeartbeatTimeout, Url};
 #[cfg(feature = "schema")]
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -35,6 +35,10 @@ pub struct JsonRunners {
     /// Defaults to 60 seconds.
     #[serde(default = "default_job_timeout_grace_period")]
     pub job_timeout_grace_period: GracePeriod,
+    /// Base URL for runner self-update downloads.
+    /// Defaults to the Bencher GitHub Releases download URL.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub update_base_url: Option<Url>,
 }
 
 fn default_heartbeat_timeout() -> HeartbeatTimeout {