test(backup): run the backup manager tests on the local backend (#11188)

`ic-backup` replays each replica version's artifacts with that version's
own `ic-replay` and sandbox binaries, and downloads them from
`download.dfinity.systems` when `binaries/<version>/` doesn't already
hold them. The backup tests pre-seeded that directory for the branch
version only, so the mainnet version — which both tests replay, as the
initial version of the upgrade test and as the target version of the
downgrade test — was still downloaded, unverified, at test time.

Two commits, so the backend switch can be reverted on its own if the
nightly job dislikes it:

1. **Seeding.** Take the mainnet binaries from `@mainnet_nns_binaries`
and seed the directory of every version that will be replayed: the
initial, the target and the branch version. One code path covers both
tests, which is necessary because they share `backup_manager_test_bin`
and `test()` cannot tell them apart. A version we have no binaries for
now fails immediately, naming the versions we do have, instead of
stalling for the whole test timeout on a download that cannot succeed.
`copy_file` also checks that what it copies is executable — a dependency
that lost its `+x` bit would otherwise only surface minutes later, as
`ic-replay` failing to spawn its sandbox.
2. **Drop `backend = "farm"`**, since nothing in these tests reaches the
network any more.

Only the `_local` variants change tagging. The plain Farm variant is
already `manual` via `colocate`, and `_colocate` is unaffected.

### Verification

Both pass on the local backend, well inside the 25-minute per-test
timeout set in `backup_manager_test.rs`:

```
//rs/tests/consensus/backup:backup_manager_upgrade_test_local     PASSED in 917.0s
//rs/tests/consensus/backup:backup_manager_downgrade_test_local   PASSED in 883.9s
```

Each log shows both versions seeded (`0000…0000` for the unstamped
branch build and `798712972b1e…` for mainnet), several successful
replays across a detected version change, and **no**
`download.dfinity.systems` anywhere. Since the `_local` sandbox has no
network, a download that was still needed would have failed the test
outright.

The downgrade direction is worth a second look from a reviewer:
`guestos_update = "mainnet_nns"` appears exactly once in the repo, so
this is its first run on the local backend.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: basvandijk

rs/tests/consensus/backup/BUILD.bazel

@@ -1,5 +1,5 @@
 load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library")
-load("//rs/tests:common.bzl", "DEFAULT_VCPUS_PER_VM", "MESSAGE_CANISTER_RUNTIME_DEPS", "MIN_LOCAL_CPUS", "UNIVERSAL_CANISTER_RUNTIME_DEPS")
+load("//rs/tests:common.bzl", "DEFAULT_VCPUS_PER_VM", "MAINNET_ENV", "MAINNET_NNS_REPLAY_RUNTIME_DEPS", "MESSAGE_CANISTER_RUNTIME_DEPS", "MIN_LOCAL_CPUS", "UNIVERSAL_CANISTER_RUNTIME_DEPS")
 load("//rs/tests:system_tests.bzl", "system_test_nns")
 
 package(default_visibility = ["//rs:system-tests-pkg"])
@@ -27,12 +27,13 @@ rust_library(
     ],
 )
 
-BACKUP_RUNTIME_DEPS = MESSAGE_CANISTER_RUNTIME_DEPS | UNIVERSAL_CANISTER_RUNTIME_DEPS | {
+BACKUP_RUNTIME_DEPS = MESSAGE_CANISTER_RUNTIME_DEPS | UNIVERSAL_CANISTER_RUNTIME_DEPS | MAINNET_NNS_REPLAY_RUNTIME_DEPS | {
     "IC_BACKUP_PATH": "//rs/backup:ic-backup",
     "IC_REPLAY_PATH": "//rs/replay:ic-replay",
     "COMPILER_SANDBOX_PATH": "//rs/canister_sandbox:compiler_sandbox",
     "SANDBOX_LAUNCHER_PATH": "//rs/canister_sandbox:sandbox_launcher",
     "CANISTER_SANDBOX_PATH": "//rs/canister_sandbox:canister_sandbox",
+    "IC_VERSION_FILE": "//bazel:version.txt",
 }
 
 rust_binary(
@@ -50,17 +51,8 @@ rust_binary(
 
 system_test_nns(
     name = "backup_manager_upgrade_test",
-    # Pinned to the Farm backend. This test drives `ic-backup`, which replays the
-    # subnet's artifacts with a version-specific `ic-replay` binary. For the initial
-    # mainnet replica version (`guestos = "mainnet_nns"`) the test does not provide
-    # that binary locally, so `ic-backup` downloads it from download.dfinity.systems.
-    # The `_local` variant runs in a network-isolated sandbox (no `requires-network`
-    # tag), so the download fails, replay never progresses, no checkpoint is ever
-    # archived, and the test times out. Supporting the local backend would require
-    # vendoring the mainnet `ic-replay`/sandbox binaries (none are available offline
-    # today) and copying them into `binaries/{initial_replica_version}/`.
-    backend = "farm",
     cpus = MIN_LOCAL_CPUS + 4 * DEFAULT_VCPUS_PER_VM,  # 4 IC Node VMs * 6 vCPUs.
+    env = MAINNET_ENV,
     guestos = "mainnet_nns",
     guestos_update = True,
     tags = [
@@ -69,28 +61,19 @@ system_test_nns(
     ],
     test_driver_target = ":backup_manager_test_bin",
     test_timeout = "eternal",  # this test often times out with the default 15 minute timeout so we allow more time
-    runtime_deps = BACKUP_RUNTIME_DEPS | {
-        "IC_VERSION_FILE": "//bazel:version.txt",
-    },
+    runtime_deps = BACKUP_RUNTIME_DEPS,
 )
 
 system_test_nns(
     name = "backup_manager_downgrade_test",
-    # Pinned to the Farm backend for the same reason as `backup_manager_upgrade_test`:
-    # `ic-backup` downloads the mainnet `ic-replay` binary (here for the
-    # `guestos_update = "mainnet_nns"` target version) from download.dfinity.systems,
-    # which the network-isolated `_local` variant cannot reach, so replay stalls and
-    # the test times out.
-    backend = "farm",
     cpus = MIN_LOCAL_CPUS + 4 * DEFAULT_VCPUS_PER_VM,  # 4 IC Node VMs * 6 vCPUs.
+    env = MAINNET_ENV,
     guestos_update = "mainnet_nns",
     tags = [
         "colocate",
         "long_test",  # since it takes longer than 5 minutes.
     ],
     test_driver_target = ":backup_manager_test_bin",
     test_timeout = "eternal",  # this test often times out with the default 15 minute timeout so we allow more time
-    runtime_deps = BACKUP_RUNTIME_DEPS | {
-        "IC_VERSION_FILE": "//bazel:version.txt",
-    },
+    runtime_deps = BACKUP_RUNTIME_DEPS,
 )

rs/tests/consensus/backup/common.rs

@@ -55,15 +55,36 @@ use ic_system_test_driver::{
 use ic_types::Height;
 use slog::{Logger, debug, error, info};
 use std::{
+    collections::BTreeSet,
     ffi::OsStr,
     fs,
     io::Write,
     net::IpAddr,
+    os::unix::fs::PermissionsExt,
     path::{Path, PathBuf},
     process::{Command, Stdio},
 };
 use std::{fs::File, time::Duration};
 
+/// (destination file name, runtime dependency env var) of the binaries `ic-backup`
+/// needs to replay a replica version, for the version built from this branch. See
+/// `BACKUP_RUNTIME_DEPS` in BUILD.bazel. `ic-replay` looks the three sandbox
+/// binaries up next to itself, which is why all four go into the same directory.
+const BRANCH_BINARIES: &[(&str, &str)] = &[
+    ("ic-replay", "IC_REPLAY_PATH"),
+    ("sandbox_launcher", "SANDBOX_LAUNCHER_PATH"),
+    ("canister_sandbox", "CANISTER_SANDBOX_PATH"),
+    ("compiler_sandbox", "COMPILER_SANDBOX_PATH"),
+];
+
+/// The same binaries, as published for the mainnet NNS subnet's replica version.
+const MAINNET_NNS_BINARIES: &[(&str, &str)] = &[
+    ("ic-replay", "MAINNET_NNS_IC_REPLAY_PATH"),
+    ("sandbox_launcher", "MAINNET_NNS_SANDBOX_LAUNCHER_PATH"),
+    ("canister_sandbox", "MAINNET_NNS_CANISTER_SANDBOX_PATH"),
+    ("compiler_sandbox", "MAINNET_NNS_COMPILER_SANDBOX_PATH"),
+];
+
 const DKG_INTERVAL: u64 = 29;
 const SUBNET_SIZE: usize = 4;
 const DIVERGENCE_LOG_STR: &str = "The state hash of the CUP at height ";
@@ -139,34 +160,53 @@ pub fn test(env: TestEnv) {
     let initial_replica_version =
         get_assigned_replica_version(&nns_node).expect("There should be assigned replica version");
 
-    info!(
-        log,
-        "Copy the binaries needed for replay of the current version"
-    );
-    let backup_binaries_dir = backup_dir.join("binaries").join(binary_version.to_string());
-    fs::create_dir_all(&backup_binaries_dir).expect("failure creating backup binaries directory");
-
-    // Copy all the binaries needed for the replay of the current version in order to avoid downloading them
-    copy_file(
-        &get_dependency_path_from_env("IC_REPLAY_PATH"),
-        &backup_binaries_dir,
-        "ic-replay",
-    );
-    copy_file(
-        &get_dependency_path_from_env("SANDBOX_LAUNCHER_PATH"),
-        &backup_binaries_dir,
-        "sandbox_launcher",
-    );
-    copy_file(
-        &get_dependency_path_from_env("CANISTER_SANDBOX_PATH"),
-        &backup_binaries_dir,
-        "canister_sandbox",
-    );
-    copy_file(
-        &get_dependency_path_from_env("COMPILER_SANDBOX_PATH"),
-        &backup_binaries_dir,
-        "compiler_sandbox",
-    );
+    // `ic-backup` replays each replica version's artifacts with that version's own
+    // binaries and downloads them from download.dfinity.systems when they are not
+    // already in `binaries/<version>/`. These tests run without external network
+    // access on the local backend, so every version that will be replayed has to be
+    // provided up front:
+    //
+    //   * backup_manager_upgrade_test:   mainnet NNS (initial) -> branch (target)
+    //   * backup_manager_downgrade_test: branch (initial)      -> mainnet NNS (target)
+    //
+    // Both are covered by seeding the initial, the target and the branch version. A
+    // version we have no binaries for fails here instead of stalling much later on a
+    // download that cannot succeed.
+    let mainnet_version =
+        get_mainnet_nns_revision().expect("could not read the mainnet NNS revision");
+    for version in BTreeSet::from([
+        binary_version.clone(),
+        initial_replica_version.clone(),
+        target_version.clone(),
+    ]) {
+        let binaries = if version == binary_version {
+            BRANCH_BINARIES
+        } else if version == mainnet_version {
+            MAINNET_NNS_BINARIES
+        } else {
+            panic!(
+                "No `ic-replay` available for replica version {version}. This test provides the \
+                 branch version ({binary_version}) and the mainnet NNS version ({mainnet_version}) \
+                 as bazel dependencies and cannot download any other version."
+            );
+        };
+        let backup_binaries_dir = backup_dir.join("binaries").join(version.to_string());
+        fs::create_dir_all(&backup_binaries_dir)
+            .expect("failure creating backup binaries directory");
+        info!(
+            log,
+            "Copying the {} binaries needed for replay to {}",
+            version,
+            backup_binaries_dir.display()
+        );
+        for (file_name, env_var) in binaries {
+            copy_file(
+                &get_dependency_path_from_env(env_var),
+                &backup_binaries_dir,
+                file_name,
+            );
+        }
+    }
 
     // Generate keypair and store the private key
     info!(log, "Create backup user credentials");
@@ -488,7 +528,23 @@ fn dir_exists_and_have_file(log: &Logger, dir: &PathBuf) -> bool {
 }
 
 fn copy_file(binary_path: &Path, backup_binaries_dir: &Path, file_name: &str) {
-    fs::copy(binary_path, backup_binaries_dir.join(file_name)).expect("failed to copy file");
+    // A dependency that lost its executable bit would only surface much later, as
+    // `ic-replay` failing to spawn its sandbox, so check it here at the source.
+    let mode = fs::metadata(binary_path)
+        .unwrap_or_else(|e| panic!("failed to stat {binary_path:?}: {e}"))
+        .permissions()
+        .mode();
+    assert!(
+        mode & 0o111 != 0,
+        "{binary_path:?} is not executable (mode {mode:o}); the bazel dependency providing it \
+         must produce an executable file"
+    );
+    let target = backup_binaries_dir.join(file_name);
+    fs::copy(binary_path, &target).expect("failed to copy file");
+    // `fs::copy` inherits the source's mode and bazel outputs are read-only, so make
+    // the copy writable in case anything wants to replace it.
+    fs::set_permissions(&target, fs::Permissions::from_mode(0o755))
+        .expect("failed to set the permissions of the copied binary");
 }
 
 fn highest_dir_entry(dir: &PathBuf, radix: u32) -> u64 {