diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40ff709943..9250dbc323 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -464,6 +464,18 @@ jobs: container_image: base limit_to_owner: "" main_pr_only: false + - name: Test axvisor aarch64 qemu (http control-plane) + use_container: false + runs_on: '["self-hosted","linux","qcs"]' + self_hosted_owner: rcore-os + timeout_minutes: 30 + command: | + cargo xtask image pull qemu-aarch64 --output-dir tmp/axbuild/images + cargo xtask axvisor test qemu --arch aarch64 --test-case http-control-plane + cache_key: "" + container_image: base + limit_to_owner: "" + main_pr_only: false - name: Test axvisor aarch64 qemu (panic modes) use_container: false runs_on: '["self-hosted","linux","qcs"]' diff --git a/Cargo.lock b/Cargo.lock index 6cfb76858d..5003bb7efc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1561,6 +1561,7 @@ dependencies = [ "axdevice", "axdevice_base", "axtest", + "axum", "axvirtio-blk", "axvirtio-common", "axvirtio-net", @@ -1572,6 +1573,7 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", + "serde_json", "shlex 2.0.1", "syn 3.0.3", "tokio", diff --git a/os/axvisor/Cargo.toml b/os/axvisor/Cargo.toml index 60a929366e..3c3cad57a2 100644 --- a/os/axvisor/Cargo.toml +++ b/os/axvisor/Cargo.toml @@ -47,6 +47,17 @@ stack-protector = ["ax-std/stack-protector"] backtrace = ["ax-std/backtrace", "dep:axbacktrace"] test-backtrace-panic = ["backtrace"] test-panic-no-backtrace = ["dep:axbacktrace"] +# axum-based management HTTP server (see src/http/ for the Router). +# Off by default. Replaces the hand-rolled pilot, which is intentionally not +# carried forward. +# axum-based management HTTP server. Pulls `ax-std/net` explicitly so the +# ArceOS network stack (which probes the QEMU virtio-net NIC and backs the +# tokio `TcpListener`) is enabled only when the control plane is built; a +# non-HTTP build must not enable the network subsystem. +http-axum = ["ax-std/net", "dep:axum", "dep:tokio", "dep:serde_json"] +# Do not auto-boot the default VMs at startup; the HTTP control plane starts +# and stops them on demand (VMs are created and stay in `Ready`). +no-auto-start = [] [dependencies] shlex.workspace = true @@ -57,6 +68,12 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(axtest)', 'cfg(feature, va [target.'cfg(any(not(any(windows, unix)), target_env = "musl"))'.dependencies] anyhow.workspace = true log = "0.4" +axum = { version = "0.8", optional = true } +serde_json = { version = "1", optional = true } +# The runtime only enables the IO driver (`enable_io()`), so no `time` driver +# and thus no `timerfd` syscall is needed. `rt` + `net` cover the manually +# built current-thread runtime and `TcpListener`. +tokio = { version = "1", optional = true, features = ["rt", "net"] } # System dependent modules provided by ArceOS. ax-api.workspace = true diff --git a/os/axvisor/src/guest_console/mod.rs b/os/axvisor/src/guest_console/mod.rs index d7158dbff0..361749275c 100644 --- a/os/axvisor/src/guest_console/mod.rs +++ b/os/axvisor/src/guest_console/mod.rs @@ -4,7 +4,15 @@ mod host; mod mux; pub(crate) use host::{configure_host_console_reader, read_host_byte, wait_for_host_input}; +#[cfg_attr( + feature = "no-auto-start", + expect( + unused_imports, + reason = "only the auto-start boot path attaches the console to a default running VM" + ) +)] +pub(crate) use mux::attach_default; pub(crate) use mux::{ - ConsoleInputEvent, activate, attach, attach_default, attached_vm, mark_running, mark_stopped, + ConsoleInputEvent, activate, attach, attached_vm, mark_running, mark_stopped, reconcile_vm_states, remove, route_host_byte, serial_backend_factory, }; diff --git a/os/axvisor/src/guest_console/mux/mod.rs b/os/axvisor/src/guest_console/mux/mod.rs index 82ea2d977a..0897efc4df 100644 --- a/os/axvisor/src/guest_console/mux/mod.rs +++ b/os/axvisor/src/guest_console/mux/mod.rs @@ -472,6 +472,13 @@ pub fn route_host_byte(byte: u8) -> ConsoleInputEvent { } /// Attach the lowest-ID member of the default running VM set. +#[cfg_attr( + feature = "no-auto-start", + expect( + dead_code, + reason = "only the auto-start boot path attaches the console to a default running VM" + ) +)] pub fn attach_default(running: impl IntoIterator) -> Option { GUEST_CONSOLE_MUX.attach_default(running) } diff --git a/os/axvisor/src/http/auth.rs b/os/axvisor/src/http/auth.rs new file mode 100644 index 0000000000..097669836c --- /dev/null +++ b/os/axvisor/src/http/auth.rs @@ -0,0 +1,63 @@ +//! Bearer-token access control for the management HTTP control plane. +//! +//! Mutating routes (`create`/`delete`/`start`/`stop`) require an +//! `Authorization: Bearer ` header matching the build-time token. The +//! token is baked into the image at +//! build time from the `[env] AXVM_HTTP_TOKEN` build-config variable — the same +//! `option_env!` mechanism `crate::shell::command::base` uses for `AX_ARCH`. +//! +//! The control plane is **deny-by-default**: if `AXVM_HTTP_TOKEN` is unset, +//! every protected route returns `401` and cannot be used. There is no +//! "fall back to allowing writes without a token" path — a build that forgets +//! the token fails its tests instead of silently exposing EL2 state changes. +//! Read-only routes (`GET`) are intentionally left open; they expose no state +//! mutation, and the default loopback bind (see [`crate::http::server`]) keeps +//! them off the management network unless an operator explicitly opts in. + +use axum::{ + extract::FromRequestParts, + http::{ + StatusCode, + header::{AUTHORIZATION, HeaderValue}, + }, +}; + +/// A request that carries a matching `Authorization: Bearer ` header. +/// +/// Attach as the first extractor on a mutating handler. Rejects the request +/// with `401 Unauthorized` when no token was baked into the image +/// (`AXVM_HTTP_TOKEN` unset) or the header is missing / does not match. +pub struct ApiToken; + +impl ApiToken { + /// Whether the given header value carries the required bearer token. + fn header_matches(value: &HeaderValue) -> bool { + let Some(token) = option_env!("AXVM_HTTP_TOKEN") else { + return false; + }; + value.to_str().ok().is_some_and(|value| { + value + .strip_prefix("Bearer ") + .is_some_and(|rest| rest == token) + }) + } +} + +impl FromRequestParts for ApiToken { + type Rejection = StatusCode; + + async fn from_request_parts( + parts: &mut axum::http::request::Parts, + _state: &S, + ) -> Result { + let authorized = parts + .headers + .get(AUTHORIZATION) + .is_some_and(Self::header_matches); + if authorized { + Ok(ApiToken) + } else { + Err(StatusCode::UNAUTHORIZED) + } + } +} diff --git a/os/axvisor/src/http/mod.rs b/os/axvisor/src/http/mod.rs new file mode 100644 index 0000000000..9534e9c8ac --- /dev/null +++ b/os/axvisor/src/http/mod.rs @@ -0,0 +1,25 @@ +//! Management HTTP control plane. +//! +//! Served by an axum `Router` running on a tokio current-thread runtime +//! (see [`server`]). The VM list/detail and start/stop lifecycle routes live +//! in [`vm`]. JSON is built with `serde_json`. +//! +//! Security boundary: mutating routes require a build-time bearer token +//! ([`auth`]); the server binds `127.0.0.1:8080` by default and only binds +//! wider when `[env] AXVM_HTTP_BIND` opts in. See the per-module docs. +//! +//! This whole module is only compiled under the `http-axum` feature, which is +//! off by default. The hand-rolled HTTP/1.0 pilot was intentionally not +//! carried forward. + +pub mod auth; +pub mod server; +pub mod vm; + +/// Blocking entry point for the management HTTP server. +/// +/// Spawned on its own task (see `crate::main`); builds the tokio runtime and +/// serves until the hypervisor shuts down. +pub fn serve() { + server::serve(); +} diff --git a/os/axvisor/src/http/server.rs b/os/axvisor/src/http/server.rs new file mode 100644 index 0000000000..b1703cf318 --- /dev/null +++ b/os/axvisor/src/http/server.rs @@ -0,0 +1,67 @@ +//! axum-based management HTTP server (`http-axum` feature). +//! +//! Runs an axum `Router` on a tokio current-thread runtime and serves the +//! management API. Routes and JSON fields mirror the hand-rolled pilot's API, +//! but dispatch and JSON construction are delegated to axum + serde_json. +//! +//! ```text +//! GET /api/vms → 200, JSON array (summary form) +//! GET /api/vms/{id} → 200, JSON detail (with vcpu_states) | 404 +//! POST /api/vms/create → 200 {"id":N} | 400 | 409 | 500 (body {"toml": "..."}) +//! DELETE /api/vms/{id} → 204 | 404 | 500 +//! POST /api/vms/{id}/start → 200 {"ok":true,"status":...} | 404 | 409 | 503 +//! POST /api/vms/{id}/stop → 200 {"ok":true,"status":...} | 404 | 409 | 503 +//! ``` +//! +//! Mutating routes (`create`/`delete`/`start`/`stop`) require +//! `Authorization: Bearer ` with the build-time `[env] AXVM_HTTP_TOKEN`; +//! see [`crate::http::auth`]. GET routes are open. The listener binds +//! [`bind_addr`], loopback by default. +//! +//! The tokio reactor is initialized with `enable_io()` only (no time driver), +//! which needs only epoll, so no `timerfd` syscall is required. + +use axum::{Router, routing::get, routing::post}; + +use crate::http::vm; + +/// Assemble the management routes. +pub fn router() -> Router { + Router::new() + .route("/api/vms", get(vm::list_vms)) + .route("/api/vms/{id}", get(vm::vm_detail).delete(vm::vm_delete)) + .route("/api/vms/create", post(vm::vm_create)) + .route("/api/vms/{id}/start", post(vm::vm_start)) + .route("/api/vms/{id}/stop", post(vm::vm_stop)) +} + +/// Bind address for the management HTTP server. +/// +/// Defaults to loopback (`127.0.0.1:8080`) so a stock `http-axum` build is not +/// reachable from the management network. Test/dev flows that need QEMU +/// hostfwd to reach the in-guest listener must opt in to all interfaces by +/// setting `[env] AXVM_HTTP_BIND = "0.0.0.0:8080"` in their build config; the +/// mutating routes still require the bearer token regardless of the bind. +fn bind_addr() -> &'static str { + option_env!("AXVM_HTTP_BIND").unwrap_or("127.0.0.1:8080") +} + +/// Blocking serve: build a tokio current-thread runtime and hand it to axum. +/// +/// `main` spawns this on its own task via `std::thread::spawn(|| http::serve())`; +/// the runtime is built here. Only the IO driver is enabled — the epoll +/// reactor suffices for `axum::serve`; a time driver would need `timerfd`. +pub fn serve() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_io() + .build() + .expect("failed to build tokio runtime"); + rt.block_on(async { + let bind = bind_addr(); + let listener = tokio::net::TcpListener::bind(bind) + .await + .expect("failed to bind management HTTP server"); + info!("management HTTP server (axum) listening on {bind}"); + axum::serve(listener, router()).await.expect("server error"); + }); +} diff --git a/os/axvisor/src/http/vm.rs b/os/axvisor/src/http/vm.rs new file mode 100644 index 0000000000..fcdcac4617 --- /dev/null +++ b/os/axvisor/src/http/vm.rs @@ -0,0 +1,226 @@ +//! VM status, lifecycle, and create/delete axum handlers. +//! +//! JSON is built with `serde_json::json!()` (no hand-written escaping). These +//! handlers are dispatched by the TCP serving path in [`super::server`]. + +use axum::{Json, extract::Path, http::StatusCode}; +use axvm::{AxVMRef, AxVmError, VmStatus, VmVcpuState}; +use axvmconfig::GuestConfig; +use serde_json::{Value, json}; + +use crate::http::auth::ApiToken; +use crate::manager::AxvmManager; + +/// `GET /api/vms` — list all known VMs (summary form). +pub async fn list_vms() -> Json> { + let items: Vec = AxvmManager::vm_list().iter().map(vm_json_summary).collect(); + Json(items) +} + +/// `GET /api/vms/{id}` — detail for one VM, or 404 if unknown. +pub async fn vm_detail(Path(id_str): Path) -> Result, StatusCode> { + let Ok(id) = id_str.parse::() else { + return Err(StatusCode::NOT_FOUND); + }; + match AxvmManager::vm_by_id(id) { + Some(vm) => Ok(Json(vm_json(&vm, true))), + None => Err(StatusCode::NOT_FOUND), + } +} + +/// `POST /api/vms/create` — create a VM from a TOML config in the JSON body. +/// +/// Body: `{"toml": "<完整 TOML 配置>"}`. The guest kernel must be a build-time +/// embedded image (`image_location = "memory"`) whose id matches the config's +/// `base.id`, and that id must not currently be registered. Because embedded +/// images are matched by id (`memory_images_for_vm`), a config whose id has no +/// embedded image fails with 500 — the runtime can only realize guest images +/// that were baked into the hypervisor at build time. +pub async fn vm_create( + _token: ApiToken, + Json(payload): Json, +) -> Result, StatusCode> { + let toml = payload + .get("toml") + .and_then(Value::as_str) + .ok_or(StatusCode::BAD_REQUEST)?; + let config = GuestConfig::from_toml(toml).map_err(|_| StatusCode::BAD_REQUEST)?; + let id = config.base.id; + // Explicit duplicate check: `create_vm_from_toml` fails on a re-registered id + // with a plain anyhow string, so surface the conflict as a contract error + // (409) instead of an opaque 500. + if AxvmManager::vm_by_id(id).is_some() { + return Err(StatusCode::CONFLICT); + } + match AxvmManager::create_vm_from_toml(toml) { + Ok(id) => { + info!("HTTP: VM[{id}] created via control API"); + Ok(Json(json!({ "id": id }))) + } + Err(error) => { + error!("HTTP: create VM[{id}] failed: {error:#}"); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } +} + +/// `DELETE /api/vms/{id}` — destroy and unregister a VM. +/// +/// Two explicit steps so a failed destroy stays retryable: `destroy()` first +/// (its result is checked), and the registry is only touched on success. This +/// avoids relying on `Drop`-time destroy, which merely warns on failure after +/// the VM is already unregistered, leaving no handle to retry with. +pub async fn vm_delete( + _token: ApiToken, + Path(id_str): Path, +) -> Result { + let Ok(id) = id_str.parse::() else { + return Err(StatusCode::NOT_FOUND); + }; + let vm = AxvmManager::vm_by_id(id).ok_or(StatusCode::NOT_FOUND)?; + vm.destroy() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + AxvmManager::remove_vm(id).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + info!("HTTP: VM[{id}] removed via control API"); + Ok(StatusCode::NO_CONTENT) +} + +/// `POST /api/vms/{id}/start` — start a VM. +pub async fn vm_start( + _token: ApiToken, + Path(id_str): Path, +) -> Result, StatusCode> { + vm_action(&id_str, VmAction::Start) +} + +/// `POST /api/vms/{id}/stop` — request a VM stop. +/// +/// `stop` has request semantics: it returns as soon as the request is accepted, +/// while the vCPU exits and the VM reaches `Stopped` asynchronously. +pub async fn vm_stop( + _token: ApiToken, + Path(id_str): Path, +) -> Result, StatusCode> { + vm_action(&id_str, VmAction::Stop) +} + +/// A lifecycle action on a VM. +enum VmAction { + Start, + Stop, +} + +/// Drive one lifecycle action, mapping host errors to HTTP status codes. +/// +/// Unknown VMs yield 404, invalid lifecycle transitions yield 409, and host +/// resource exhaustion yields 503. +fn vm_action(id_str: &str, action: VmAction) -> Result, StatusCode> { + let Ok(id) = id_str.parse::() else { + return Err(StatusCode::NOT_FOUND); + }; + // No existence pre-check: an unknown VM surfaces as `VmNotFound` from the + // action and maps to 404 below, keeping the check-then-act window closed. + // Restart-after-stop is not supported: a fresh vCPU task on an idled pinned + // CPU is never scheduled (no IPI wake source), so `start_vm` would accept + // the start and leave the VM stuck in `Running`. Reject it explicitly so the + // limitation is a contract error rather than an implicit hang. + if matches!(action, VmAction::Start) + && AxvmManager::vm_by_id(id).is_some_and(|vm| vm.status() == VmStatus::Stopped) + { + return Err(StatusCode::CONFLICT); + } + let result = match action { + VmAction::Start => AxvmManager::start_vm(id), + VmAction::Stop => AxvmManager::stop_vm(id), + }; + match result { + Ok(()) => Ok(Json(vm_action_json(id, action))), + Err(error) => Err(map_axvm_error(error)), + } +} + +/// Report the VM status right after a lifecycle action was accepted. +/// +/// `stop` is a request: the `Stopped` state arrives only once the vCPU observes +/// the request and exits asynchronously, so the reported status may still be +/// `running`/`stopping`. The `"async": true` marker makes that explicit so +/// callers do not mistake the accepted-request response for a completed stop. +fn vm_action_json(id: usize, action: VmAction) -> Value { + let status = AxvmManager::vm_by_id(id) + .map(|vm| vm.status().as_str()) + .unwrap_or("unknown"); + json!({ + "ok": true, + "status": status, + "async": matches!(action, VmAction::Stop), + }) +} + +/// Map an AxVM runtime error to an HTTP status code. +fn map_axvm_error(error: anyhow::Error) -> StatusCode { + let cause = error.root_cause(); + match cause.downcast_ref::() { + // A lifecycle transition that the current state does not allow. + Some(AxVmError::InvalidTransition { .. } | AxVmError::InvalidState { .. }) => { + StatusCode::CONFLICT + } + // Host resources (memory, vCPU list, devices, ...) were unavailable. + Some(AxVmError::OutOfMemory { .. } | AxVmError::ResourceUnavailable { .. }) => { + StatusCode::SERVICE_UNAVAILABLE + } + // Unknown VMs surface as `VmNotFound` from the action (there is no + // existence pre-check), mapping to 404. Anything else is a host-side + // fault. + Some(AxVmError::VmNotFound { .. }) => StatusCode::NOT_FOUND, + _ => { + error!("management HTTP action failed: {error:#}"); + StatusCode::INTERNAL_SERVER_ERROR + } + } +} + +fn vm_json_summary(vm: &AxVMRef) -> Value { + vm_json(vm, false) +} + +fn vm_json(vm: &AxVMRef, with_vcpus: bool) -> Value { + let memory_mb = vm + .memory_regions() + .iter() + .fold(0usize, |acc, region| acc.saturating_add(region.size())) + / (1024 * 1024); + let mut json = json!({ + "id": vm.id(), + "name": vm.name(), + "status": vm.status().as_str(), + "cpu_num": vm.vcpu_num(), + "memory_mb": memory_mb, + }); + if with_vcpus { + let vcpus: Vec = vm + .vcpu_snapshots() + .iter() + .map(|vcpu| { + json!({ + "id": vcpu.id, + "state": vcpu_state_str(vcpu.state), + "phys_cpu_set": vcpu.phys_cpu_set, + }) + }) + .collect(); + json["vcpu_states"] = json!(vcpus); + } + json +} + +fn vcpu_state_str(state: VmVcpuState) -> &'static str { + match state { + VmVcpuState::Invalid => "invalid", + VmVcpuState::Created => "created", + VmVcpuState::Free => "free", + VmVcpuState::Ready => "ready", + VmVcpuState::Running => "running", + VmVcpuState::Blocked => "blocked", + VmVcpuState::Starting => "starting", + } +} diff --git a/os/axvisor/src/main.rs b/os/axvisor/src/main.rs index bf05901788..60e5af8fbf 100644 --- a/os/axvisor/src/main.rs +++ b/os/axvisor/src/main.rs @@ -32,6 +32,8 @@ use ax_std as _; mod banner; mod config; mod guest_console; +#[cfg(feature = "http-axum")] +mod http; mod manager; mod shell; mod virtio_blk; @@ -55,8 +57,12 @@ fn init_panic_hook() { /// /// 1. Print the startup banner. /// 2. Check and enable hardware virtualization on every CPU. -/// 3. Build and start configured guest VMs. -/// 4. Run the VM completion waiter and management console concurrently. +/// 3. Build the default guest VMs. +/// 4. Spawn the management plane first — the HTTP server so the API is live +/// before any guest boots — then the VM lifecycle waiter and the shell. +/// +/// The vCPU tasks are pinned to the secondary CPUs via `phys_cpu_ids` in the +/// VM configs, while the management console stays on the primary CPU. fn main() { #[cfg(any(feature = "backtrace", feature = "test-panic-no-backtrace"))] init_panic_hook(); @@ -76,18 +82,44 @@ fn main() { .unwrap_or_else(|error| panic!("failed to initialize AxVM manager: {error:#}")); manager.init_default_vms(); + + // The management HTTP server accepts connections in a loop and needs its + // own task so neither the shell nor the VMM blocks it. It is spawned first + // so the management API is ready as early as possible. `ax_std::thread::spawn` + // only enqueues the task — the main task keeps running until it yields or + // blocks — so the server's bind does not necessarily happen before + // `launch_default_vms` queues the vCPU tasks; the ordering is best-effort. + #[cfg(feature = "http-axum")] + std::thread::Builder::new() + .name("axvisor-http".into()) + .spawn(http::serve) + .unwrap_or_else(|error| panic!("failed to start management HTTP server: {error}")); + let default_vms = manager::AxvmManager::vm_list(); guest_console::configure_host_console_reader(&default_vms) .unwrap_or_else(|error| panic!("failed to configure host console input: {error:#}")); + + // With `no-auto-start` the default VMs are only created (staying in + // `Ready`) and the management plane boots them on demand, so nothing is + // launched or waited on here. + #[cfg(not(feature = "no-auto-start"))] let started_vms = manager.launch_default_vms(); + #[cfg(not(feature = "no-auto-start"))] guest_console::attach_default(started_vms); + #[cfg(not(feature = "no-auto-start"))] std::thread::Builder::new() .name("axvisor-vm-wait".into()) .spawn(manager::AxvmManager::wait_for_default_vms) .unwrap_or_else(|error| panic!("failed to start VM completion waiter: {error}")); + #[cfg(not(feature = "no-auto-start"))] info!("[OK] Default guest initialized"); + // The management console runs on the primary CPU (Core 0) while the vCPU + // tasks are pinned to Core 1 via `phys_cpu_ids`, so it stays responsive + // regardless of guest behavior. + info!("shell task on CPU{}", axvm::host::cpu::current_id()); + shell::console_init(); } diff --git a/os/axvisor/src/manager.rs b/os/axvisor/src/manager.rs index 5c7f5ef6f9..2a9c9ac317 100644 --- a/os/axvisor/src/manager.rs +++ b/os/axvisor/src/manager.rs @@ -37,11 +37,25 @@ impl AxvmManager { } /// Start the default VM set without blocking the management console. + #[cfg_attr( + feature = "no-auto-start", + expect( + dead_code, + reason = "only the auto-start boot path launches the default VMs" + ) + )] pub fn launch_default_vms(&self) -> Vec { self.runtime.launch_default_vms() } /// Wait until every running VM has stopped. + #[cfg_attr( + feature = "no-auto-start", + expect( + dead_code, + reason = "only the auto-start boot path waits for default-VM completion" + ) + )] pub fn wait_for_default_vms() { AxvmRuntime::wait_for_all_vms(); } diff --git a/scripts/axbuild/src/axvisor/test/host_probe.rs b/scripts/axbuild/src/axvisor/test/host_probe.rs new file mode 100644 index 0000000000..32ff797045 --- /dev/null +++ b/scripts/axbuild/src/axvisor/test/host_probe.rs @@ -0,0 +1,297 @@ +//! Host-side probe runner for QEMU hostfwd integration tests. +//! +//! The probe is the reverse of the generic host fixture server +//! ([`crate::test::host_http`]): instead of serving host fixtures to the guest, +//! it acts as a *client* that dials the AxVisor management HTTP API running +//! *inside* the guest through QEMU user-mode networking +//! (`-netdev user,hostfwd=tcp::-:`). The concrete HTTP +//! requests and assertions live with the test-suit case as an executable probe +//! asset (see [`crate::axvisor::test::http_probe`], which executes the asset +//! and collects its exit code); this module only provides the orchestration: +//! wait for the forwarded port, invoke the probe, and store its result as the +//! verdict. Nothing in the hypervisor knows a test is running. +//! +//! When the probe finishes — pass or fail — the guard quits QEMU over the QMP +//! monitor socket the runner added (`-qmp unix:...,server=on,wait=off`), so the +//! QEMU process exits cleanly and the runner reads the stored verdict from the +//! guard as the test result. The runner owns the QEMU child, so the case +//! `timeout` remains the backstop if the probe or its QMP quit fails. + +use std::{ + net::TcpStream, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + mpsc, + }, + thread, + time::{Duration, Instant}, +}; + +use anyhow::{Context, bail}; + +use super::types::AxvisorHttpProbeConfig; + +/// The probe callback invoked by the guard once the forwarded port accepts +/// connections. Returns the verdict (`Ok` = pass, `Err` = fail). The probe is a +/// `FnOnce` so it may own everything it needs (base address, token, config +/// paths) and runs on the guard's worker thread. +pub(crate) type HostHttpProbeFn = Box anyhow::Result<()> + Send + 'static>; + +/// Sleep between readiness retries. +const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(100); +/// How long to keep retrying the QMP connect before giving up on quitting QEMU. +const QMP_CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(100); +const QMP_CONNECT_RETRIES: usize = 10; +/// How long to keep a QMP `quit` connection open waiting for QEMU to exit +/// before re-issuing `quit` on a fresh connection. +const QMP_EXIT_WAIT: Duration = Duration::from_secs(4); +/// Poll interval while waiting for QEMU to exit after a `quit`. +const QMP_READ_POLL_INTERVAL: Duration = Duration::from_millis(100); +/// How many times to re-issue `quit` before giving up and letting the case +/// timeout fail the run. QEMU drops a `quit` that arrives while the guest is +/// still tearing a VM down, so a stuck QEMU must be re-quit rather than left to +/// time out. +const QMP_QUIT_RETRIES: usize = 4; + +pub(crate) struct HostHttpProbeGuard { + stop: Arc, + result: Arc>>>, + thread: Option>, +} + +impl HostHttpProbeGuard { + /// Spawn the probe runner thread and return a guard that owns its + /// lifecycle. + /// + /// `probe` is the host-side probe callback (the typed HTTP assertions); + /// the guard waits for the forwarded port to accept connections, invokes + /// it, and stores its result as the verdict. `qmp_socket` is the path QEMU + /// binds from its `-qmp unix:...` argument; the guard connects to it after + /// the probe finishes to quit QEMU. When `None`, the guard only stores the + /// verdict and relies on the case timeout to end the run. + /// + /// `stop` is the shared abort flag the probe's poll loops check so a run + /// whose QEMU already failed (fail_regex match, timeout, spawn error) can + /// abort the probe thread on its next poll instead of waiting out the + /// deadline. The runner owns it: it stores `true` when the case is over. + pub(crate) fn start( + config: &AxvisorHttpProbeConfig, + host_port: u16, + case_name: &str, + qmp_socket: Option, + stop: Arc, + probe: HostHttpProbeFn, + ) -> anyhow::Result { + let addr = format!("127.0.0.1:{host_port}"); + let connect_timeout = Duration::from_secs(config.connect_timeout_secs); + let thread_stop = stop.clone(); + let result = Arc::new(Mutex::new(None)); + let thread_result = result.clone(); + let case_name = case_name.to_string(); + let (ready_tx, ready_rx) = mpsc::channel(); + + let thread_addr = addr.clone(); + let thread_case_name = case_name.clone(); + let thread = thread::spawn(move || { + let _ = ready_tx.send(()); + // The guard waits for the forwarded port (guest boot + network + // init); the probe then runs the HTTP assertions. The probe is + // consumed exactly once. + let verdict = (|| -> anyhow::Result<()> { + wait_for_port_ready(&thread_addr, connect_timeout, &thread_stop).with_context( + || { + format!( + "guest HTTP server never became reachable within {connect_timeout:?}" + ) + }, + )?; + probe() + })(); + *thread_result.lock().unwrap() = Some(verdict); + // Quit QEMU so a successful run ends promptly on the probe verdict + // instead of the serial-timeout path. The runner owns the QEMU + // child, so it decides whether QEMU actually exits: a `quit` that + // is ignored degrades to the case timeout, which then fails the + // run — a stuck QEMU must not be reported as a probe success. + if let Some(socket) = qmp_socket + && let Err(err) = request_qmp_quit(&socket) + { + eprintln!( + " host http probe: {thread_case_name}: failed to quit QEMU via QMP: {err:#}" + ); + } + }); + + if ready_rx.recv_timeout(Duration::from_secs(1)).is_err() { + stop.store(true, Ordering::Release); + bail!("host http probe for `{case_name}` did not become ready"); + } + + println!(" host http probe: {addr} -> guest:{}", config.guest_port); + Ok(Self { + stop, + result, + thread: Some(thread), + }) + } + + /// Take the probe's stored verdict, if the thread produced one. + /// + /// Called once, after QEMU has exited. The probe always stores a verdict + /// *before* it quits QEMU, so a clean QEMU exit implies a verdict exists. + pub(crate) fn take_result(&self) -> Option> { + self.result.lock().unwrap().take() + } +} + +impl Drop for HostHttpProbeGuard { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// Poll the forwarded host port until a TCP connection succeeds, the deadline +/// elapses, or a stop is requested. A successful connect means the guest's +/// network stack is up; the in-guest server may still be booting, so the probe +/// itself should retry its first request. +fn wait_for_port_ready( + addr: &str, + connect_timeout: Duration, + stop: &AtomicBool, +) -> anyhow::Result<()> { + let started = Instant::now(); + loop { + if stop.load(Ordering::Acquire) { + bail!("host http probe stopped"); + } + if started.elapsed() >= connect_timeout { + bail!("timed out after {connect_timeout:?}"); + } + if TcpStream::connect(addr).is_ok() { + return Ok(()); + } + thread::sleep(CONNECT_RETRY_INTERVAL); + } +} + +/// Quit QEMU by connecting to its QMP monitor socket and issuing `quit`. The +/// socket path comes from the `-qmp unix:...,server=on,wait=off` argument the +/// runner added. Returns once QEMU has begun exiting, or `Ok` after all retries +/// are exhausted (the runner's case timeout then fails the run — a stuck QEMU +/// must not be reported as a probe success). +/// +/// Two QEMU quirks shape this routine: +/// +/// - A `quit` that arrives while the guest is still tearing a VM down is +/// silently dropped (the QMP monitor stays responsive, but no shutdown +/// happens). The probe's final poll can see the VM removed from the HTTP layer +/// before the guest's `Dropping VM[..]` cleanup finishes, so the guard may +/// send `quit` into that window. To avoid every probe case hitting the case +/// timeout on this race, re-issue `quit` on a fresh connection if QEMU is +/// still alive after a wait window. +/// - QEMU only shuts down cleanly when the `quit` connection stays open: closing +/// it right after writing `quit` makes QEMU hang during exit while the guest +/// vCPU spins in a tight PL011 poll (TCG cannot preempt the translation block, +/// so the exit never completes). The guard therefore keeps the connection +/// alive and waits for the exit signal — EOF on the held stream, or the +/// listener socket refusing new connects. +#[cfg(unix)] +fn request_qmp_quit(socket: &Path) -> anyhow::Result<()> { + use std::{ + io::{ErrorKind, Read, Write}, + os::unix::net::UnixStream, + }; + + /// Connect with retries, reporting the last error if QEMU never bound the + /// socket. + fn connect_with_retries(socket: &Path) -> anyhow::Result { + let mut last_err = None; + for _ in 0..QMP_CONNECT_RETRIES { + match UnixStream::connect(socket) { + Ok(stream) => return Ok(stream), + Err(err) => { + last_err = Some(err); + thread::sleep(QMP_CONNECT_RETRY_INTERVAL); + } + } + } + bail!( + "failed to connect QMP socket {}: {}", + socket.display(), + last_err.as_ref().expect("at least one connect attempted") + ) + } + + /// Do the QMP handshake (greeting, capabilities) and write `quit`, leaving + /// the connection open. + fn qmp_handshake_quit(stream: &mut UnixStream) -> std::io::Result<()> { + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .ok(); + stream + .set_write_timeout(Some(Duration::from_millis(200))) + .ok(); + let mut buf = [0_u8; 512]; + let _ = stream.read(&mut buf); // QMP greeting + stream.write_all(b"{\"execute\":\"qmp_capabilities\"}\r\n")?; + buf.fill(0); + let _ = stream.read(&mut buf); // capabilities response + stream.write_all(b"{\"execute\":\"quit\"}\r\n")?; + stream.flush() + } + + /// Whether the socket path still accepts connections. A refused or missing + /// socket means QEMU closed its listener (exiting or exited). + fn socket_connectable(socket: &Path) -> bool { + UnixStream::connect(socket).is_ok() + } + + for _ in 0..QMP_QUIT_RETRIES { + let mut stream = connect_with_retries(socket)?; + if let Err(err) = qmp_handshake_quit(&mut stream) { + // A failed handshake usually means QEMU was already closing its + // listener (it began exiting from an earlier `quit`), so the run + // can end. Surface the error only when QEMU is demonstrably still + // listening. + if socket_connectable(socket) { + bail!("failed to send QMP quit: {err}"); + } + return Ok(()); + } + + // Keep the connection open and wait for QEMU to exit. EOF on the held + // stream, or the listener refusing connects, means QEMU is shutting + // down; a still-connectable listener after the wait window means QEMU + // dropped the `quit` (guest teardown in flight), so retry on a fresh + // connection. + let wait_started = Instant::now(); + loop { + if wait_started.elapsed() >= QMP_EXIT_WAIT { + break; + } + let mut buf = [0_u8; 512]; + match stream.read(&mut buf) { + Ok(0) => return Ok(()), // EOF: QEMU closed the connection + Err(err) if matches!(err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => { + if !socket_connectable(socket) { + return Ok(()); // listener gone: QEMU exiting/exited + } + } + Err(_) => return Ok(()), // connection reset: QEMU gone + Ok(_) => {} // SHUTDOWN event / response; keep waiting + } + thread::sleep(QMP_READ_POLL_INTERVAL); + } + } + Ok(()) +} + +#[cfg(not(unix))] +fn request_qmp_quit(_socket: &Path) -> anyhow::Result<()> { + bail!("QMP unix sockets are not supported on this host") +} diff --git a/scripts/axbuild/src/axvisor/test/http_probe.rs b/scripts/axbuild/src/axvisor/test/http_probe.rs new file mode 100644 index 0000000000..4b4ec45f97 --- /dev/null +++ b/scripts/axbuild/src/axvisor/test/http_probe.rs @@ -0,0 +1,279 @@ +//! Generic host-side probe runner for the AxVisor management HTTP control plane. +//! +//! Direction is host -> guest: the probe dials the axum management API running +//! *inside* the AxVisor guest through QEMU user-mode networking hostfwd, and +//! asserts the responses entirely host-side. The *test content* — the concrete +//! requests, fixtures, and assertions — lives with the test-suit case as an +//! executable probe asset (default `http_probe.py` in the case directory; see +//! [`AxvisorHttpProbeConfig::probe_script`](super::types::AxvisorHttpProbeConfig::probe_script)). +//! New HTTP scenarios or API-contract changes therefore edit the case asset, +//! never this crate. +//! +//! This module is generic orchestration only: resolve the probe asset, spawn it +//! once the forwarded port is reachable, and collect its exit code as the +//! verdict. The runner's +//! [`HostHttpProbeGuard`](super::host_probe::HostHttpProbeGuard) does the rest +//! of the orchestration: wait for the forwarded port, invoke this probe, store +//! its verdict, and quit QEMU over QMP. +//! +//! The probe asset is executed directly (its shebang selects the interpreter) +//! with the environment: +//! +//! ```text +//! AXVISOR_HTTP_BASE http://127.0.0.1: (forwarded) +//! AXVISOR_HTTP_TOKEN bearer token (may be empty) +//! AXVISOR_HTTP_CASE_DIR case directory (fixtures like `vm-memory.toml`) +//! AXVISOR_HTTP_CONNECT_TIMEOUT seconds for the initial reachability wait +//! AXVISOR_HTTP_REQUEST_TIMEOUT seconds per HTTP request +//! ``` +//! +//! Exit code 0 is a pass; any nonzero exit fails the case. The asset streams +//! its own progress to the runner's stdout/stderr so CI logs show the steps. + +use std::{ + path::Path, + process::{Child, Command, ExitStatus, Stdio}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread, + time::Duration, +}; + +use anyhow::{Context, bail}; + +use super::types::AxvisorHttpProbeConfig; + +/// Poll interval while waiting for the probe asset to exit. +const PROBE_EXIT_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Run the case's HTTP probe asset against one boot. +/// +/// `addr` is the forwarded host address (`127.0.0.1:`). `config` carries +/// the bearer token, timeouts, and the probe-asset name; `case_dir` locates +/// the asset (and its fixtures). `stop` is the shared abort flag: when the +/// runner marks the case over (QEMU failure, timeout), a still-running asset is +/// killed instead of waiting it out. +pub(crate) fn run( + addr: &str, + config: &AxvisorHttpProbeConfig, + case_dir: &Path, + stop: Arc, +) -> anyhow::Result<()> { + let script = case_dir.join(&config.probe_script); + ensure_probe_asset(&script)?; + println!( + " host http probe: running probe asset {}", + script.display() + ); + let mut child = spawn_probe_asset(&script, addr, config, case_dir)?; + match wait_probe_asset(&mut child, &stop) { + Some(status) if status.success() => Ok(()), + Some(status) => bail!( + "probe asset {} exited with code {}", + script.display(), + status.code().unwrap_or(-1) + ), + None => bail!("probe asset {} was killed", script.display()), + } +} + +/// Fail fast when the configured probe asset is missing, so a case that +/// references a nonexistent asset errors clearly instead of spawning a `not +/// found` and misreporting it as a probe failure. +fn ensure_probe_asset(script: &Path) -> anyhow::Result<()> { + if !script.is_file() { + bail!( + "probe asset {} does not exist; add it to the case directory (or set \ + [host_http_probe] probe_script)", + script.display() + ); + } + Ok(()) +} + +/// Spawn the probe asset with the forwarded base URL, token, and timeouts as +/// environment. The asset is executed directly so its shebang picks the +/// interpreter; stdout/stderr are inherited so CI logs show the asset's steps. +fn spawn_probe_asset( + script: &Path, + addr: &str, + config: &AxvisorHttpProbeConfig, + case_dir: &Path, +) -> anyhow::Result { + Command::new(script) + .env("AXVISOR_HTTP_BASE", format!("http://{addr}")) + .env( + "AXVISOR_HTTP_TOKEN", + config.token.clone().unwrap_or_default(), + ) + .env("AXVISOR_HTTP_CASE_DIR", case_dir) + .env( + "AXVISOR_HTTP_CONNECT_TIMEOUT", + config.connect_timeout_secs.to_string(), + ) + .env( + "AXVISOR_HTTP_REQUEST_TIMEOUT", + config.request_timeout_secs.to_string(), + ) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .with_context(|| format!("failed to spawn probe asset {}", script.display())) +} + +/// Wait for the probe asset to exit, killing it if the runner marks the case +/// over. Returns `Some(status)` on a normal exit and `None` if it was killed. +fn wait_probe_asset(child: &mut Child, stop: &AtomicBool) -> Option { + loop { + if stop.load(Ordering::Acquire) { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + if let Some(status) = child.try_wait().ok().flatten() { + return Some(status); + } + thread::sleep(PROBE_EXIT_POLL_INTERVAL); + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde::Deserialize; + + use super::{super::types::DEFAULT_PROBE_SCRIPT, *}; + + fn test_config(probe_script: PathBuf) -> AxvisorHttpProbeConfig { + AxvisorHttpProbeConfig { + guest_port: 8080, + connect_timeout_secs: 120, + request_timeout_secs: 5, + probe_script, + token: Some("t".into()), + } + } + + /// Parse a `[host_http_probe]` section like + /// [`load_axvisor_http_probe_config`](super::super::qemu::load_axvisor_http_probe_config). + fn parse_probe_section(toml_body: &str) -> AxvisorHttpProbeConfig { + #[derive(Deserialize)] + struct ProbeSection { + #[serde(default)] + host_http_probe: Option, + } + toml::from_str::(toml_body) + .expect("probe section parses") + .host_http_probe + .expect("host_http_probe present") + } + + /// Write an executable probe asset that records its environment and exits + /// with `code`. + #[cfg(unix)] + fn write_fixture_probe(dir: &Path, name: &str, code: i32) -> PathBuf { + use std::{fs, os::unix::fs::PermissionsExt}; + let path = dir.join(name); + let script = format!( + "#!/bin/sh\nprintf '%s' \ + \"$AXVISOR_HTTP_BASE|$AXVISOR_HTTP_TOKEN|$AXVISOR_HTTP_CASE_DIR\" > \ + \"$AXVISOR_HTTP_CASE_DIR/env.txt\"\nexit {code}\n" + ); + fs::write(&path, script).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + path + } + + #[test] + fn probe_script_defaults_to_http_probe_py() { + let config = parse_probe_section("[host_http_probe]\ntoken = \"t\"\n"); + assert_eq!(config.probe_script, PathBuf::from(DEFAULT_PROBE_SCRIPT)); + } + + #[test] + fn probe_script_is_configurable() { + let config = parse_probe_section("[host_http_probe]\nprobe_script = \"custom_probe.sh\"\n"); + assert_eq!(config.probe_script, PathBuf::from("custom_probe.sh")); + } + + #[cfg(unix)] + #[test] + fn run_executes_the_case_probe_asset_with_env() { + let dir = tempfile::tempdir().unwrap(); + let probe = write_fixture_probe(dir.path(), "http_probe.py", 0); + let config = test_config(PathBuf::from("http_probe.py")); + let stop = Arc::new(AtomicBool::new(false)); + + let result = run("127.0.0.1:12345", &config, dir.path(), stop); + + assert!(result.is_ok(), "probe asset should pass: {result:?}"); + // The generic mechanism really executed the asset and forwarded the + // env the asset needs to dial the guest API. + let recorded = std::fs::read_to_string(dir.path().join("env.txt")).unwrap(); + assert_eq!( + recorded, + "http://127.0.0.1:12345|t|".to_string() + &dir.path().to_string_lossy() + ); + assert!(probe.exists()); + } + + #[cfg(unix)] + #[test] + fn run_propagates_nonzero_probe_exit() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_probe(dir.path(), "http_probe.py", 1); + let config = test_config(PathBuf::from("http_probe.py")); + let stop = Arc::new(AtomicBool::new(false)); + + let error = run("127.0.0.1:12345", &config, dir.path(), stop).unwrap_err(); + assert!(error.to_string().contains("exited with code 1")); + } + + #[test] + fn run_rejects_a_missing_probe_asset() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(PathBuf::from("http_probe.py")); + let stop = Arc::new(AtomicBool::new(false)); + + let error = run("127.0.0.1:12345", &config, dir.path(), stop).unwrap_err(); + assert!(error.to_string().contains("does not exist")); + } + + #[cfg(unix)] + #[test] + fn run_kills_the_probe_asset_when_stop_is_requested() { + let dir = tempfile::tempdir().unwrap(); + write_fixture_probe(dir.path(), "http_probe.py", 0); + let config = test_config(PathBuf::from("http_probe.py")); + // The case is already over before the asset even starts. + let stop = Arc::new(AtomicBool::new(true)); + + let error = run("127.0.0.1:12345", &config, dir.path(), stop).unwrap_err(); + assert!(error.to_string().contains("was killed")); + } + + /// The generic mechanism must execute the actual case asset: the + /// `http-control-plane` test-suit case carries `http_probe.py` next to its + /// `qemu-aarch64.toml` and `vm-memory.toml` fixtures. This pins that + /// contract so a missing/renamed case asset fails this test, not the CI run. + #[test] + fn http_control_plane_case_carries_a_probe_asset() { + let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("..").join(".."); + let case_asset = workspace_root.join( + "test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/http_probe.py", + ); + assert!( + case_asset.is_file(), + "http-control-plane case missing probe asset: {}", + case_asset.display() + ); + // The default `[host_http_probe]` config resolves the asset by name, so + // the generic runner executes the real case asset unchanged. + let name = case_asset.file_name().and_then(|s| s.to_str()).unwrap(); + assert_eq!(name, DEFAULT_PROBE_SCRIPT); + } +} diff --git a/scripts/axbuild/src/axvisor/test/mod.rs b/scripts/axbuild/src/axvisor/test/mod.rs index 44ef5f019f..1878cda3ef 100644 --- a/scripts/axbuild/src/axvisor/test/mod.rs +++ b/scripts/axbuild/src/axvisor/test/mod.rs @@ -1,6 +1,8 @@ mod assets; mod board; mod discovery; +mod host_probe; +mod http_probe; mod initramfs; mod ovmf; mod qemu; diff --git a/scripts/axbuild/src/axvisor/test/qemu.rs b/scripts/axbuild/src/axvisor/test/qemu.rs index 7862ac230c..e3195a6a57 100644 --- a/scripts/axbuild/src/axvisor/test/qemu.rs +++ b/scripts/axbuild/src/axvisor/test/qemu.rs @@ -1,11 +1,13 @@ use std::{ collections::BTreeMap, path::{Path, PathBuf}, + sync::{Arc, atomic::AtomicBool}, time::Instant, }; use anyhow::Context; use ostool::{build::config::Cargo, run::qemu::QemuConfig}; +use serde::Deserialize; use super::{ AXVISOR_NORMAL_GROUP, AxvisorQemuCase, @@ -14,9 +16,10 @@ use super::{ discovery::{ discover_test_group_names, qemu_list_error_is_ignorable, test_suite_dir, test_suite_root, }, + host_probe, initramfs::prepare_configured_busybox_initramfs, parse_target, - types::PreparedAxvisorQemuCase, + types::{AxvisorHttpProbeConfig, PreparedAxvisorQemuCase}, }; use crate::{ axvisor::{ArgsTestQemu, Axvisor, build, rootfs}, @@ -335,10 +338,82 @@ impl Axvisor { asset_config: &test_case::CaseAssetConfig, ) -> anyhow::Result<()> { let prepare_started = Instant::now(); - let (qemu, prepared_assets) = self + let (mut qemu, prepared_assets) = self .load_qemu_case_config(request, case, asset_config) .await?; - test_case::run_qemu_with_prepared_case_assets( + + // Optional host->guest TCP probe over QEMU user-mode networking. When + // `[host_http_probe]` is configured, the host acts as a *client* that + // dials a management API inside the guest through a hostfwd port and + // asserts the responses entirely host-side. The concrete requests, + // fixtures, and assertions live with the test-suit case as an + // executable probe asset (default `http_probe.py` in the case + // directory); axbuild only orchestrates: forward the port, execute the + // asset, collect its exit code, and report the result. The guard must + // live for the whole run, so it is spawned here and dropped at scope + // end (after QEMU exits). + // + // The guard also ends the run: after it stores its verdict it connects + // to a QMP monitor socket and sends `quit`, so a successful run ends + // on the probe result instead of the serial-timeout path. The runner + // owns the QEMU child, so a `quit` QEMU ignores degrades to the case + // timeout and fails the run (no `/__probe_result` relay inside the + // guest). + let mut host_probe_guard = None; + if let Some(probe_config) = + load_axvisor_http_probe_config(&case.case.case.qemu_config_path)? + { + let host_port = pick_free_local_port()?; + let qmp_socket = std::env::temp_dir().join(format!( + "axvisor-qmp-{}-{}.sock", + case.case.case.name, + std::process::id() + )); + // Each QEMU option and its value must be a separate argv element + // (QEMU takes the value of `-netdev`/`-device` from the following + // argument), matching how the `.toml` config stores them. + qemu.args.extend([ + "-netdev".to_string(), + format!( + "user,id=net0,hostfwd=tcp::{host_port}-:{}", + probe_config.guest_port + ), + "-device".to_string(), + "virtio-net-pci,netdev=net0".to_string(), + "-qmp".to_string(), + format!("unix:{},server=on,wait=off", qmp_socket.to_string_lossy()), + ]); + // Stop flag shared with the probe thread's poll loops: the runner + // stores `true` when the case is over (QEMU failure, timeout, or + // the guard's Drop), so the probe aborts on its next poll instead + // of waiting out its deadline. + let stop = Arc::new(AtomicBool::new(false)); + // The probe asset owns the concrete test content (fixtures such as + // `vm-memory.toml` and the assertions) in the case directory; the + // guard stays orchestration-only. + let probe_addr = format!("127.0.0.1:{host_port}"); + let probe_owned = probe_config.clone(); + let probe_case_dir = case.case.case.case_dir.clone(); + let probe_stop = stop.clone(); + let probe: host_probe::HostHttpProbeFn = Box::new(move || { + super::http_probe::run(&probe_addr, &probe_owned, &probe_case_dir, probe_stop) + }); + host_probe_guard = Some(host_probe::HostHttpProbeGuard::start( + &probe_config, + host_port, + &case.case.case.name, + Some(qmp_socket), + stop, + probe, + )?); + } + + // Both conditions must hold for a probe case: QEMU must run cleanly (a + // fail_regex match, terminal timeout, or spawn/exit error fails the run + // even when the probe passed), and the HTTP probe verdict must be + // `Ok`. For non-probe cases the serial-success path in + // `run_qemu_with_prepared_case_assets` still applies unchanged. + let qemu_result = test_case::run_qemu_with_prepared_case_assets( &mut self.app, cargo, qemu, @@ -350,10 +425,71 @@ impl Axvisor { qemu_timing_fields: None, }, ) - .await + .await; + + // Joins the probe thread now that QEMU has exited. + let probe_configured = host_probe_guard.is_some(); + let probe_result = host_probe_guard + .as_ref() + .and_then(|guard| guard.take_result()); + drop(host_probe_guard); + + combine_results(qemu_result, probe_configured, probe_result) + } +} + +/// Combine the QEMU runner result and the HTTP probe verdict into the final +/// case result. Both conditions must succeed: `qemu_result` first, then the +/// probe verdict. A fail_regex match, terminal timeout, or QEMU spawn/exit +/// failure therefore fails the run even when the probe passed — the probe may +/// only contribute its verdict once QEMU has run cleanly (e.g. exited via the +/// probe's QMP `quit`). +fn combine_results( + qemu_result: anyhow::Result<()>, + probe_configured: bool, + probe_result: Option>, +) -> anyhow::Result<()> { + qemu_result?; + + match (probe_configured, probe_result) { + (false, _) => Ok(()), + (true, Some(result)) => result, + (true, None) => anyhow::bail!("host http probe produced no verdict"), } } +/// Pick a free loopback port for the QEMU hostfwd listen, then release it so +/// QEMU can bind it. A freshly-assigned ephemeral port avoids stale-port +/// collisions from CI runner reuse (the same ports are never parked on a +/// previous run's leftover QEMU). A small bind-release-bind TOCTOU window +/// exists but is acceptable for a local test harness. +fn pick_free_local_port() -> anyhow::Result { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .context("failed to pick a free local port for QEMU hostfwd")?; + Ok(listener.local_addr()?.port()) +} + +/// Parse the optional `[host_http_probe]` section from an Axvisor qemu case +/// config. The section is axvisor-specific, so it is read directly from the +/// case toml here instead of going through the generic +/// [`test_qemu::load_qemu_case_extra_config`] (which no longer carries the +/// field). +fn load_axvisor_http_probe_config( + qemu_config_path: &Path, +) -> anyhow::Result> { + #[derive(Deserialize)] + struct ProbeSection { + #[serde(default)] + host_http_probe: Option, + } + + let content = std::fs::read_to_string(qemu_config_path) + .with_context(|| format!("failed to read {}", qemu_config_path.display()))?; + Ok(toml::from_str::(&content) + .with_context(|| format!("failed to parse {}", qemu_config_path.display()))? + .host_http_probe) +} + fn axvisor_qemu_test_build_args(arch: &str, config: Option) -> AxvisorCliArgs { AxvisorCliArgs { config, @@ -430,3 +566,87 @@ pub(super) fn plan_qemu_case_artifacts<'case, 'artifact, T>( }) .collect()) } + +#[cfg(test)] +mod tests { + use super::{combine_results, load_axvisor_http_probe_config}; + + fn ok() -> anyhow::Result<()> { + Ok(()) + } + + fn err(message: &str) -> anyhow::Result<()> { + Err(anyhow::anyhow!("{message}")) + } + + #[test] + fn qemu_error_wins_over_successful_probe() { + // A fail_regex match, terminal timeout, or QEMU spawn/exit failure must + // fail the run even when the HTTP probe passed. + assert!( + combine_results( + err("Fail pattern matched '(?i)panic': panicked at ..."), + true, + Some(ok()), + ) + .is_err() + ); + assert!(combine_results(err("QEMU timeout"), true, Some(ok())).is_err()); + assert!(combine_results(err("failed to spawn qemu"), true, Some(ok())).is_err()); + } + + #[test] + fn probe_error_wins_on_clean_qemu_exit() { + let verdict = combine_results(ok(), true, Some(err("probe: expected 200 got 404"))); + assert!( + verdict + .unwrap_err() + .to_string() + .contains("probe: expected 200") + ); + } + + #[test] + fn both_ok_passes() { + assert!(combine_results(ok(), true, Some(ok())).is_ok()); + } + + #[test] + fn missing_probe_verdict_on_clean_qemu_exit_fails() { + let verdict = combine_results(ok(), true, None); + assert!(verdict.unwrap_err().to_string().contains("no verdict")); + } + + #[test] + fn non_probe_case_uses_qemu_result() { + assert!(combine_results(ok(), false, None).is_ok()); + assert!(combine_results(err("boot failed"), false, None).is_err()); + } + + #[test] + fn probe_config_parses_token_with_serde_defaults() { + let dir = + std::env::temp_dir().join(format!("axvisor-probe-config-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let probe_path = dir.join("qemu-aarch64.toml"); + let plain_path = dir.join("plain.toml"); + std::fs::write(&probe_path, "[host_http_probe]\ntoken = \"t\"\n").unwrap(); + std::fs::write(&plain_path, "args = []\n").unwrap(); + + let config = load_axvisor_http_probe_config(&probe_path) + .unwrap() + .expect("host_http_probe present"); + assert_eq!(config.token.as_deref(), Some("t")); + assert_eq!(config.guest_port, 8080); + assert_eq!(config.connect_timeout_secs, 120); + assert_eq!(config.request_timeout_secs, 5); + + assert!( + load_axvisor_http_probe_config(&plain_path) + .unwrap() + .is_none() + ); + + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/scripts/axbuild/src/axvisor/test/types.rs b/scripts/axbuild/src/axvisor/test/types.rs index 533dddcc31..44c37006d1 100644 --- a/scripts/axbuild/src/axvisor/test/types.rs +++ b/scripts/axbuild/src/axvisor/test/types.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use ostool::run::qemu::QemuConfig; +use serde::Deserialize; use crate::test::{board as board_test, case::TestQemuCase, qemu as test_qemu}; @@ -44,3 +45,70 @@ impl board_test::BoardTestGroupInfo for BoardTestGroup { &self.board_name } } + +/// Host-side probe configuration for the AxVisor management HTTP control plane. +/// +/// Direction is the reverse of the generic [`HostHttpServerConfig`](crate::test::case::HostHttpServerConfig): +/// instead of the host serving fixtures to the guest, the host acts as a +/// *client* that probes the axum management API running *inside* the guest, +/// over QEMU user-mode networking hostfwd +/// (`-netdev user,hostfwd=tcp::-:`). The *test content* +/// — the concrete requests, fixtures, and assertions — lives with the test-suit +/// case as an executable probe asset (see [`probe_script`](Self::probe_script)); +/// the generic runner only orchestrates: forward the port, execute the asset, +/// collect its exit code, and report the result. This config is +/// AxVisor-specific: it carries the bearer token, timeouts, and probe-asset +/// name the runner passes on, so it lives in the AxVisor test layer rather than +/// the generic test layer. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub(crate) struct AxvisorHttpProbeConfig { + /// Guest-side port the in-guest HTTP server binds to. The harness forwards a + /// freshly picked host port to it via hostfwd, so the two never collide. + #[serde(default = "default_probe_guest_port")] + pub(crate) guest_port: u16, + /// Total seconds the probe may spend retrying the initial TCP connect before + /// giving up (guest boot + network init). Must be less than the QEMU case + /// `timeout` so a broken server fails on the probe, not on the QEMU timeout. + /// Passed to the probe asset as `AXVISOR_HTTP_CONNECT_TIMEOUT`. + #[serde(default = "default_probe_connect_timeout_secs")] + pub(crate) connect_timeout_secs: u64, + /// Per-request HTTP timeout so a hung in-guest server fails a single request + /// fast and the probe asset's poll loops can retry instead of blocking the + /// runner thread forever. Passed to the probe asset as + /// `AXVISOR_HTTP_REQUEST_TIMEOUT`. + #[serde(default = "default_probe_request_timeout_secs")] + pub(crate) request_timeout_secs: u64, + /// Executable probe asset, resolved against the case directory. It owns all + /// concrete requests/assertions for the case, so new HTTP scenarios or + /// API-contract changes edit the case asset rather than this crate. The + /// runner spawns it once the forwarded port is reachable and treats the + /// exit code as the verdict (0 = pass). Defaults to `http_probe.py`. + #[serde(default = "default_probe_script")] + pub(crate) probe_script: PathBuf, + /// Bearer token the probe must send on authenticated requests, matching the + /// guest build's `[env] AXVM_HTTP_TOKEN`. The probe also asserts that an + /// *unauthenticated* write request is rejected with 401 (the access-denied + /// regression the management-control-plane security review requires). + /// Passed to the probe asset as `AXVISOR_HTTP_TOKEN`. + #[serde(default)] + pub(crate) token: Option, +} + +/// Default probe-asset file name inside the case directory. +pub(crate) const DEFAULT_PROBE_SCRIPT: &str = "http_probe.py"; + +fn default_probe_guest_port() -> u16 { + 8080 +} + +fn default_probe_script() -> PathBuf { + PathBuf::from(DEFAULT_PROBE_SCRIPT) +} + +fn default_probe_connect_timeout_secs() -> u64 { + 120 +} + +fn default_probe_request_timeout_secs() -> u64 { + 5 +} diff --git a/test-suit/axvisor/normal/qemu-http-control-plane/build-aarch64-unknown-none-softfloat.toml b/test-suit/axvisor/normal/qemu-http-control-plane/build-aarch64-unknown-none-softfloat.toml new file mode 100644 index 0000000000..76d37dfc92 --- /dev/null +++ b/test-suit/axvisor/normal/qemu-http-control-plane/build-aarch64-unknown-none-softfloat.toml @@ -0,0 +1,25 @@ +# Converged HTTP control-plane build for the axum management API test +# (http-control-plane). +# `no-auto-start` keeps the default VM (`http-control-plane/vm-memory.toml`, a +# build-time embedded `memory` guest) in `Ready` (registered but not booted); the +# typed probe drives the full lifecycle through the HTTP control API. The guest +# kernel and its BusyBox initramfs are embedded at build time, so no `fs`/NVMe +# feature is needed. CI runs `cargo xtask image pull` before the test to +# provision the QEMU images and the managed rootfs the initramfs is built from. +features = [ + "http-axum", + "no-auto-start", +] +log = "Info" +target = "aarch64-unknown-none-softfloat" +vm_configs = ["test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/vm-memory.toml"] + +# Control-plane auth + bind. The mutating routes require +# `Authorization: Bearer `; the typed probe sends this same token +# (`[host_http_probe] token`), and hostfwd reachability requires binding all +# interfaces. The BusyBox initramfs is generated from the managed rootfs into +# the path the fixture's `ramdisk_path` points at. +[env] +AXVM_HTTP_TOKEN = "axvisor-http-test-token" +AXVM_HTTP_BIND = "0.0.0.0:8080" +AXVISOR_TEST_BUSYBOX_INITRAMFS = "tmp/axbuild/axvisor/qemu-http-control-plane/initramfs-aarch64.cpio.gz" diff --git a/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/http_probe.py b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/http_probe.py new file mode 100755 index 0000000000..63d67d4edb --- /dev/null +++ b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/http_probe.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Host-side probe asset for the AxVisor management HTTP control plane. + +Case asset for the `http-control-plane` test case +(`test-suit/axvisor/normal/qemu-http-control-plane/`). It owns the *test +content* — the concrete requests, the `vm-memory.toml` fixture, and the +assertions — and can evolve independently of the axbuild runner. + +The generic axbuild probe runner +(`scripts/axbuild/src/axvisor/test/http_probe.rs`) executes this script after +the QEMU hostfwd port is reachable, then treats the exit code as the verdict: +0 = all assertions passed, nonzero = a step failed. The script dials the axum +management API running *inside* the AxVisor guest through QEMU user-mode +networking hostfwd. Nothing in the hypervisor knows a test is running. + +Environment (set by the generic runner): + + AXVISOR_HTTP_BASE http://127.0.0.1: (forwarded) + AXVISOR_HTTP_TOKEN bearer token for authenticated requests + AXVISOR_HTTP_CASE_DIR case directory holding `vm-memory.toml` + (default: this file's directory) + AXVISOR_HTTP_CONNECT_TIMEOUT seconds for the initial reachability wait + AXVISOR_HTTP_REQUEST_TIMEOUT seconds per HTTP request + +The probe drives the whole `/api/vms` lifecycle contract in one boot, +including the destroy-then-recreate resource re-acquire regression, mirroring +`os/axvisor/doc/http-control-plane-quickstart.md`: + + GET /api/vms -> 200 (list; id=1 present) + GET /api/vms/1 -> 200 ready (detail; id/name/cpu_num/vcpu_states) + GET /api/vms/not-an-id -> 404 (non-numeric id) + GET /api/vms/999 -> 404 (unknown VM) + POST /api/vms/create -> 401 (no token) + POST /api/vms/1/start -> 401 (no token) + POST /api/vms/1/stop -> 401 (no token) + DELETE /api/vms/1 -> 401 (no token) + POST /api/vms/create {} -> 400 (missing toml) + POST /api/vms/create -> 400 (invalid TOML) + POST /api/vms/999/start -> 404 (auth'd unknown VM) + POST /api/vms/999/stop -> 404 (auth'd unknown VM) + DELETE /api/vms/999 -> 404 (auth'd unknown VM) + POST /api/vms/create -> 409 (id=1 already registered) + POST /api/vms/1/start -> 200 -> running (async=false) + POST /api/vms/1/start -> 409 (already running) + POST /api/vms/1/stop -> 200 -> stopped (async=true) + POST /api/vms/1/start -> 409 (restart-after-stop) + DELETE /api/vms/1 -> 204 -> 404 (gone) + POST /api/vms/create -> 200 {id:1} (recreate after delete) + POST /api/vms/create -> 409 (id=1 re-registered) + POST /api/vms/1/start -> 200 -> running (recreated VM usable) + POST /api/vms/1/stop -> 200 -> stopped + DELETE /api/vms/1 -> 204 -> 404 (cleanup) + +The last recreate -> start -> stop -> delete block is the resource re-acquire +regression: it proves destroy freed guest memory, vCPUs, devices, and the +registry entry so a fresh VM can be rebuilt from the same embedded image. +`vm-memory.toml` is matched by `base.id` against the build-time embedded +images, so the create body carries that file verbatim (the `kernel_path` / +`ramdisk_path` `${workspace}` placeholders are unused at runtime for memory +images). +""" + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ.get("AXVISOR_HTTP_BASE", "http://127.0.0.1:8080").rstrip("/") +TOKEN = os.environ.get("AXVISOR_HTTP_TOKEN", "") +CASE_DIR = os.environ.get( + "AXVISOR_HTTP_CASE_DIR", os.path.dirname(os.path.abspath(__file__)) +) +CONNECT_TIMEOUT = float(os.environ.get("AXVISOR_HTTP_CONNECT_TIMEOUT", "120")) +REQUEST_TIMEOUT = float(os.environ.get("AXVISOR_HTTP_REQUEST_TIMEOUT", "5")) +# Deadline for VM state transitions (boot, stop, delete): must stay well below +# the case `timeout` (600s) so a stuck transition fails on the probe, not on +# the QEMU timeout. +POLL_DEADLINE = 120.0 +POLL_INTERVAL = 1.0 + + +def request(method, path, token=None, body=None): + """One HTTP request; returns (status, parsed JSON or None). + + `token` defaults to `None`: the unauthenticated steps assert the 401 + rejections, and the poll loops mirror the runner's no-token GETs. The + authenticated steps pass `token=TOKEN` explicitly. + + A JSON `body` is sent with `Content-Type: application/json`. A non-2xx + response is not an error here — the caller asserts the status. A transport + error (connection refused/reset/timeout while the guest server is coming up + or mid-transition) raises RuntimeError for the caller to retry or fail. + """ + headers = {} + if token: + headers["Authorization"] = "Bearer " + token + data = None + if body is not None: + headers["Content-Type"] = "application/json" + data = body.encode("utf-8") + req = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp: + status = resp.status + raw = resp.read() + except urllib.error.HTTPError as err: + status = err.code + raw = err.read() + except urllib.error.URLError as err: + raise RuntimeError("request %s %s failed: %s" % (method, path, err.reason)) + except OSError as err: + # `resp.read()` raises a bare `socket.timeout` (an OSError) that the + # URLError handler above does not wrap. QEMU's user-mode hostfwd accepts + # the host-side connection as soon as QEMU starts, before the in-guest + # management server binds, so a first request can stall to the request + # timeout. Converting it to a retryable RuntimeError here lets the poll + # loops retry instead of crashing the probe in the boot window. + raise RuntimeError("request %s %s failed: %s" % (method, path, err)) + if not raw: + return status, None + return status, json.loads(raw.decode("utf-8")) + + +def check(label, actual, expected): + """Assert a status code, printing a progress line.""" + if actual != expected: + raise AssertionError("%s returned %s, expected %s" % (label, actual, expected)) + print(" http probe: %s -> %s (expect %s)" % (label, actual, expected)) + + +def vm_status(body): + """Extract the top-level `status` string of a VM detail body.""" + if not isinstance(body, dict): + raise AssertionError("VM detail response was not a JSON object") + status = body.get("status") + if not isinstance(status, str): + raise AssertionError("VM detail response had no status string: %r" % (body,)) + return status + + +def check_vm_status(label, body, expected): + status = vm_status(body) + print(" http probe: %s -> status %s (expect %s)" % (label, status, expected)) + if status != expected: + raise AssertionError( + "%s reported status %s, expected %s" % (label, status, expected) + ) + + +def check_action(label, body, ok_expected, async_expected): + """Assert a lifecycle action response's `ok` and `async` markers.""" + if not isinstance(body, dict): + raise AssertionError("%s had no JSON body" % (label,)) + ok = body.get("ok") + is_async = body.get("async") + print( + " http probe: %s -> ok=%r async=%r (expect ok=%r async=%r)" + % (label, ok, is_async, ok_expected, async_expected) + ) + if ok != ok_expected: + raise AssertionError( + "%s reported ok=%r, expected %r" % (label, ok, ok_expected) + ) + if is_async != async_expected: + raise AssertionError( + "%s reported async=%r, expected %r" % (label, is_async, async_expected) + ) + + +def list_has_vm(body, vm_id): + """Whether a `GET /api/vms` body lists a VM with the given id.""" + return isinstance(body, list) and any( + isinstance(item, dict) and item.get("id") == vm_id for item in body + ) + + +def poll_ready(): + """Poll `GET /api/vms` until it returns 200 or the connect deadline passes. + + The runner's TCP port wait proves the guest is listening, but the axum + router may still be wiring up, so the first request is retried here. + """ + start = time.monotonic() + while True: + if time.monotonic() - start > CONNECT_TIMEOUT: + raise AssertionError( + "guest management HTTP server never became reachable within %.0fs" + % CONNECT_TIMEOUT + ) + try: + status, _ = request("GET", "/api/vms") + if status == 200: + return + except RuntimeError: + pass + time.sleep(POLL_INTERVAL) + + +def poll_vm_status(vm_id, expected): + """Poll `GET /api/vms/{id}` until its status equals `expected`.""" + start = time.monotonic() + while True: + if time.monotonic() - start > POLL_DEADLINE: + raise AssertionError( + "VM[%d] never became %s within %.0fs" % (vm_id, expected, POLL_DEADLINE) + ) + try: + status, body = request("GET", "/api/vms/%d" % vm_id) + if status == 200 and vm_status(body) == expected: + print(" http probe: VM[%d] -> %s" % (vm_id, expected)) + return + except (RuntimeError, AssertionError): + # A non-200 or transport error during a transition (e.g. the VM is + # being torn down) is transient; keep polling until the deadline. + pass + time.sleep(POLL_INTERVAL) + + +def poll_vm_gone(vm_id): + """Poll `GET /api/vms/{id}` until it returns 404 (the VM was deleted).""" + start = time.monotonic() + while True: + if time.monotonic() - start > POLL_DEADLINE: + raise AssertionError( + "VM[%d] never disappeared within %.0fs" % (vm_id, POLL_DEADLINE) + ) + try: + status, _ = request("GET", "/api/vms/%d" % vm_id) + if status == 404: + print(" http probe: VM[%d] -> gone" % vm_id) + return + except RuntimeError: + pass + time.sleep(POLL_INTERVAL) + + +def main(): + with open(os.path.join(CASE_DIR, "vm-memory.toml"), "r", encoding="utf-8") as f: + vm_config = f.read() + create_body = json.dumps({"toml": vm_config}) + bad_body = json.dumps({"toml": "this is not [[ valid toml {{{"}) + + # 1. Readiness: the runner already waited for the TCP port; retry the first + # request briefly in case the axum router is still binding. + poll_ready() + print(" http probe: guest management server reachable") + + # 2. List: the default VM (id 1) is registered and `Ready`. + status, body = request("GET", "/api/vms") + check("GET /api/vms", status, 200) + if not list_has_vm(body, 1): + raise AssertionError("GET /api/vms did not list the default VM id=1") + + # 3. Detail of the default VM: identity, shape, and ready status. + status, body = request("GET", "/api/vms/1") + check("GET /api/vms/1", status, 200) + check_vm_status("GET /api/vms/1", body, "ready") + if body.get("id") != 1: + raise AssertionError("GET /api/vms/1 did not report id=1") + if body.get("name") != "linux-http-control-plane": + raise AssertionError("GET /api/vms/1 did not report the fixture name") + if body.get("cpu_num") != 1: + raise AssertionError("GET /api/vms/1 did not report cpu_num=1") + if not isinstance(body.get("vcpu_states"), list) or not body["vcpu_states"]: + raise AssertionError("GET /api/vms/1 reported an empty vcpu_states array") + + # 4-5. Error path: non-numeric and unknown ids are 404. + status, _ = request("GET", "/api/vms/not-an-id") + check("GET /api/vms/not-an-id", status, 404) + status, _ = request("GET", "/api/vms/999") + check("GET /api/vms/999", status, 404) + + # 6-9. Auth: every mutating route rejects an unauthenticated write with + # 401, before any VM lookup or body parse. + status, _ = request("POST", "/api/vms/create") + check("POST /api/vms/create (no auth)", status, 401) + status, _ = request("POST", "/api/vms/1/start") + check("POST /api/vms/1/start (no auth)", status, 401) + status, _ = request("POST", "/api/vms/1/stop") + check("POST /api/vms/1/stop (no auth)", status, 401) + status, _ = request("DELETE", "/api/vms/1") + check("DELETE /api/vms/1 (no auth)", status, 401) + + # 10-11. Create validates its body: a missing `toml` and an invalid TOML + # document both reject with 400. + status, _ = request("POST", "/api/vms/create", token=TOKEN, body="{}") + check("POST /api/vms/create (missing toml)", status, 400) + status, _ = request("POST", "/api/vms/create", token=TOKEN, body=bad_body) + check("POST /api/vms/create (invalid toml)", status, 400) + + # 12-14. Authenticated writes to an unknown VM are 404. + status, _ = request("POST", "/api/vms/999/start", token=TOKEN) + check("POST /api/vms/999/start (auth'd)", status, 404) + status, _ = request("POST", "/api/vms/999/stop", token=TOKEN) + check("POST /api/vms/999/stop (auth'd)", status, 404) + status, _ = request("DELETE", "/api/vms/999", token=TOKEN) + check("DELETE /api/vms/999 (auth'd)", status, 404) + + # 15. Duplicate create while id=1 is registered conflicts. + status, _ = request("POST", "/api/vms/create", token=TOKEN, body=create_body) + check("POST /api/vms/create (duplicate id=1)", status, 409) + + # 16. Start the default VM: accepted synchronously (`async=false`), then + # poll the detail into `running`. + status, body = request("POST", "/api/vms/1/start", token=TOKEN) + check("POST /api/vms/1/start", status, 200) + check_action("POST /api/vms/1/start", body, True, False) + poll_vm_status(1, "running") + + # 17. Re-starting an already-running VM conflicts. + status, _ = request("POST", "/api/vms/1/start", token=TOKEN) + check("POST /api/vms/1/start (already running)", status, 409) + + # 18. Stop is a request (`async=true`): the `stopped` state arrives + # asynchronously once the vCPU observes it and exits. + status, body = request("POST", "/api/vms/1/stop", token=TOKEN) + check("POST /api/vms/1/stop", status, 200) + check_action("POST /api/vms/1/stop", body, True, True) + poll_vm_status(1, "stopped") + + # 19. Restart-after-stop is a known scheduling limitation; the contract + # rejects it with 409 rather than hanging the VM in `running`. + status, _ = request("POST", "/api/vms/1/start", token=TOKEN) + check("POST /api/vms/1/start (restart-after-stop)", status, 409) + + # 20. Delete the stopped VM, then poll until it is gone. + status, _ = request("DELETE", "/api/vms/1", token=TOKEN) + check("DELETE /api/vms/1", status, 204) + poll_vm_gone(1) + + # 21. Recreate after delete: the embedded image is matched by id, so a + # fresh create with the same config succeeds and registers id 1 again. + status, body = request("POST", "/api/vms/create", token=TOKEN, body=create_body) + check("POST /api/vms/create (recreate)", status, 200) + if not isinstance(body, dict) or body.get("id") != 1: + raise AssertionError("recreate did not return id=1") + poll_vm_status(1, "ready") + + # 22. The re-registered id conflicts with a second create. + status, _ = request("POST", "/api/vms/create", token=TOKEN, body=create_body) + check("POST /api/vms/create (recreate duplicate)", status, 409) + + # 23-24. The recreated VM must be fully usable, not merely re-registered: + # destroy must have freed guest memory, vCPUs, devices, and the + # registry entry so a fresh VM can be rebuilt and run from the same + # embedded image. This is the resource re-acquire regression. + status, _ = request("POST", "/api/vms/1/start", token=TOKEN) + check("POST /api/vms/1/start (recreated)", status, 200) + poll_vm_status(1, "running") + status, _ = request("POST", "/api/vms/1/stop", token=TOKEN) + check("POST /api/vms/1/stop (recreated)", status, 200) + poll_vm_status(1, "stopped") + + # 25. Cleanup: leave the hypervisor without a registered VM. + status, _ = request("DELETE", "/api/vms/1", token=TOKEN) + check("DELETE /api/vms/1 (cleanup)", status, 204) + poll_vm_gone(1) + + print(" http probe: full control-plane contract passed") + + +if __name__ == "__main__": + try: + main() + except AssertionError as exc: + print(" http probe: FAILED: %s" % exc, file=sys.stderr) + sys.exit(1) + except Exception as exc: + print(" http probe: ERROR: %s" % exc, file=sys.stderr) + sys.exit(2) diff --git a/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/qemu-aarch64.toml b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/qemu-aarch64.toml new file mode 100644 index 0000000000..d88b67acfe --- /dev/null +++ b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/qemu-aarch64.toml @@ -0,0 +1,40 @@ +args = [ + "-nographic", + "-cpu", + "cortex-a72", + "-machine", + "virt,virtualization=on,gic-version=3", + "-smp", + "2", + # The memory-backed guest boots its embedded kernel + BusyBox initramfs; the + # initramfs is the guest rootfs (`rdinit=/init`), so no NVMe disk is needed. + "-append", + "console=ttyAMA0 rdinit=/init devtmpfs.mount=1 loglevel=7", + # `-m 1g`: the guest uses a 256M MAP_ALLOC region, and the hypervisor plus + # that region must both fit in QEMU's total RAM (same sizing as the previous + # HTTP control case). + "-m", + "1g", +] +# Full lifecycle verification via the case's host-side probe asset. +# `[host_http_probe]` makes the runner append hostfwd netdev + virtio-net-pci +# device and execute the probe asset (`http_probe.py`, next to this file) over +# real TCP. The asset drives the default VM (id 1, `vm-memory.toml`, kept +# `Ready` by `no-auto-start`) through the full contract — +# list/detail/auth/error/start/stop/delete/recreate — with polling, then quits +# QEMU over QMP; the runner collects the asset's exit code as the verdict. The +# test content is owned by this case and can evolve here without touching the +# axbuild runner. +# `success_regex` is empty because the probe result, not serial output, is the +# verdict; a guest panic still fails via `fail_regex`. +fail_regex = [ + "(?i)\\bpanic(?:ked)?\\b", + "(?i)kernel panic", +] +success_regex = [] +timeout = 600 +to_bin = true +uefi = false + +[host_http_probe] +token = "axvisor-http-test-token" diff --git a/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/vm-memory.toml b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/vm-memory.toml new file mode 100644 index 0000000000..7e75592c2b --- /dev/null +++ b/test-suit/axvisor/normal/qemu-http-control-plane/http-control-plane/vm-memory.toml @@ -0,0 +1,54 @@ +# Memory-backed guest fixture for the converged HTTP control-plane test. +# The kernel and its BusyBox initramfs are embedded into the hypervisor binary +# at build time (`image_location = "memory"`, see os/axvisor/build.rs) and +# matched at runtime only by `base.id` — the `${workspace}` paths below are +# resolved by axbuild at build time and ignored by the runtime create handler. +# +# `id = 1`: the runtime can only realize guest images baked into the hypervisor, +# so the typed probe's `POST /api/vms/create` body uses this same config, whose +# `base.id` re-matches the embedded image. `phys_cpu_ids = [1]` keeps the vCPU on +# Core 1 while the management console runs on Core 0. +[base] +id = 1 +name = "linux-http-control-plane" +guest_type = "passthrough" +cpu_num = 1 +phys_cpu_ids = [1] + +# +# Vm kernel configs +# +[kernel] +# The entry point of the kernel image. +entry_point = 0x8020_0000 +# The location of image: "memory" | "fs". Memory embeds the image at build time. +image_location = "memory" +# The file path of the kernel image (build-time only for memory images). +kernel_path = "${workspace}/tmp/axbuild/images/qemu-aarch64/linux/linux-qemu" +# The load address of the kernel image. +kernel_load_addr = 0x8020_0000 +# The load address of the device tree blob (DTB). +dtb_load_addr = 0x8000_0000 +# BusyBox initramfs generated from the managed rootfs (the guest rootfs; the +# QEMU case appends `rdinit=/init`). +ramdisk_path = "${workspace}/tmp/axbuild/axvisor/qemu-http-control-plane/initramfs-aarch64.cpio.gz" +ramdisk_load_addr = 0x8400_0000 + +# Memory regions with format (`base_paddr`, `size`, `flags`, `map_type`). +# For `map_type`, 0 means `MAP_ALLOC`, 1 means `MAP_IDENTICAL`, 2 means `MAP_RESERVED`. +# MAP_ALLOC maps guest GPA 0x80000000 to hypervisor-allocated host pages; the +# QEMU virt host RAM is 0x40000000..0x80000000, so an identity-mapped region +# would land at the host heap address rather than the configured 0x80000000 and +# the guest images at 0x80200000 would not resolve. +memory_regions = [ + [0x8000_0000, 0x1000_0000, 0x7, 0], # System RAM 256M MAP_ALLOC +] + +# +# Device specifications +# + +# Physical-device selection. Virtual platform devices are machine-owned. +[devices] +passthrough = [] +disabled = [] diff --git a/virtualization/axvm/src/architecture/ops.rs b/virtualization/axvm/src/architecture/ops.rs index 51451adc1b..091e70f6a2 100644 --- a/virtualization/axvm/src/architecture/ops.rs +++ b/virtualization/axvm/src/architecture/ops.rs @@ -483,4 +483,28 @@ mod tests { ); assert!(dispatcher.drain(0).is_empty()); } + + #[test] + fn default_vcpu_affinities_pins_single_cpu_to_isolated_core() { + // aarch64/riscv64 arceos-smp1.toml: cpu_num=1, phys_cpu_ids=[1] → pin to physical Core 1. + assert_eq!( + default_vcpu_affinities(1, Some(&[1]), None), + vec![(0, None, 1)] + ); + // x86_64 arceos-smp1.toml: phys_cpu_sets=[2] → affinity mask 0b10 (Core 1). + assert_eq!( + default_vcpu_affinities(1, None, Some(&[2])), + vec![(0, Some(2), 0)] + ); + // Default: no pinning, vcpu id equals physical id. + assert_eq!(default_vcpu_affinities(1, None, None), vec![(0, None, 0)]); + } + + #[test] + fn default_vcpu_affinities_multi_cpu_falls_back_to_vcpu_id() { + assert_eq!( + default_vcpu_affinities(2, None, None), + vec![(0, None, 0), (1, None, 1)] + ); + } } diff --git a/virtualization/axvm/src/config.rs b/virtualization/axvm/src/config.rs index 063409c0b7..d2eda04eb7 100644 --- a/virtualization/axvm/src/config.rs +++ b/virtualization/axvm/src/config.rs @@ -594,4 +594,19 @@ mod tests { Err(crate::AxVmError::InvalidConfig { .. }) )); } + + #[test] + fn phys_cpu_list_pins_single_vcpu_to_isolated_core() { + // aarch64/riscv64 arceos-smp1.toml: cpu_num=1, phys_cpu_ids=[1] → physical Core 1. + let list = PhysCpuList::new(1, Some(vec![1]), None); + assert_eq!(list.get_vcpu_affinities_pcpu_ids(), vec![(0, None, 1)]); + + // x86_64 arceos-smp1.toml: phys_cpu_sets=[2] → affinity mask 0b10 (Core 1). + let list = PhysCpuList::new(1, None, Some(vec![2])); + assert_eq!(list.get_vcpu_affinities_pcpu_ids(), vec![(0, Some(2), 0)]); + + // Default: no pinning, vcpu id equals physical id. + let list = PhysCpuList::new(1, None, None); + assert_eq!(list.get_vcpu_affinities_pcpu_ids(), vec![(0, None, 0)]); + } } diff --git a/virtualization/axvm/src/runtime/mod.rs b/virtualization/axvm/src/runtime/mod.rs index 281ca0494a..06a37824ae 100644 --- a/virtualization/axvm/src/runtime/mod.rs +++ b/virtualization/axvm/src/runtime/mod.rs @@ -137,11 +137,48 @@ fn notify_runtime_for_device_poll(runtime: &crate::vm::VmRuntimeHandle, vcpu_num pub fn stop_vm(vm_id: usize) -> AxVmResult { let vm = vm_by_id(vm_id)?; + if matches!(vm.status(), VmStatus::Running) { + // `start_vm` flips the status to `Running` synchronously while the + // vCPU task may still be queued on another CPU. Requesting a stop in + // that window strands the task in its startup gate (which needs a + // `Running` window it already missed), so wait for the first vCPU + // entry before accepting the stop. + wait_until_vcpu_entered(|| vm.running_vcpu_count() > 0, || vm.stopping())?; + } vm.stop(StopReason::Forced)?; vcpus::notify_all_vcpus(vm_id); Ok(()) } +/// Boundedly wait for at least one vCPU task to enter the guest run loop +/// before a request-stop is accepted. +/// +/// `start_vm` flips the VM status to `Running` synchronously while the vCPU +/// task may still be queued on another CPU. If the stop is accepted in that +/// window, the task's startup gate blocks on a `Running` window it already +/// missed and parks forever, so the VM never reaches `Stopped`. Waiting for +/// the first vCPU entry closes that window. +/// +/// The wait is bounded (`MAX_YIELDS`, mirroring the `wait_until_stopped` +/// pattern): a vCPU task that never runs yields an error instead of hanging +/// the caller, leaving the VM `Running` so the stop can be retried. +fn wait_until_vcpu_entered( + vcpu_entered: impl Fn() -> bool, + vm_stopping: impl Fn() -> bool, +) -> AxVmResult { + const MAX_YIELDS: usize = 10_000; + for _ in 0..MAX_YIELDS { + if vcpu_entered() || vm_stopping() { + return Ok(()); + } + crate::host::task::yield_now(); + } + ax_err!( + BadState, + "vCPU task did not enter the guest before request-stop" + ) +} + pub fn resume_vm(vm_id: usize) -> AxVmResult { let vm = vm_by_id(vm_id)?; vm.resume()?; @@ -179,6 +216,11 @@ const fn missing_vm_error(vm_id: usize) -> AxVmError { #[cfg(test)] mod tests { + use std::{ + cell::Cell, + sync::{Arc, atomic::AtomicBool}, + }; + use super::*; #[test] @@ -222,4 +264,72 @@ mod tests { assert!(runtime.device_poll_requested()); } + + #[test] + fn request_stop_wait_returns_immediately_once_a_vcpu_has_entered() { + assert!(wait_until_vcpu_entered(|| true, || false).is_ok()); + } + + #[test] + fn request_stop_wait_bails_out_when_vm_is_already_stopping() { + assert!(wait_until_vcpu_entered(|| false, || true).is_ok()); + } + + #[test] + fn request_stop_wait_times_out_instead_of_accepting_a_never_entering_vcpu() { + let err = wait_until_vcpu_entered(|| false, || false).unwrap_err(); + + assert!(matches!(err, AxVmError::InvalidState { .. })); + } + + #[test] + fn request_stop_waits_for_vcpu_entry_when_stop_precedes_entry() { + // Force the scheduling order that previously stranded the vCPU task: + // the request-stop arrives while no vCPU task has entered the guest + // run loop, and the task only enters after the wait has begun. The + // stop must be held back until entry, never accepted-and-stranded. + let entered = Arc::new(AtomicBool::new(false)); + let stopping = Arc::new(AtomicBool::new(false)); + let first_poll = Arc::new(std::sync::Barrier::new(2)); + let release_entered = Arc::new(std::sync::Barrier::new(2)); + + let entered_for_task = entered.clone(); + let first_poll_for_task = first_poll.clone(); + let release_entered_for_task = release_entered.clone(); + let vcpu_task = std::thread::spawn(move || { + // The vCPU task is queued but has not entered the guest yet. + first_poll_for_task.wait(); + release_entered_for_task.wait(); + entered_for_task.store(true, Ordering::Release); + }); + + let entered_for_wait = entered.clone(); + let stopping_for_wait = stopping.clone(); + let poll_count = Cell::new(0); + let result = wait_until_vcpu_entered( + || { + let is_entered = entered_for_wait.load(Ordering::Acquire); + if poll_count.get() == 0 { + // First poll observed the pre-entry state. Only now release + // the vCPU task to enter the guest, deterministically + // ordering stop-before-entry. + poll_count.set(1); + first_poll.wait(); + release_entered.wait(); + } + is_entered + }, + || stopping_for_wait.load(Ordering::Acquire), + ); + + vcpu_task.join().unwrap(); + assert!( + result.is_ok(), + "stop must wait for vCPU entry, not strand it" + ); + assert!( + entered.load(Ordering::Acquire), + "vCPU task must have entered the guest run loop" + ); + } } diff --git a/virtualization/axvm/src/runtime/vcpus.rs b/virtualization/axvm/src/runtime/vcpus.rs index 978be97f95..d51b47a6e1 100644 --- a/virtualization/axvm/src/runtime/vcpus.rs +++ b/virtualization/axvm/src/runtime/vcpus.rs @@ -475,7 +475,12 @@ fn vcpu_run() { mark_vcpu_running(&vm); } - info!("VM[{}] VCpu[{}] running...", vm.id(), vcpu.id()); + info!( + "VM[{}] VCpu[{}] running on CPU{}...", + vm.id(), + vcpu.id(), + crate::host::cpu::current_id() + ); loop { if vcpu_id == 0 { diff --git a/virtualization/axvm/src/vm/mod.rs b/virtualization/axvm/src/vm/mod.rs index c68f2474e2..e3aad01611 100644 --- a/virtualization/axvm/src/vm/mod.rs +++ b/virtualization/axvm/src/vm/mod.rs @@ -430,8 +430,11 @@ impl VmRuntimeHandle { } pub(crate) fn mark_vcpu_running(&self) { + // Release publishes the "a vCPU has entered the guest" signal to the + // control plane, which observes it with an Acquire load + // (`running_halting_vcpu_count`) before issuing a request-stop. self.running_halting_vcpu_count - .fetch_add(1, Ordering::Relaxed); + .fetch_add(1, Ordering::Release); } pub(crate) fn publish_cpu_on_start_success(&self, ack: &crate::runtime::vcpus::CpuOnStartAck) { @@ -447,6 +450,12 @@ impl VmRuntimeHandle { == Ok(1) } + pub(crate) fn running_halting_vcpu_count(&self) -> usize { + // Acquire pairs with the Release increment in `mark_vcpu_running`: the + // caller observes "a vCPU has entered the guest" before acting on it. + self.running_halting_vcpu_count.load(Ordering::Acquire) + } + pub(crate) fn record_lifecycle_error(&self, error: AxVmError) { let mut recorded = self.lifecycle_error.lock_unpoisoned(); if recorded.is_none() { @@ -991,6 +1000,30 @@ impl AxVM { .collect() } + /// Returns the number of vCPUs whose task has entered the guest run loop + /// and not yet finished exiting. + /// + /// A vCPU increments the count once, right before its first guest entry + /// (`vcpu_run`), and decrements it when it stops, so the count covers the + /// whole running + halting window: a non-zero value means at least one vCPU + /// task has been scheduled and is executing the guest. This differs from + /// `start_vm()`, which flips the VM status to `Running` synchronously while + /// the vCPU task may still be queued on another CPU. + /// + /// The count is a publish/observe signal between the vCPU cores and the + /// control plane: `mark_vcpu_running` increments it with `Release`, and this + /// getter loads it with `Acquire`. The control plane polls it to `> 0` + /// before issuing a request-stop, so a stop is only requested after a vCPU + /// has actually entered the guest; otherwise a stop issued before the vCPU + /// is scheduled would strand the vCPU task waiting forever for a `Running` + /// window it already missed. Because the count includes the halting window, + /// it must be read only as a monotone "a vCPU has entered" signal, not as an + /// exact "still running" count. + pub fn running_vcpu_count(&self) -> usize { + self.with_runtime(|runtime| Ok(runtime.running_halting_vcpu_count())) + .unwrap_or(0) + } + /// Returns the root address of the nested page table for the VM. pub fn nested_page_table_root(&self) -> AxVmResult { self.with_resources(|resources| Ok(resources.address_space.page_table_root()))