Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -10963,6 +10963,7 @@ name = "virtiofs"
version = "0.0.0"
dependencies = [
"anyhow",
"chipset_device",
"event-listener",
"fuse",
"futures",
Expand Down
5 changes: 4 additions & 1 deletion Guide/src/user_guide/openvmm/snapshots.md
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,10 @@ completion.
OpenVMM stops at that exact post-write boundary, completes the deferred write
while the vCPU is stopped, and then releases device input before resuming the
vCPU. Until then, the guest's virtio-fs requests wait on their virtqueues, so
repair must not wait for a virtio-fs reply. The acknowledgement is bounded by
repair must not wait for a virtio-fs reply. Resetting a device does not release
its input either: a device that the guest resets and starts again before the
acknowledgement stays gated until OpenVMM releases device input. The
acknowledgement is bounded by
`--restore-gate-timeout-ms` (60000 milliseconds by default). A control-console
host can connect while input is gated, but OpenVMM reads its authentication
only after the release, so its `--microvm-control-auth-timeout-ms` deadline
Expand Down
12 changes: 11 additions & 1 deletion vm/devices/virtio/virtio/src/device.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,13 +120,23 @@ pub trait VirtioDevice: InspectMut + Send {

/// Reset device-internal state to initial values.
///
/// Called after all queues have been stopped on guest-initiated reset.
/// Called after all queues have been stopped, on a guest-initiated reset
/// and on a reset of the VM. The reset keeps the host-input gate that
/// [`quiesce_input`](Self::quiesce_input) closed: only
/// [`resume_input`](Self::resume_input) opens it, so a guest that resets
/// the device while the VMM holds its input gets none until the VMM
/// releases it.
///
/// Default: no-op.
fn reset(&mut self) -> impl Future<Output = ()> + Send {
async {}
}

/// Stops accepting new host input while preserving device and queue state.
///
/// The gate holds, across queue restarts and resets, until
/// [`resume_input`](Self::resume_input). Queues that start while it is
/// closed start gated.
fn quiesce_input(&mut self) -> impl Future<Output = anyhow::Result<()>> + Send {
async { Ok(()) }
}
Expand Down
4 changes: 2 additions & 2 deletions vm/devices/virtio/virtio_console/src/saved_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@ pub(crate) const MAX_SAVED_STATE_BYTES: usize = 512 * 1024;
pub(crate) const SAVED_STATE_VERSION: u32 = 1;

impl VirtioConsoleDevice {
/// Resets the private state that persists across queue restarts.
/// Resets the private state that persists across queue restarts. The
/// host-input gate is kept: only resuming input releases it.
pub(crate) fn reset_private_state(&mut self) {
self.config = VirtioConsoleConfig::default();
let (worker, mut state) = self.worker.get_mut();
let state = state.as_mut().unwrap();
state.partial_transmit = 0;
state.staged_rx.clear();
state.input_gated = false;
state.mem = GuestMemory::empty();
if let ConsoleWorkerMode::Broker(mode) = &mut worker.mode {
mode.reset_for_device();
Expand Down
1 change: 1 addition & 0 deletions vm/devices/virtio/virtio_console/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ mod broker;
mod direct;
mod harness;
mod saved_state;
mod transport;

use crate::VirtioConsoleDevice;
use futures::AsyncRead;
Expand Down
33 changes: 33 additions & 0 deletions vm/devices/virtio/virtio_console/src/tests/broker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,39 @@ async fn broker_quiesce_stops_new_host_input(driver: DefaultDriver) {
);
}

/// A guest that resets the control console while host input is gated, and
/// then starts it again, receives no host input, the host's attachment among
/// it, until the gate is released.
#[async_test]
async fn broker_guest_reset_keeps_host_input_gated(driver: DefaultDriver) {
let mut harness = TestHarness::new_broker(&driver, BROKER_INSTANCE, BROKER_CAPABILITY);
harness.enable().await;
harness.device.quiesce_input().await.unwrap();

// The guest's reset, as the transport makes it when the guest writes 0
// to the device status. It detaches a host that the broker attached.
harness.disable().await;
harness.reset_rings();
harness.enable().await;
harness.handle.reconnect();

let host_attach = encode(&Record::bootstrap(
RecordType::HostAttach,
BROKER_CAPABILITY.to_vec(),
));
harness.handle.inject_rx_data(&host_attach);
for _ in 0..20 {
yield_now().await;
}
assert_eq!(harness.handle.pending_rx_len(), host_attach.len());
assert!(harness.handle.tx_data().is_empty());

harness.device.resume_input().await.unwrap();
yield_until(|| harness.handle.tx_data().len() >= control_session_protocol::HEADER_LEN).await;
let wait = decode(&harness.handle.take_tx_data());
assert_eq!(wait.record_type, RecordType::Wait);
}

#[async_test]
async fn broker_host_lifecycle_error_does_not_stop_guest_worker(driver: DefaultDriver) {
let mut harness = TestHarness::new_broker(&driver, BROKER_INSTANCE, BROKER_CAPABILITY);
Expand Down
32 changes: 32 additions & 0 deletions vm/devices/virtio/virtio_console/src/tests/direct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,38 @@ async fn input_gate_cancels_rx_but_keeps_tx_running(driver: DefaultDriver) {
assert_eq!(received, b"gated-input");
}

/// A guest that resets the console while host input is gated, as a snapshot
/// boundary or a gated restore gates it, and then starts it again receives no
/// host input until the gate is released.
#[async_test]
async fn guest_reset_keeps_host_input_gated(driver: DefaultDriver) {
let mut harness = TestHarness::new(&driver);
harness.enable().await;
harness.device.quiesce_input().await.unwrap();

// The guest's reset, as the transport makes it when the guest writes 0
// to the device status.
harness.disable().await;
harness.reset_rings();
harness.enable().await;

harness.handle.inject_rx_data(b"gated-input");
let gpa = harness.post_rx_buffer_and_signal(0, 64);
// The worker runs: it forwards the guest's output.
harness.post_tx_and_signal(0, b"tx-while-gated");
assert_eq!(harness.wait_for_tx_used().await, (0, 0));
for _ in 0..10 {
yield_now().await;
}
assert_eq!(harness.handle.pending_rx_len(), b"gated-input".len());

harness.device.resume_input().await.unwrap();
let (_, used_len) = harness.wait_for_rx_used().await;
let mut received = vec![0; used_len as usize];
harness.mem.read_at(gpa, &mut received).unwrap();
assert_eq!(received, b"gated-input");
}

#[async_test]
async fn read_error_drives_disconnect_before_reconnect(driver: DefaultDriver) {
let mut harness = TestHarness::new_with_policy(&driver, VirtioConsoleDisconnectPolicy::Discard);
Expand Down
187 changes: 187 additions & 0 deletions vm/devices/virtio/virtio_console/src/tests/transport.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

//! Tests that drive the console through the virtio-mmio transport, as a
//! guest's driver does.

use super::DATA_BASE;
use super::QUEUE_SIZE;
use super::RX_AVAIL_ADDR;
use super::RX_DESC_ADDR;
use super::RX_USED_ADDR;
use super::TX_AVAIL_ADDR;
use super::TX_DESC_ADDR;
use super::TX_USED_ADDR;
use super::TestHarness;
use super::harness::yield_now;
use chipset_device::io::IoResult;
use chipset_device::mmio::MmioIntercept;
use chipset_device::poll_device::PollDevice;
use guestmem::GuestMemory;
use pal_async::DefaultDriver;
use pal_async::async_test;
use pal_async::timer::PolledTimer;
use std::task::Context;
use std::task::Poll;
use std::task::Waker;
use std::time::Duration;
use test_with_tracing::test;
use virtio::spec::queue::DescriptorFlags;
use virtio::test_helpers::init_avail_ring;
use virtio::test_helpers::init_used_ring;
use virtio::test_helpers::make_available;
use virtio::test_helpers::read_used;
use virtio::test_helpers::write_descriptor;
use virtio::transport::VirtioMmioDevice;
use vmcore::device_state::ChangeDeviceState;
use vmcore::line_interrupt::LineInterrupt;

// The virtio-mmio registers that a guest's driver writes (virtio 1.2, section
// 4.2.2).
const DRIVER_FEATURES: u64 = 0x20;
const DRIVER_FEATURES_SEL: u64 = 0x24;
const QUEUE_SEL: u64 = 0x30;
const QUEUE_NUM: u64 = 0x38;
const QUEUE_READY: u64 = 0x44;
const QUEUE_NOTIFY: u64 = 0x50;
const STATUS: u64 = 0x70;
const QUEUE_DESC_LOW: u64 = 0x80;
const QUEUE_DRIVER_LOW: u64 = 0x90;
const QUEUE_DEVICE_LOW: u64 = 0xa0;

// The device status bits (virtio 1.2, section 2.1).
const ACKNOWLEDGE: u32 = 1;
const DRIVER: u32 = 2;
const DRIVER_OK: u32 = 4;
const FEATURES_OK: u32 = 8;

/// The console behind the virtio-mmio transport, and the guest memory of its
/// rings, which lie where the harness puts them.
struct MmioConsole {
transport: VirtioMmioDevice,
mem: GuestMemory,
}

impl MmioConsole {
/// Writes `value` to the transport register at `offset`, as a vCPU does,
/// and waits for the write to complete, polling the transport as the
/// chipset does.
async fn write(&mut self, offset: u64, value: u32) {
let mut token = match self.transport.mmio_write(offset, &value.to_le_bytes()) {
IoResult::Ok => return,
IoResult::Defer(token) => token,
IoResult::Err(error) => panic!("register write failed: {error:?}"),
};
for _ in 0..1000 {
let mut cx = Context::from_waker(Waker::noop());
self.transport.poll_device(&mut cx);
if let Poll::Ready(result) = token.poll_write(&mut cx) {
result.unwrap();
return;
}
yield_now().await;
}
panic!("register write did not complete");
}

/// Initializes the device as a guest's driver does, up to DRIVER_OK, with
/// empty rings.
async fn initialize(&mut self) {
for (avail, used) in [(RX_AVAIL_ADDR, RX_USED_ADDR), (TX_AVAIL_ADDR, TX_USED_ADDR)] {
init_avail_ring(&self.mem, avail);
init_used_ring(&self.mem, used);
}
self.write(STATUS, ACKNOWLEDGE).await;
self.write(STATUS, ACKNOWLEDGE | DRIVER).await;
// VIRTIO_F_VERSION_1, bit 0 of the second feature bank.
self.write(DRIVER_FEATURES_SEL, 1).await;
self.write(DRIVER_FEATURES, 1).await;
self.write(STATUS, ACKNOWLEDGE | DRIVER | FEATURES_OK).await;
for (queue, desc, avail, used) in [
(0, RX_DESC_ADDR, RX_AVAIL_ADDR, RX_USED_ADDR),
(1, TX_DESC_ADDR, TX_AVAIL_ADDR, TX_USED_ADDR),
] {
self.write(QUEUE_SEL, queue).await;
self.write(QUEUE_NUM, QUEUE_SIZE.into()).await;
self.write(QUEUE_DESC_LOW, desc.try_into().unwrap()).await;
self.write(QUEUE_DRIVER_LOW, avail.try_into().unwrap())
.await;
self.write(QUEUE_DEVICE_LOW, used.try_into().unwrap()).await;
self.write(QUEUE_READY, 1).await;
}
self.write(STATUS, ACKNOWLEDGE | DRIVER | FEATURES_OK | DRIVER_OK)
.await;
}

/// Makes a receive buffer of `len` bytes at the start of the data area
/// available in descriptor 0, and notifies the receive queue.
async fn post_rx_buffer(&mut self, len: u32) {
write_descriptor(
&self.mem,
RX_DESC_ADDR,
0,
DATA_BASE,
len,
DescriptorFlags::new().with_write(true),
0,
);
let mut avail_idx = 0;
make_available(&self.mem, RX_AVAIL_ADDR, QUEUE_SIZE, 0, &mut avail_idx);
self.write(QUEUE_NOTIFY, 0).await;
}
}

/// A guest that writes 0 to the status of the running console while host
/// input is gated resets the device, and its driver then starts it again: the
/// console receives no host input until the gate is released.
#[async_test]
async fn guest_status_reset_keeps_host_input_gated(driver: DefaultDriver) {
let harness = TestHarness::new(&driver);
let handle = harness.handle;
let mut console = MmioConsole {
transport: VirtioMmioDevice::new(
Box::new(harness.device),
&driver,
harness.mem.clone(),
LineInterrupt::detached(),
None,
0,
0x1000,
)
.unwrap(),
mem: harness.mem,
};
console.initialize().await;
ChangeDeviceState::quiesce_input(&mut console.transport)
.await
.unwrap();

console.write(STATUS, 0).await;
console.initialize().await;
handle.inject_rx_data(b"gated-input");
console.post_rx_buffer(64).await;
let mut timer = PolledTimer::new(&driver);
timer.sleep(Duration::from_millis(100)).await;
let mut used_idx = 0;
assert_eq!(
read_used(&console.mem, RX_USED_ADDR, QUEUE_SIZE, &mut used_idx),
None
);
assert_eq!(handle.pending_rx_len(), b"gated-input".len());

ChangeDeviceState::resume_input(&mut console.transport)
.await
.unwrap();
let mut used = None;
for _ in 0..500 {
used = read_used(&console.mem, RX_USED_ADDR, QUEUE_SIZE, &mut used_idx);
if used.is_some() {
break;
}
timer.sleep(Duration::from_millis(10)).await;
}
assert_eq!(used, Some((0, b"gated-input".len() as u32)));
let mut received = [0; b"gated-input".len()];
console.mem.read_at(DATA_BASE, &mut received).unwrap();
assert_eq!(&received, b"gated-input");
}
11 changes: 6 additions & 5 deletions vm/devices/virtio/virtio_net/src/quiesce.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
//!
//! Quiescing input stops the coordinator and the workers, lets each endpoint
//! queue finish the work it already accepted while it stops admitting new host
//! RX, drains the resulting completions, and restarts the workers. The
//! coordinator latches the input gate and re-establishes it whenever it
//! restarts the endpoint queues; resuming input releases it. Only devices built
//! with [`NicBuilder::save_restore`](crate::NicBuilder::save_restore) support
//! input quiesce.
//! RX, drains the resulting completions, and restarts the workers. The device
//! latches the input gate, which a reset of the device keeps, and its
//! coordinator re-establishes the gate whenever it starts or restarts the
//! endpoint queues; resuming input releases it. Only devices built with
//! [`NicBuilder::save_restore`](crate::NicBuilder::save_restore) support input
//! quiesce.

use crate::Coordinator;
use crate::Device;
Expand Down
3 changes: 1 addition & 2 deletions vm/devices/virtio/virtio_net/src/saved_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,11 @@ pub(crate) struct Lifecycle {

impl Lifecycle {
/// Clears the queue-pair state on device reset. The endpoint generation is
/// kept.
/// kept, and so is the input gate, which only resuming input releases.
pub(crate) fn reset(&mut self) {
self.stop_error = None;
self.stopped_feature_banks = None;
self.stopped_queue_count = 0;
self.input_quiesced = false;
}
}

Expand Down
Loading
Loading