From b5543262394f7d0f08768cb17702b767f78275cc Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 13:04:42 -0800 Subject: [PATCH 1/4] nvx-rs: activate a clone from its restore packet's record A platform clone took its activation record in the adapter's Activate, which the host sent once it attached. The clone's launch now passes the record to OpenVMM with --restore-activation-record, which seals it in the restore packet before the first restored vCPU runs, and the guest activates from the packet as soon as it runs, without waiting for the host (microsoft/nvx#544). - nvx::guest::clone: the repair hands the packet's record over, Repaired::activation_record, and Activation takes it from there: accept() decodes and checks it, and the activation is ready once the record applied and the host asked for Ready, the first request of each of its sessions. OpenVMM's broker refuses the guest's records while no host is attached, which the guest cannot tell, so the guest cannot send Ready unasked. A record that applies before the host asks leaves the activation in the new Applied state, its fence closed and its deadline holding, so that a clone whose host never asks still ends, and any other request before Ready still fails it. A reset changes nothing of the record, so it only abandons the requests for Ready of the earlier session: the Transaction, Changed, and Reset causes go, and Missing comes for a packet without a record. - nvx::host::template: ClonePlan::launch passes the record that it writes to the sandbox with --restore-activation-record, and the restore sequence asks for Ready, CloneControl::ready in place of activate, so that a session that replaces a lost one resends nothing. A reattach asks for Ready too, and checks it against that record. - nvx::activation::CONTRACT_VERSION, 2, which template keys hold as their activation contract, versions how a clone receives its record, so that no build restores a template whose guest awaits an Activate. The record's schema, VERSION, stays 1. - The runtime adapter's Await, kind 28, replaces Activate, kind 27, which no message takes, and its worker applies the record at the restore's reset, once it wrote its acknowledgement, so that the host's attachment overlaps the application. Its activation probe also reports when the application completed. Tests: the guest activation's state machine, applied before or after the host's request, its deadline, resets, and every failure class; the launch's arguments and the restore sequence with a fake VMM that keeps the record that the launch supplied, retries that resend nothing, and the reattach; the template key's activation contract; the adapter's framing, client, and worker against a fake root. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- nvx-rs/README.md | 181 ++-- nvx-rs/public-api/linux.txt | 22 +- nvx-rs/public-api/windows.txt | 3 +- nvx-rs/src/activation/mod.rs | 16 +- nvx-rs/src/contract.rs | 5 +- nvx-rs/src/guest/clone.rs | 100 ++- nvx-rs/src/guest/clone/activation.rs | 605 ++++++------- nvx-rs/src/guest/clone/activation/apply.rs | 12 +- .../src/guest/clone/activation/apply/tests.rs | 249 +++--- nvx-rs/src/guest/clone/activation/tests.rs | 802 +++++++++--------- nvx-rs/src/guest/clone/repair.rs | 22 +- nvx-rs/src/guest/mod.rs | 4 +- nvx-rs/src/guest/time/api.rs | 3 +- nvx-rs/src/host/template/clone/failure.rs | 12 +- nvx-rs/src/host/template/clone/launch.rs | 18 +- nvx-rs/src/host/template/clone/reattach.rs | 44 +- nvx-rs/src/host/template/clone/records.rs | 6 +- nvx-rs/src/host/template/clone/restore.rs | 162 ++-- .../src/host/template/clone/tests/launch.rs | 104 ++- .../src/host/template/clone/tests/restore.rs | 250 +++--- nvx-rs/src/host/template/key.rs | 7 +- nvx-rs/src/host/template/key/tests.rs | 16 + nvx-rs/src/host/template/mod.rs | 17 +- nvx-rs/src/protocol/status.rs | 30 +- tests/runtime-adapter/README.md | 324 +++---- .../src/bin/runtime-adapter-host.rs | 45 +- tests/runtime-adapter/src/frame.rs | 53 +- tests/runtime-adapter/src/frame/tests.rs | 53 +- tests/runtime-adapter/src/hook.rs | 4 +- tests/runtime-adapter/src/host.rs | 94 +- tests/runtime-adapter/src/lib.rs | 5 +- tests/runtime-adapter/src/root.rs | 7 +- tests/runtime-adapter/src/worker.rs | 96 +-- .../runtime-adapter/src/worker/activation.rs | 191 +++-- .../src/worker/activation/tests.rs | 98 +-- tests/runtime-adapter/src/worker/capture.rs | 10 +- tests/runtime-adapter/tests/guest_session.rs | 501 ++++++----- tests/runtime-adapter/tests/host_session.rs | 141 ++- tests/runtime-adapter/tests/vm_clone.rs | 48 +- 39 files changed, 2138 insertions(+), 2222 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index e181a90ce..166231f68 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -305,6 +305,13 @@ more for a cold boot, in a canonical encoding of `COLD_LEN` or `CLONE_LEN` bytes, 1 or 17, that `ReadyActivation::encode` writes and `ReadyActivation::decode` reads. +`CONTRACT_VERSION`, 2, which a template's compatibility key holds, versions the +activation contract: the record's schema, at `VERSION`, and how a clone receives +its record, at the end of its restore packet, which OpenVMM seals before the +first restored vCPU runs, rather than in the adapter's Activate of version 1, so +that no build restores a clone of a template whose guest takes its record +otherwise. + The records and both encodings need no features. ### `nvx::contract` @@ -853,24 +860,27 @@ lease, that the sandbox records no failure of the clone, which fails once under its lease, and, with `ExportPlan::verify`, that the objects of its export, if it has one, are still those that its plan found, and returns a `Restoring`: it writes the clone's activation record in its canonical encoding, -`clone-activation.bin`, from which a caller resends it unchanged; it listens for -the VMM's restore-ready event, at the socket `restore.sock` on Linux, and at a -named pipe of a random name on Windows; and it runs `supervisor::launch` with -OpenVMM's restore of the clone's devices, with `--restore-absent-scratch` for a -clone in RAM, and of its network, whose policy `--restore-network-policy` -installs on the template's NIC in place of the complete deny-all policy, its -export bound to the dormant slot as a cold launch attaches it, -`--mount-aggregate` with a `--mount-child` of each of its directories and files, -their hidden, re-exposed, and writable paths, and its owner, -`--restore-snapshot` with the generation's snapshot in place, which no clone -copies or changes, so that clones of one generation restore it at once, +`clone-activation.bin`, which OpenVMM hands the clone's guest at the end of its +restore packet, and from which a caller that takes the clone over checks its +Ready; it listens for the VMM's restore-ready event, at the socket +`restore.sock` on Linux, and at a named pipe of a random name on Windows; and it +runs `supervisor::launch` with OpenVMM's restore of the clone's devices, with +`--restore-absent-scratch` for a clone in RAM, and of its network, whose policy +`--restore-network-policy` installs on the template's NIC in place of the +complete deny-all policy, its export bound to the dormant slot as a cold launch +attaches it, `--mount-aggregate` with a `--mount-child` of each of its +directories and files, their hidden, re-exposed, and writable paths, and its +owner, `--restore-snapshot` with the generation's snapshot in place, which no +clone copies or changes, so that clones of one generation restore it at once, `--restore-memory` with the memory target, `--restore-processors` with the processor target, for a template whose processors were not all online at boot, -and `--restore-ready-path`, under the -template's backend, CPU profile, and processors, with a boot console that -discards its output, as cold boots have, unless the plan asks for one that the -caller reads, and the control console, but no kernel, initramfs, command line, -or guest address, which the snapshot fixes. +`--restore-activation-record` with `clone-activation.bin`, which OpenVMM reads +before any restored vCPU runs, and seals in the restore packet, gating the +host's input until the guest acknowledged the restore, and +`--restore-ready-path`, under the template's backend, CPU profile, and +processors, with a boot console that discards its output, as cold boots have, +unless the plan asks for one that the caller reads, and the control console, but +no kernel, initramfs, command line, or guest address, which the snapshot fixes. On Linux, where OpenVMM requires a restore's control socket in the snapshot's parent, the clone's VMM listens in the generation's payload, at `.sock`, @@ -911,13 +921,15 @@ refusing another process's connection and bytes that differ from the event, and reading nothing past it, since OpenVMM on Windows waits until the host has read it, and then closes its end; only then connects to the clone's control endpoint, whose console bounds the host's authentication from the connection on; attaches -with the launch's capability; sends the record, read again from -`clone-activation.bin`, which must still be byte for byte the launch's, in the -adapter's Activate, as the session's first request; and checks that Ready -reports the record's transaction ID. It replaces a session that the adapter -lost, as `CloneControl::lost` tells, with a new one that sends the same record, -in at most `MAX_ACTIVATION_SESSIONS` sessions by the same deadline, and retries -nothing else. +with the launch's capability; asks for the clone's Ready, as the session's first +request, `CloneControl::ready`, which the clone answers once it activated with +the record of its restore packet, whatever the host did meanwhile; and checks +that Ready reports the record's transaction ID. The host asks, rather than the +clone sending Ready unasked, since OpenVMM's broker refuses the guest's records +while no host is attached, which the guest cannot tell. It replaces a session +that the adapter lost, as `CloneControl::lost` tells, with a new one that asks +again, in at most `MAX_ACTIVATION_SESSIONS` sessions by the same deadline, and +retries nothing else. `Activated::start` then sends the clone's mapping table, if it has an export, through `CloneControl::map`, in order, as a cold launch's guest receives it, @@ -934,15 +946,15 @@ runs. A failure of a clone is a `CloneFailure`, whose `Evidence` attributes it to an `Outcome`, at a `Stage`, once the clone got as far as its `Progress`: the request, for a denied authorization, a stale binding, invalid targets, format, -settings, Activate, or mappings, such as an export that no clone takes, or whose -objects changed since its plan, or a mapping table that the guest refused, as -`CloneControl::refused` tells, or an image, a scratch, or a root that the -template cannot serve, and for a supplied file of the wrong identity, length, or -geometry, which the host's staging takes instead once the caller proves it, -`CloneFailure::with_staging_proven`; the host, for an endpoint that cannot be -bound, resource exhaustion, a host that cannot restore the template, another -runtime, or a failure of its own operations; unknown, for a lost status, a -timeout, a reset, a forced kill, or a broken protocol; the template, for a +settings, activation record, or mappings, such as an export that no clone takes, +or whose objects changed since its plan, or a mapping table that the guest +refused, as `CloneControl::refused` tells, or an image, a scratch, or a root +that the template cannot serve, and for a supplied file of the wrong identity, +length, or geometry, which the host's staging takes instead once the caller +proves it, `CloneFailure::with_staging_proven`; the host, for an endpoint that +cannot be bound, resource exhaustion, a host that cannot restore the template, +another runtime, or a failure of its own operations; unknown, for a lost status, +a timeout, a reset, a forced kill, or a broken protocol; the template, for a malformed generation, or a defect of its capture or repair that a diagnosis attributes to it, the only outcome that quarantines the generation, through its tombstone, `CloneFailure::quarantine`; and the repair stage, for a status of the @@ -986,22 +998,23 @@ failure as the sequence's own failures do. `reattach_clone` takes a clone over after the caller of its launch restarted: it reattaches by the identity of its VMM's process, checks a committed clone in a -new session, in which the clone answers the byte-equal Activate of -`clone-activation.bin` with the Ready that it answered once, and ends a clone -that another runtime identity launched, or that fails the check, through its -verified identity; it records the failure of a launch that a crash interrupted, -but not the journal that a commit may leave beside its runtime record, -classifies a VM that exited by its outcome report, which is the clone's VM's -once its launch began, whether or not a crash left a record of its spawn, -records a failure before it ends a VMM or clears the runtime record of one that -exited, and fails, leaving them as they were, if it cannot, finishes a failure -that a crash left pending as its teardown would have recorded it, keeping its -transaction, resolves the teardown of an earlier failure with what the VMM left -as it ended, its outcome report and OpenVMM's fatal error, attributes a status -of the guest's as one that a workload may have given once the clone's progress -log is lost, damaged, or was written during another boot of the host, whose -crash may have lost its last lines, and reads nothing of the template store, so -that a tombstone or a retirement of the generation leaves the clone its lease. +new session, in which the clone answers the request for its Ready with the Ready +of the transaction of `clone-activation.bin`, which it answers in every session, +and ends a clone that another runtime identity launched, or that fails the +check, through its verified identity; it records the failure of a launch that a +crash interrupted, but not the journal that a commit may leave beside its +runtime record, classifies a VM that exited by its outcome report, which is the +clone's VM's once its launch began, whether or not a crash left a record of its +spawn, records a failure before it ends a VMM or clears the runtime record of +one that exited, and fails, leaving them as they were, if it cannot, finishes a +failure that a crash left pending as its teardown would have recorded it, +keeping its transaction, resolves the teardown of an earlier failure with what +the VMM left as it ended, its outcome report and OpenVMM's fatal error, +attributes a status of the guest's as one that a workload may have given once +the clone's progress log is lost, damaged, or was written during another boot of +the host, whose crash may have lost its last lines, and reads nothing of the +template store, so that a tombstone or a retirement of the generation leaves the +clone its lease. `CloneRecords::read` reads what the sandbox keeps of a clone, for the operations that list, inspect, or remove clones. @@ -1429,9 +1442,10 @@ until then, which hands the deferred time work to the time daemon that the caller supervises. It returns the `Repaired` clone, with its generation ID, its online CPUs and -memory, and when it acknowledged the restore, and never touches the fence. It -adds no memory block twice, and reseeds and finishes at most once, so a failure -at or after the reseed is final. +memory, when it acknowledged the restore, and the activation record that the +restore packet carried, and never touches the fence. It adds no memory block +twice, and reseeds and finishes at most once, so a failure at or after the +reseed is final. Each failure is a `RepairError`, with its `RepairStage`, its `RepairCause`, and the repair's `RepairProgress`, and `RepairError::status` selects the status, of @@ -1442,22 +1456,20 @@ of it fails them, with its status, rather than stopping the VM, and only the capture's preparation and the time daemon enforce it. An `Activation` of the `Repaired` clone, or, with `Activation::with_resources`, -of the resources and the instant of the acknowledgement that a repair reports, -as in an adapter's tests, and of the caller's closed `AdmissionFence` then takes -the clone's own configuration from the host, in one Activate, on a restricted -control session: a state machine without I/O, independent of any framing, to -which the adapter passes what its session receives, as an `ActivationInput`, and -which returns the `ActivationAction` that the adapter takes. - -From its transport bootstrap, which takes the reset that the restore queued for -the NVXS session, the restricted session takes only flow control and one -Activate, and anything else, a request or another reset, fails it; the -activation decodes the Activate's record with `ActivationRecord::decode`, and +of the resources, the record, and the instant of the acknowledgement that a +repair reports, as in an adapter's tests, and of the caller's closed +`AdmissionFence` then takes the clone's own configuration from the activation +record that its restore packet carried, which the host supplied at the launch, +and OpenVMM sealed before the first restored vCPU ran: a state machine without +I/O, independent of any framing, to which the adapter passes what its session +receives, as an `ActivationInput`, and which returns the `ActivationAction` that +the adapter takes. + +`Activation::accept` decodes the record with `ActivationRecord::decode`, and accepts only a clone's, with a restore section, that expects the CPU count and -the memory size that the repair reports, before any side effect, while an -Activate whose record the adapter's framing refused unread, as longer than a -record can be, fails as a record that long does; the adapter applies the record -with `apply`, as at a cold boot: it assembles the root as `root::assemble` does, +the memory size that the repair reports, before any side effect, while a packet +without a record fails the activation; the adapter applies the record with +`apply`, as at a cold boot: it assembles the root as `root::assemble` does, with the record's image, an EROFS image whose UUID it checks if the record expects one, or a GPT disk whose GUID it checks if the record expects one, its upper, a tmpfs or the scratch with the record's journal policy, and its overlay @@ -1506,20 +1518,29 @@ mounted or bound stays mounted unless the error reports the unmounts that failed, and no announcer that it started runs. Once the adapter passed the activation what the session received while the -application ran, which came before Ready, `Activation::complete` opens the fence -and returns the `ReadyActivation` that Ready reports beside the adapter's own -identities, that the activation applied, with the record's transaction ID, after -which the host's mapping table and start follow in the adapter's order, as after -a cold boot, and `Activation::start` records the start. - -A retry with the same transaction ID and a byte-equal record joins the running -activation, or gets its cached result, also in a later session once the clone is -ready; a changed record, another transaction, another reset before Ready, or a -failed activation is final. - -Its deadline, which the caller chooses, and which neither input nor retries -extend, starts at the restore's acknowledgement, and the adapter ticks it -however its input trickles and its output blocks. Each failure is an +application ran, which came before Ready, `Activation::complete` returns the +`ReadyActivation` that Ready reports beside the adapter's own identities, that +the activation applied, with the record's transaction ID. The activation is +ready, which opens the fence, once the record applied and the host asked for +Ready: at the completion if a request for Ready joined the application, and +otherwise at the host's next request, until which the activation is `Applied`, +its fence closed and its deadline holding, so that a clone whose host never asks +ends. The host's mapping table and start then follow in the adapter's order, as +after a cold boot, and `Activation::start` records the start. + +The clone's session runs beside the activation: the adapter acknowledges each +reset, the one that the restore queued for the NVXS session and any later one, +and answers the host's request for Ready, the first request of each of its +sessions, with Ready, since OpenVMM's broker refuses the guest's records while +no host is attached, which the guest cannot tell. A request for Ready while the +record applies joins the activation, and gets what its completion returns unless +a reset abandoned it, one once the record applied makes the activation ready, +and one once the clone is ready gets the cached result, also in a later session; +any other request before Ready, or a failed activation, is final. + +Its deadline, which the caller chooses, and which nothing extends, starts at the +restore's acknowledgement, and the adapter ticks it however its input trickles +and its output blocks. Each failure is an `ActivationFailure`, with its `ActivationState` and `ActivationCause`, and never powers the VM off: the adapter ends it. diff --git a/nvx-rs/public-api/linux.txt b/nvx-rs/public-api/linux.txt index 8895dcb86..1d704930d 100644 --- a/nvx-rs/public-api/linux.txt +++ b/nvx-rs/public-api/linux.txt @@ -126,6 +126,7 @@ method nvx::activation::ActivationRecord::encode method nvx::activation::ActivationRecord::new method nvx::activation::ActivationRecord::restore method nvx::activation::ActivationRecord::version +const nvx::activation::CONTRACT_VERSION struct nvx::activation::Configuration method nvx::activation::Configuration::check field nvx::activation::Configuration::export @@ -614,6 +615,7 @@ variant nvx::guest::cgroup::Scope::Execution variant nvx::guest::cgroup::Scope::Workload mod nvx::guest::clone struct nvx::guest::clone::Activation +method nvx::guest::clone::Activation::accept method nvx::guest::clone::Activation::complete method nvx::guest::clone::Activation::deadline method nvx::guest::clone::Activation::established @@ -625,48 +627,39 @@ method nvx::guest::clone::Activation::tick method nvx::guest::clone::Activation::with_resources enum nvx::guest::clone::ActivationAction variant nvx::guest::clone::ActivationAction::Acknowledge -variant nvx::guest::clone::ActivationAction::Apply -field nvx::guest::clone::ActivationAction::Apply::0 variant nvx::guest::clone::ActivationAction::Continue variant nvx::guest::clone::ActivationAction::Join variant nvx::guest::clone::ActivationAction::Ready field nvx::guest::clone::ActivationAction::Ready::0 variant nvx::guest::clone::ActivationAction::Serve enum nvx::guest::clone::ActivationCause -variant nvx::guest::clone::ActivationCause::Changed variant nvx::guest::clone::ActivationCause::Cold variant nvx::guest::clone::ActivationCause::Completion variant nvx::guest::clone::ActivationCause::Expired variant nvx::guest::clone::ActivationCause::Failed field nvx::guest::clone::ActivationCause::Failed::0 +variant nvx::guest::clone::ActivationCause::Missing variant nvx::guest::clone::ActivationCause::Record field nvx::guest::clone::ActivationCause::Record::0 variant nvx::guest::clone::ActivationCause::Request -variant nvx::guest::clone::ActivationCause::Reset variant nvx::guest::clone::ActivationCause::Resources field nvx::guest::clone::ActivationCause::Resources::actual field nvx::guest::clone::ActivationCause::Resources::expected variant nvx::guest::clone::ActivationCause::Start -variant nvx::guest::clone::ActivationCause::Transaction -field nvx::guest::clone::ActivationCause::Transaction::accepted -field nvx::guest::clone::ActivationCause::Transaction::offered struct nvx::guest::clone::ActivationFailure method nvx::guest::clone::ActivationFailure::cause method nvx::guest::clone::ActivationFailure::state enum nvx::guest::clone::ActivationInput -variant nvx::guest::clone::ActivationInput::Activate -field nvx::guest::clone::ActivationInput::Activate::0 +variant nvx::guest::clone::ActivationInput::Await variant nvx::guest::clone::ActivationInput::Flow variant nvx::guest::clone::ActivationInput::Request variant nvx::guest::clone::ActivationInput::Reset -variant nvx::guest::clone::ActivationInput::TooLong -field nvx::guest::clone::ActivationInput::TooLong::0 enum nvx::guest::clone::ActivationState variant nvx::guest::clone::ActivationState::Activating -variant nvx::guest::clone::ActivationState::Bootstrap +variant nvx::guest::clone::ActivationState::Applied variant nvx::guest::clone::ActivationState::Failed +variant nvx::guest::clone::ActivationState::Pending variant nvx::guest::clone::ActivationState::Ready -variant nvx::guest::clone::ActivationState::Restricted variant nvx::guest::clone::ActivationState::Started trait nvx::guest::clone::AdmissionFence method nvx::guest::clone::AdmissionFence::close @@ -814,6 +807,7 @@ variant nvx::guest::clone::RepairStage::Reseed variant nvx::guest::clone::RepairStage::Restore struct nvx::guest::clone::Repaired method nvx::guest::clone::Repaired::acknowledged_at +method nvx::guest::clone::Repaired::activation_record method nvx::guest::clone::Repaired::cpus method nvx::guest::clone::Repaired::generation_id method nvx::guest::clone::Repaired::memory @@ -2695,10 +2689,10 @@ method nvx::host::template::ClassIdentity::from_bytes trait nvx::host::template::CloneControl type nvx::host::template::CloneControl::Error type nvx::host::template::CloneControl::Session -method nvx::host::template::CloneControl::activate method nvx::host::template::CloneControl::attach method nvx::host::template::CloneControl::lost method nvx::host::template::CloneControl::map +method nvx::host::template::CloneControl::ready method nvx::host::template::CloneControl::refused method nvx::host::template::CloneControl::start struct nvx::host::template::CloneFailure diff --git a/nvx-rs/public-api/windows.txt b/nvx-rs/public-api/windows.txt index 7b2d8a0e8..b52056edd 100644 --- a/nvx-rs/public-api/windows.txt +++ b/nvx-rs/public-api/windows.txt @@ -126,6 +126,7 @@ method nvx::activation::ActivationRecord::encode method nvx::activation::ActivationRecord::new method nvx::activation::ActivationRecord::restore method nvx::activation::ActivationRecord::version +const nvx::activation::CONTRACT_VERSION struct nvx::activation::Configuration method nvx::activation::Configuration::check field nvx::activation::Configuration::export @@ -1631,10 +1632,10 @@ method nvx::host::template::ClassIdentity::from_bytes trait nvx::host::template::CloneControl type nvx::host::template::CloneControl::Error type nvx::host::template::CloneControl::Session -method nvx::host::template::CloneControl::activate method nvx::host::template::CloneControl::attach method nvx::host::template::CloneControl::lost method nvx::host::template::CloneControl::map +method nvx::host::template::CloneControl::ready method nvx::host::template::CloneControl::refused method nvx::host::template::CloneControl::start struct nvx::host::template::CloneFailure diff --git a/nvx-rs/src/activation/mod.rs b/nvx-rs/src/activation/mod.rs index 9a2972203..be4414fce 100644 --- a/nvx-rs/src/activation/mod.rs +++ b/nvx-rs/src/activation/mod.rs @@ -189,10 +189,22 @@ mod tests; pub use self::codec::Field; -/// The version of the activation contract that this build speaks: the schema of an -/// [`ActivationRecord`], which every record names. A record of another version is refused. +/// The version of the schema of an [`ActivationRecord`] that this build speaks, which every +/// record names. A record of another version is refused. pub const VERSION: u16 = 1; +/// The version of the activation contract that this build speaks, which the compatibility key of +/// a template holds: the schema of an [`ActivationRecord`], at [`VERSION`], and how a clone of +/// the template receives it, so that a template whose guest takes its record another way restores +/// no clone of this build. +/// +/// Version 1 sent the record in the adapter's Activate, on the clone's control session, once the +/// host attached. Version 2 hands it to the clone's guest at the end of its restore packet, which +/// OpenVMM seals before the first restored vCPU runs, from `--restore-activation-record`, and the +/// host asks for the clone's Ready instead: a guest of version 1, which predates the packet's +/// activation record, refuses a packet that carries one, or awaits an Activate that never comes. +pub const CONTRACT_VERSION: u64 = 2; + /// The most bytes of a record's encoding, so that a restore packet could carry the same bytes. pub const MAX_RECORD_BYTES: usize = 1024; diff --git a/nvx-rs/src/contract.rs b/nvx-rs/src/contract.rs index 52fe8d284..2be3b30f1 100644 --- a/nvx-rs/src/contract.rs +++ b/nvx-rs/src/contract.rs @@ -44,8 +44,9 @@ //! Each of these four must match the pinned OpenVMM's, which the ignored test //! `tests/openvmm_vectors.rs` checks in the jobs that check out the OpenVMM submodule. The key //! also holds the version of the activation contract, -//! [`activation::VERSION`](crate::activation::VERSION), through which each clone receives the -//! settings of its own launch. +//! [`activation::CONTRACT_VERSION`](crate::activation::CONTRACT_VERSION): the schema of the +//! activation record through which each clone receives the settings of its own launch, and how it +//! receives it, at the end of its restore packet. //! //! The build script includes this file to check bundled artifacts, so it must not depend on the //! rest of the crate. diff --git a/nvx-rs/src/guest/clone.rs b/nvx-rs/src/guest/clone.rs index ccfa542a3..6a1e15621 100644 --- a/nvx-rs/src/guest/clone.rs +++ b/nvx-rs/src/guest/clone.rs @@ -81,15 +81,16 @@ //! reports them as violations of the time ABI, which stop the VM, with the time ABI's status, //! unless the ABI is report-only, since the daemon enforces it. //! -//! It returns the [`Repaired`] clone: its new VM generation ID, its number of online CPUs, the -//! size of its online memory blocks, and when it acknowledged the restore, which an activation -//! checks against the resources that it expects. The repair records how far it got, so that it -//! never brings a CPU online twice nor adds a memory block twice, and runs the reseed and the -//! finish at most once, whoever retries it: a failure at or after the reseed is final. It never -//! touches the fence. A failure is a [`RepairError`], which names the [`RepairStage`] that -//! failed, its [`RepairCause`], and the repair's [`RepairProgress`], and selects the status with -//! which the caller ends the VM, [`RepairError::status`], which `nvx::protocol::status` -//! documents. +//! It returns the [`Repaired`] clone: its new VM generation ID, its number of online CPUs and the +//! size of its online memory blocks, which an activation checks against the resources that it +//! expects, when it acknowledged the restore, and the activation record that its restore packet +//! carried, from which the activation takes the clone's configuration. The repair records how far +//! it got, so that it never brings a CPU online twice nor adds a memory block twice, and runs the +//! reseed and the finish at most once, whoever retries it: a failure at or after the reseed is +//! final. It never touches the fence. A failure is a [`RepairError`], which names the +//! [`RepairStage`] that failed, its [`RepairCause`], and the repair's [`RepairProgress`], and +//! selects the status with which the caller ends the VM, [`RepairError::status`], which +//! `nvx::protocol::status` documents. //! //! # Strict mode //! @@ -106,44 +107,54 @@ //! //! # Activation //! -//! A repaired clone then takes its own configuration from the host, in one Activate, on a -//! restricted control session, through an [`Activation`] of the [`Repaired`] clone and of the -//! caller's fence, which stays closed until the activation is ready, so that no execution starts -//! before. The activation is a state machine without I/O, independent of any framing: the adapter -//! passes it what it receives on the clone's NVXS session, each an [`ActivationInput`], and does -//! what each [`ActivationAction`] that it returns says. Its [`ActivationState`]s, in order, are: +//! A repaired clone then takes its own configuration from the activation record that its restore +//! packet carried, [`Repaired::activation_record`], through an [`Activation`] of the +//! [`Repaired`] clone and of the caller's fence, which stays closed until the activation is +//! ready, so that no execution starts before. The host supplied the record at the clone's launch, +//! and OpenVMM sealed it in the packet before the first restored vCPU ran, so the clone takes +//! nothing of the host to activate. The activation is a state machine without I/O, independent of +//! any framing: the adapter applies the record that the activation accepts, passes the activation +//! what it receives on the clone's NVXS session, each an [`ActivationInput`], and does what each +//! [`ActivationAction`] that it returns says. Its [`ActivationState`]s, in order, are: //! -//! 1. the transport bootstrap: the clone awaits the reset that the restore queued for the session -//! that the guest attached before the capture, which the adapter acknowledges; -//! 2. restricted: the session takes only flow control and one Activate, and anything else, a -//! request or another reset, fails the activation. The activation decodes the Activate's -//! record with [`ActivationRecord::decode`](crate::activation::ActivationRecord::decode), and -//! accepts it only as a clone's, with a restore section, that expects the CPU count and the -//! memory size that the repair reports, before anything has a side effect. An Activate whose -//! record the adapter's framing refused unread, as longer than a record can be, fails as a -//! record that long does; -//! 3. activating: the adapter applies the accepted record with `apply`, which +//! 1. pending: [`Activation::accept`] decodes the record with +//! [`ActivationRecord::decode`](crate::activation::ActivationRecord::decode), and accepts it +//! only as a clone's, with a restore section, that expects the CPU count and the memory size +//! that the repair reports, before anything has a side effect. A packet without a record fails +//! the activation. The adapter accepts the record as soon as the clone runs, without waiting +//! for the host; +//! 2. activating: the adapter applies the accepted record with `apply`, which //! activates the root and the workload with its settings, as below, passes the activation //! what the session received meanwhile, which came before Ready, then reports the result to //! [`Activation::complete`]; -//! 4. ready, with the mappings pending: the activation completed, opened the fence, and returned -//! what Ready reports beside the adapter's own identities, such as its build, a -//! [`ReadyActivation`](crate::activation::ReadyActivation): that the activation applied, with -//! the record's transaction ID. The host's mapping table, which the record announced, and the -//! start of the sandbox follow in the adapter's order, as after a cold boot; +//! 3. applied, if the application completed before the host asked for Ready in its current +//! session: the activation awaits the host's request, which makes it ready; +//! 4. ready, with the mappings pending: the application completed, and the host asked for Ready, +//! which opened the fence, and got what Ready reports beside the adapter's own identities, such +//! as its build, a [`ReadyActivation`](crate::activation::ReadyActivation): that the activation +//! applied, with the record's transaction ID. The host's mapping table, which the record +//! announced, and the start of the sandbox follow in the adapter's order, as after a cold boot; //! 5. started: the adapter's order started the sandbox, which [`Activation::start`] records. //! -//! The host retries an Activate with the same transaction ID and a byte-equal record, and such a -//! retry joins the activation while it runs, or gets its cached result afterwards, without -//! activating again, also in a later session, such as once the host's attachment ended, whose -//! reset a ready clone takes. A changed record, another transaction, another reset before the +//! The clone's control session runs beside the activation. The clone takes the reset that the +//! restore queued for the session that the guest attached before the capture, which the adapter +//! acknowledges, and the host then asks for the clone's Ready, first in each of its sessions: +//! OpenVMM's broker refuses the guest's records while no host is attached, which the guest cannot +//! tell, so the guest answers the host's request rather than sending Ready unasked. A request for +//! Ready that comes while the record applies joins the activation, whose completion answers it, +//! one that comes once the record applied makes the activation ready, and gets its Ready, and one +//! that comes once it is ready gets the established Ready, without activating again: the clone is +//! ready only once a Ready answers the host, so that its deadline ends a clone whose host never +//! asks. A reset of the session, such as once the host's attachment ended, changes nothing of the +//! activation, whose record came with the restore, so that a later session gets the same Ready: +//! it abandons only the requests for Ready of the earlier session. Any other request before the //! clone is ready, and an activation that failed, wholly or partway, are final. The activation's //! deadline, which the caller chooses, starts at the restore's acknowledgement, or, for a restore -//! that the host did not gate until one, when the caller makes the activation, and neither input -//! nor retries extend it: the adapter wakes by it, and [ticks](Activation::tick) the activation -//! then, however slowly its input trickles in, and whether or not its output drains. A failure -//! is an [`ActivationFailure`], which names the [`ActivationState`] in which the activation failed -//! and its [`ActivationCause`], and which every later call returns again. The activation never +//! that the host did not gate until one, when the caller makes the activation, and nothing +//! extends it: the adapter wakes by it, and [ticks](Activation::tick) the activation then, +//! however slowly its input trickles in, and whether or not its output drains. A failure is an +//! [`ActivationFailure`], which names the [`ActivationState`] in which the activation failed and +//! its [`ActivationCause`], and which every later call returns again. The activation never //! powers the VM off: the adapter reports the failure, and ends the VM. //! //! The activation's application, `apply` and its types, is NVX's internal reference code of the @@ -253,11 +264,14 @@ //! } //! }; //! println!("{} CPUs, {} bytes of memory", repaired.cpus(), repaired.memory()); -//! // Its activation takes the host's Activate, and opens the fence once the clone is -//! // ready: the agent drives it with what its control session receives. +//! // Its activation takes the activation record of the restore packet, and opens the +//! // fence once the clone is ready: the agent applies the record, and serves the clone's +//! // control session. //! let timeout = Duration::from_secs(30); -//! let activation = Activation::new(&repaired, fence, timeout, Instant::now()); -//! println!("activate by {:?}", activation.deadline()); +//! let mut activation = Activation::new(&repaired, fence, timeout, Instant::now()); +//! let record = activation.accept(Instant::now())?; +//! let mappings = record.configuration().mappings; +//! println!("activate {mappings} mappings by {:?}", activation.deadline()); //! } //! } //! # Ok(()) diff --git a/nvx-rs/src/guest/clone/activation.rs b/nvx-rs/src/guest/clone/activation.rs index f9af8ae4f..e9ba20fa0 100644 --- a/nvx-rs/src/guest/clone/activation.rs +++ b/nvx-rs/src/guest/clone/activation.rs @@ -1,7 +1,8 @@ //! The activation of a repaired clone, in the caller's process: [`Activation`], the state machine -//! of the clone's restricted control session, through which the clone takes its configuration from -//! the host in one Activate, and [`apply`](fn@apply), which activates the clone's root and workload -//! with that configuration, as the [module](super#activation) describes. +//! through which the clone takes its configuration from the activation record of its restore +//! packet, and through which its control session reports the activation to the host, and +//! [`apply`](fn@apply), which activates the clone's root and workload with that configuration, as +//! the [module](super#activation) describes. mod apply; #[cfg(test)] @@ -13,9 +14,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use super::{AdmissionFence, Repaired}; -use crate::activation::{ - ActivationError, ActivationRecord, ExpectedResources, ReadyActivation, TransactionId, -}; +use crate::activation::{ActivationError, ActivationRecord, ExpectedResources, ReadyActivation}; pub use self::apply::{ Applied, AppliedTeardownError, ApplyError, MACHINE_ID_FILE, NetworkError, RESOLVER_FILE, @@ -23,29 +22,37 @@ pub use self::apply::{ }; /// The activation of a repaired clone: the state machine, without I/O and independent of any -/// framing, of the clone's control session, which takes the clone's configuration from the host -/// in one Activate, behind the caller's fence of workload admission, which it opens once the -/// activation is ready; see the [module](super#activation). +/// framing, through which the clone takes its configuration from the activation record that its +/// restore packet carried, behind the caller's fence of workload admission, which it opens once +/// the activation is ready, and through which the clone's control session reports the activation +/// to the host; see the [module](super#activation). /// -/// The adapter passes it what it receives on the session, each an [`ActivationInput`], through +/// The adapter takes the record with [`accept`](Self::accept), which decodes and validates it, +/// as soon as the clone runs, without waiting for the host, applies it with `apply`, whose waits +/// end by the activation's [`deadline`](Self::deadline), passes the activation what the session +/// received while the application ran, which came before Ready, then reports the result through +/// [`complete`](Self::complete), whose Ready answers each of the host's requests for it that +/// [joined](ActivationAction::Join) the application, and it reports the start of the sandbox, +/// which its own order takes once the activation is ready, through [`start`](Self::start). It +/// passes the activation what it receives on the session, each an [`ActivationInput`], through /// [`input`](Self::input), and does what the [`ActivationAction`] that it returns says. The -/// adapter applies the record of the Activate that the activation accepted, with -/// `apply`, whose waits end by the activation's [`deadline`](Self::deadline), passes -/// the activation what the session received while the application ran, which came before Ready, -/// then reports the result through [`complete`](Self::complete), whose Ready answers the Activate -/// and each retry that [joined](ActivationAction::Join) it, and it reports the start of the -/// sandbox, which its own order takes once the activation is ready, through -/// [`start`](Self::start). The adapter's loop wakes by the deadline, and [ticks](Self::tick) the -/// activation then, whether input arrives or not, and whether its output drains or not. +/// adapter's loop wakes by the deadline, and [ticks](Self::tick) the activation then, whether +/// input arrives or not, and whether its output drains or not. +/// +/// The activation is ready once its application succeeded and the host asked for Ready, which +/// the host does first in each of its sessions, so that the activation is ready only once a Ready +/// answers the host: an application that completes before the host's request leaves the +/// activation [applied](ActivationState::Applied), and the host's next request for Ready makes it +/// ready. Until then, the deadline holds, so that a clone whose host never asks ends, and any +/// other request of the host fails the activation. /// /// It goes through the [`ActivationState`]s in order, and each takes these inputs: /// -/// | State | Reset | Flow | Activate | Request | +/// | State | Reset | Flow | Await | Request | /// | --- | --- | --- | --- | --- | -/// | [`Bootstrap`](ActivationState::Bootstrap) | [`Acknowledge`](ActivationAction::Acknowledge), restricted next | [`Continue`](ActivationAction::Continue) | Fails | Fails | -/// | [`Restricted`](ActivationState::Restricted) | Fails | [`Continue`](ActivationAction::Continue) | [`Apply`](ActivationAction::Apply), activating next, once its record is valid | Fails | -/// | [`Activating`](ActivationState::Activating) | Fails | [`Continue`](ActivationAction::Continue) | [`Join`](ActivationAction::Join) if byte-equal | Fails | -/// | [`Ready`](ActivationState::Ready), [`Started`](ActivationState::Started) | [`Acknowledge`](ActivationAction::Acknowledge), a later session | [`Continue`](ActivationAction::Continue) | [`Ready`](ActivationAction::Ready) if byte-equal | [`Serve`](ActivationAction::Serve) | +/// | [`Pending`](ActivationState::Pending), [`Activating`](ActivationState::Activating) | [`Acknowledge`](ActivationAction::Acknowledge), which abandons the requests for Ready that joined | [`Continue`](ActivationAction::Continue) | [`Join`](ActivationAction::Join) | Fails | +/// | [`Applied`](ActivationState::Applied) | [`Acknowledge`](ActivationAction::Acknowledge) | [`Continue`](ActivationAction::Continue) | [`Ready`](ActivationAction::Ready), which makes the activation ready | Fails | +/// | [`Ready`](ActivationState::Ready), [`Started`](ActivationState::Started) | [`Acknowledge`](ActivationAction::Acknowledge), a later session | [`Continue`](ActivationAction::Continue) | [`Ready`](ActivationAction::Ready) | [`Serve`](ActivationAction::Serve) | /// /// Every failure is final, and leaves the activation [failed](ActivationState::Failed): every /// later call returns the same [`ActivationFailure`]. A failure before the activation is ready @@ -69,11 +76,8 @@ pub use self::apply::{ /// Reset, /// /// Bytes that complete no message, such as a part of a DATA record's payload. /// Partial, -/// /// The host's Activate, with its record's bytes. -/// Activate(Vec), -/// /// The host's Activate, whose header announced this many bytes, more than a record has, -/// /// which the framing refused before it read any of them. -/// TooLong(usize), +/// /// The host's request for the clone's Ready, the first message of each of its sessions. +/// Await, /// /// Any other message of the host, such as a part of the mapping table, or the start. /// Other(Vec), /// } @@ -102,51 +106,56 @@ pub use self::apply::{ /// fn serve(&mut self, message: Vec) -> bool; /// } /// -/// /// Serves the control session of the clone whose activation is `activation`, and whose -/// /// template captured `template`, the inputs of its root's assembly, until it fails. +/// /// Activates the clone whose activation is `activation`, and whose template captured +/// /// `template`, the inputs of its root's assembly, and serves its control session until it +/// /// fails. /// fn serve( /// agent: &mut impl Agent, /// template: &RootTemplate, /// mut activation: Activation, /// ) -> Result<(), ActivationFailure> { +/// // The record of the restore packet is valid, and nothing happened before: the +/// // activation's side effects start here, and the fence opens once they all succeeded. The +/// // application waits for the devices until the activation's deadline at the latest, so +/// // that it ends by then. +/// let record = activation.accept(Instant::now())?; +/// let applied = clone::apply(&record, template, activation.deadline()); +/// // What the host sent meanwhile came before Ready: a request for Ready joins the +/// // application, a reset abandons those of the earlier session, and any other request fails +/// // the activation, which drops what the application assembled. +/// let mut joined = 0; +/// while let Some(received) = agent.receive(Some(Instant::now())) { +/// match activation.input(Instant::now(), input(&received))? { +/// ActivationAction::Acknowledge => { +/// agent.acknowledge(); +/// joined = 0; +/// } +/// ActivationAction::Join => joined += 1, +/// _ => {} +/// } +/// } +/// let ready = match applied { +/// Ok(applied) => { +/// agent.keep(applied); +/// activation.complete(Instant::now(), Ok(()))? +/// } +/// // A failed application fails the activation, which nothing retries. +/// Err(error) => activation.complete(Instant::now(), Err(error.into()))?, +/// }; +/// // Ready answers each request for it that joined the application, which made the +/// // activation ready. Without one, the host's next request for Ready makes it ready. +/// for _ in 0..joined { +/// agent.ready(ready); +/// } /// loop { /// let Some(received) = agent.receive(activation.deadline()) else { -/// // The deadline passed, unless the clone is ready, which ends the activation. +/// // Until the activation is ready, its deadline fails it. /// activation.tick(Instant::now())?; /// continue; /// }; /// match activation.input(Instant::now(), input(&received))? { /// ActivationAction::Acknowledge => agent.acknowledge(), /// ActivationAction::Continue | ActivationAction::Join => {} -/// ActivationAction::Apply(record) => { -/// // The record is valid, and nothing happened before: the activation's side -/// // effects start here, and the fence opens once they all succeeded. The -/// // application waits for the devices until the activation's deadline at the -/// // latest, so that it ends by then. -/// let applied = clone::apply(&record, template, activation.deadline()); -/// // What the host sent meanwhile came before Ready: a retry of the Activate, -/// // byte for byte, joins the application, and anything else, a reset included, -/// // fails the activation, which drops what the application assembled. -/// let mut joined = 0; -/// while let Some(received) = agent.receive(Some(Instant::now())) { -/// let action = activation.input(Instant::now(), input(&received))?; -/// if action == ActivationAction::Join { -/// joined += 1; -/// } -/// } -/// let ready = match applied { -/// Ok(applied) => { -/// agent.keep(applied); -/// activation.complete(Instant::now(), Ok(()))? -/// } -/// // A failed application fails the activation: the fence stays closed. -/// Err(error) => activation.complete(Instant::now(), Err(error.into()))?, -/// }; -/// // Ready answers the Activate, and each retry that joined its application. -/// for _ in 0..=joined { -/// agent.ready(ready); -/// } -/// } /// ActivationAction::Ready(ready) => agent.ready(ready), /// // The activation serves only another message, once it is ready. /// ActivationAction::Serve => { @@ -161,12 +170,11 @@ pub use self::apply::{ /// } /// /// /// Returns the activation's input of `received`. -/// fn input(received: &Received) -> ActivationInput<'_> { +/// fn input(received: &Received) -> ActivationInput { /// match received { /// Received::Reset => ActivationInput::Reset, /// Received::Partial => ActivationInput::Flow, -/// Received::Activate(record) => ActivationInput::Activate(record), -/// Received::TooLong(length) => ActivationInput::TooLong(*length), +/// Received::Await => ActivationInput::Await, /// Received::Other(_) => ActivationInput::Request, /// } /// } @@ -174,40 +182,54 @@ pub use self::apply::{ pub struct Activation { /// The caller's fence of workload admission, closed until the activation is ready. fence: F, - /// The resources that the clone has, as its repair reports them, which the record of an - /// Activate must expect. + /// The resources that the clone has, as its repair reports them, which the record must + /// expect. resources: ExpectedResources, + /// The bytes of the activation record of the clone's restore packet, if it carried one, until + /// the activation accepts it. + record: Option>, /// When the activation fails unless it is ready, or `None` if no instant is that late. deadline: Option, + /// Whether a request for Ready of the host's current session joined the activation, so that + /// its completion makes it ready. + awaited: bool, /// Where the activation is. state: State, } impl Activation { - /// Returns the activation of the clone that `repaired` describes, in its - /// [transport bootstrap](ActivationState::Bootstrap), behind the caller's `fence`, which - /// [`Restored::into_parts`](super::Restored::into_parts) handed over closed, and which the - /// activation opens once it is ready. + /// Returns the activation of the clone that `repaired` describes, with the activation record + /// of its restore packet, [`Repaired::activation_record`], [pending](ActivationState::Pending), + /// behind the caller's `fence`, which [`Restored::into_parts`](super::Restored::into_parts) + /// handed over closed, and which the activation opens once it is ready. /// /// The activation must be ready by its deadline: `timeout`, which the caller chooses, after /// the repair acknowledged the restore, as [`Repaired::acknowledged_at`] tells, or after `now` /// for a restore that the host did not gate until an acknowledgement, whose repair - /// acknowledged nothing. No input or retry extends it. A deadline that no instant can hold - /// never passes. + /// acknowledged nothing. Nothing extends it. A deadline that no instant can hold never passes. pub fn new(repaired: &Repaired, fence: F, timeout: Duration, now: Instant) -> Self { let resources = ExpectedResources { cpus: repaired.cpus(), memory: repaired.memory(), }; - Self::with_resources(resources, repaired.acknowledged_at(), fence, timeout, now) + Self::with_resources( + resources, + repaired.activation_record(), + repaired.acknowledged_at(), + fence, + timeout, + now, + ) } - /// Returns the activation of a clone that has `resources`, and whose repair acknowledged its - /// restore at `acknowledged_at`, if it did, as [`new`](Self::new) does with the resources and - /// the acknowledgement of a [`Repaired`] clone: for a caller that has them otherwise, such as - /// the tests of an adapter's framing, which run its session without a restore. + /// Returns the activation of a clone that has `resources`, whose restore packet carried the + /// activation record `record`, if it carried one, and whose repair acknowledged its restore at + /// `acknowledged_at`, if it did, as [`new`](Self::new) does with those of a [`Repaired`] + /// clone: for a caller that has them otherwise, such as the tests of an adapter's framing, + /// which run its session without a restore. pub fn with_resources( resources: ExpectedResources, + record: Option<&[u8]>, acknowledged_at: Option, fence: F, timeout: Duration, @@ -217,113 +239,139 @@ impl Activation { Self { fence, resources, + record: record.map(Box::from), deadline: start.checked_add(timeout), - state: State::Bootstrap, + awaited: false, + state: State::Pending, } } + /// Accepts at `now` the activation record that the clone's restore packet carried, once it + /// decoded and validated it, and returns it, for the adapter to apply with `apply`: nothing + /// has a side effect before. The activation is [activating](ActivationState::Activating) + /// next. + /// + /// The packet must carry a record, or the activation fails with [`ActivationCause::Missing`]: + /// a restore of a platform snapshot hands its clone the record that its launcher supplied, + /// which NVX's launch of a clone always supplies. [`ActivationRecord::decode`] refuses a record + /// with [`ActivationCause::Record`] unless it is canonical and keeps every limit; the record + /// must be a clone's, with a restore section, or the activation fails with + /// [`ActivationCause::Cold`]; and it must expect the resources that the clone has, the CPU + /// count and the memory size that its repair reports, or the activation fails with + /// [`ActivationCause::Resources`]. Past the deadline, the activation fails with + /// [`ActivationCause::Expired`] instead, and once it accepted the record, another acceptance + /// fails it with [`ActivationCause::Completion`]. The activation keeps nothing of the record + /// but what Ready reports about it, its transaction ID. + pub fn accept(&mut self, now: Instant) -> Result, ActivationFailure> { + self.check(now)?; + if !matches!(self.state, State::Pending) { + return Err(self.fail(ActivationCause::Completion)); + } + let Some(record) = self.record.take() else { + return Err(self.fail(ActivationCause::Missing)); + }; + let decoded = match ActivationRecord::decode(&record) { + Ok(decoded) => decoded, + Err(error) => return Err(self.fail(ActivationCause::Record(error))), + }; + if decoded.restore().is_none() { + return Err(self.fail(ActivationCause::Cold)); + } + let expected = decoded.configuration().resources; + if expected != self.resources { + let actual = self.resources; + return Err(self.fail(ActivationCause::Resources { expected, actual })); + } + self.state = State::Activating(ReadyActivation::of(&decoded)); + Ok(Box::new(decoded)) + } + /// Takes `input`, which the adapter received at `now` on the clone's control session, and /// returns what the adapter does next, as the [type](Self) tabulates it, or fails the /// activation. /// /// Until the activation is ready, it fails once `now` reaches its deadline, whatever the - /// input, with [`ActivationCause::Expired`]. In order, it then takes: + /// input, with [`ActivationCause::Expired`]. Otherwise, in any state, it takes flow control, + /// and each reset of the session: the one that the restore queued, and any later one, such as + /// once the host's attachment ended, which the adapter acknowledges, abandoning the work of + /// the earlier epoch, the host's requests for Ready that joined the activation in it included. + /// A reset changes nothing else of the activation, whose record came with the restore, before + /// the host could send anything, so that each session gets the same Ready. /// - /// 1. In the transport bootstrap, the session's reset, which the restore queued: the adapter - /// acknowledges it, and the session is restricted next. Before it, the host can send - /// nothing, so an Activate or another request fails with [`ActivationCause::Request`]. - /// 2. While restricted, flow control, and one Activate. The activation decodes its record with - /// [`ActivationRecord::decode`], which refuses it with [`ActivationCause::Record`] unless it - /// is canonical and keeps every limit, checks that it is a clone's, with a restore - /// section, or refuses it with [`ActivationCause::Cold`], and checks that it expects the - /// resources that the clone has, the CPU count and the memory size that its repair - /// reports, or refuses it with [`ActivationCause::Resources`]. Only then does it accept it, - /// and hand the record to the adapter with [`ActivationAction::Apply`]: nothing has a side - /// effect before. Any other request fails with [`ActivationCause::Request`], and another - /// reset with [`ActivationCause::Reset`]. - /// 3. While activating, flow control, and retries of the Activate: one of the same - /// transaction, with a byte-equal record, joins the activation that runs. Any other - /// Activate fails: one whose record does not decode, or is a cold boot's, with the cause - /// with which the first would have failed, one of another transaction with - /// [`ActivationCause::Transaction`], and any other of the same transaction with - /// [`ActivationCause::Changed`]. A request or a reset fails as while restricted. - /// 4. Once ready, or started, everything: a reset starts a later session, which the adapter - /// acknowledges, a byte-equal retry of the Activate gets the cached result, what Ready - /// reports, without activating again, and any other Activate fails as while activating. - /// Every other request goes to the adapter's order, as after a cold boot. - /// - /// An Activate whose record the adapter's framing refused unread, as longer than a record can - /// be, [`ActivationInput::TooLong`], fails as one whose record has that many bytes: in the - /// transport bootstrap with [`ActivationCause::Request`], and from then on with - /// [`ActivationCause::Record`]. + /// The host's request for the clone's Ready, [`ActivationInput::Await`], comes first in each + /// of its sessions: before the activation applied the record, it joins the activation, whose + /// [completion](Self::complete) answers it, unless a reset abandoned it; once the activation + /// applied the record, it makes the activation ready, which opens the fence, and gets what + /// Ready reports; and once the activation is ready, or started, it gets the same, without + /// activating again. Any other request before the activation is ready fails it with + /// [`ActivationCause::Request`], since the host sends its mapping table and the start only + /// once Ready came, and once the activation is ready, or started, every other request goes to + /// the adapter's order, as after a cold boot. pub fn input( &mut self, now: Instant, - input: ActivationInput<'_>, + input: ActivationInput, ) -> Result { self.check(now)?; - match (self.state.name(), input) { - (_, ActivationInput::Flow) => Ok(ActivationAction::Continue), - (ActivationState::Bootstrap, ActivationInput::Reset) => { - self.state = State::Restricted; - Ok(ActivationAction::Acknowledge) - } - (ActivationState::Ready | ActivationState::Started, ActivationInput::Reset) => { - Ok(ActivationAction::Acknowledge) + Ok(match input { + ActivationInput::Flow => ActivationAction::Continue, + ActivationInput::Reset => { + self.awaited = false; + ActivationAction::Acknowledge } - (_, ActivationInput::Reset) => Err(self.fail(ActivationCause::Reset)), - (ActivationState::Restricted, ActivationInput::Activate(record)) => self.accept(record), - ( - ActivationState::Activating | ActivationState::Ready | ActivationState::Started, - ActivationInput::Activate(record), - ) => self.retry(record), - // A record that long never decodes, so its Activate fails as one whose bytes the - // adapter read would, whether the activation accepted another or not. - ( - ActivationState::Restricted - | ActivationState::Activating - | ActivationState::Ready - | ActivationState::Started, - ActivationInput::TooLong(length), - ) => Err(self.fail(ActivationCause::Record(ActivationError::TooLong(length)))), - (ActivationState::Ready | ActivationState::Started, ActivationInput::Request) => { - Ok(ActivationAction::Serve) + ActivationInput::Await => match self.state { + State::Applied(ready) => { + self.state = State::Ready(ready); + self.fence.open(); + ActivationAction::Ready(ready) + } + State::Ready(ready) | State::Started(ready) => ActivationAction::Ready(ready), + _ => { + self.awaited = true; + ActivationAction::Join + } + }, + ActivationInput::Request => { + if self.established().is_none() { + return Err(self.fail(ActivationCause::Request)); + } + ActivationAction::Serve } - ( - _, - ActivationInput::Activate(_) - | ActivationInput::TooLong(_) - | ActivationInput::Request, - ) => Err(self.fail(ActivationCause::Request)), - } + }) } - /// Completes the activation of the accepted Activate's record at `now`, with the `result` of - /// the adapter's application of the record, and returns what Ready reports: that the - /// activation applied, with the record's transaction ID. + /// Completes the activation of the accepted record at `now`, with the `result` of the + /// adapter's application of the record, and returns what Ready reports: that the activation + /// applied, with the record's transaction ID, which answers each of the host's requests for + /// Ready that joined the activation in the current session. /// - /// A result that succeeded by the deadline makes the activation ready: it opens the fence, - /// which no input before opened, and the adapter reports Ready. A failure, which may have - /// applied part of the record, is final, with [`ActivationCause::Failed`], so that no retry - /// activates the record again, and the fence stays closed. Past the deadline, the activation - /// fails with [`ActivationCause::Expired`] instead, whatever the result. Without an activation - /// that runs, it fails with [`ActivationCause::Completion`]. + /// A result that succeeded by the deadline makes the activation ready once a request for + /// Ready joined it in the current session: it opens the fence, which no input before opened. + /// Without one, it leaves the activation [applied](ActivationState::Applied), with the fence + /// closed and the deadline holding, until the host's next request for Ready, which makes it + /// ready. A failure, which may have applied part of the record, is final, with + /// [`ActivationCause::Failed`], so that nothing activates the record again, and the fence + /// stays closed. Past the deadline, the activation fails with [`ActivationCause::Expired`] + /// instead, whatever the result. Without an activation that runs, it fails with + /// [`ActivationCause::Completion`]. pub fn complete( &mut self, now: Instant, result: Result<(), Box>, ) -> Result { self.check(now)?; - let accepted = match &self.state { - State::Activating(accepted) => accepted.clone(), - _ => return Err(self.fail(ActivationCause::Completion)), + let State::Activating(ready) = self.state else { + return Err(self.fail(ActivationCause::Completion)); }; if let Err(error) = result { return Err(self.fail(ActivationCause::Failed(error))); } - let ready = accepted.ready; - self.state = State::Ready(accepted); - self.fence.open(); + if self.awaited { + self.state = State::Ready(ready); + self.fence.open(); + } else { + self.state = State::Applied(ready); + } Ok(ready) } @@ -333,12 +381,12 @@ impl Activation { /// A start before the activation is ready, or a second one, fails with /// [`ActivationCause::Start`]. pub fn start(&mut self) -> Result<(), ActivationFailure> { - let accepted = match &self.state { + let ready = match &self.state { State::Failed(failure) => return Err(failure.clone()), - State::Ready(accepted) => accepted.clone(), + State::Ready(ready) => *ready, _ => return Err(self.fail(ActivationCause::Start)), }; - self.state = State::Started(accepted); + self.state = State::Started(ready); Ok(()) } @@ -350,11 +398,11 @@ impl Activation { } /// Returns when the activation fails unless it is ready, for the adapter to - /// [tick](Self::tick) it then, or `None` once it is ready or failed, or if no instant is that + /// [tick](Self::tick) it then, or `None` once it is ready, or failed, or if no instant is that /// late. pub fn deadline(&self) -> Option { match self.state { - State::Bootstrap | State::Restricted | State::Activating(_) => self.deadline, + State::Pending | State::Activating(_) | State::Applied(_) => self.deadline, State::Ready(_) | State::Started(_) | State::Failed(_) => None, } } @@ -365,58 +413,15 @@ impl Activation { } /// Returns what Ready reports about the established activation, once it is ready: that the - /// activation applied, with the record's transaction ID, which a later session gets without - /// activating again. + /// activation applied, with the record's transaction ID, which every later session gets + /// without activating again. pub fn established(&self) -> Option { match &self.state { - State::Ready(accepted) | State::Started(accepted) => Some(accepted.ready), + State::Ready(ready) | State::Started(ready) => Some(*ready), _ => None, } } - /// Accepts the host's first Activate, whose record's bytes are `record`, once it decoded and - /// validated it: a clone's record, with a restore section, that expects the clone's resources. - fn accept(&mut self, record: &[u8]) -> Result { - let decoded = match ActivationRecord::decode(record) { - Ok(decoded) => decoded, - Err(error) => return Err(self.fail(ActivationCause::Record(error))), - }; - let Some(restore) = decoded.restore() else { - return Err(self.fail(ActivationCause::Cold)); - }; - let expected = decoded.configuration().resources; - if expected != self.resources { - let actual = self.resources; - return Err(self.fail(ActivationCause::Resources { expected, actual })); - } - self.state = State::Activating(Accepted { - record: record.into(), - transaction: restore.transaction, - ready: ReadyActivation::of(&decoded), - }); - Ok(ActivationAction::Apply(Box::new(decoded))) - } - - /// Takes a retry of the accepted Activate, whose record's bytes are `record`: a byte-equal - /// record joins the activation that runs, or gets its cached result once it is ready, and - /// any other fails the activation. - fn retry(&mut self, record: &[u8]) -> Result { - let (accepted, ready) = match &self.state { - State::Activating(accepted) => (accepted, false), - State::Ready(accepted) | State::Started(accepted) => (accepted, true), - _ => return Err(self.fail(ActivationCause::Request)), - }; - if *accepted.record == *record { - return Ok(if ready { - ActivationAction::Ready(accepted.ready) - } else { - ActivationAction::Join - }); - } - let cause = accepted.refusal(record); - Err(self.fail(cause)) - } - /// Fails the activation with the failure that it already had, or with /// [`ActivationCause::Expired`] once `now` reaches its deadline, unless it is ready. fn check(&mut self, now: Instant) -> Result<(), ActivationFailure> { @@ -456,10 +461,10 @@ impl fmt::Debug for Activation { } /// What the adapter received on a clone's control session, in the terms of its [`Activation`], -/// whatever its framing: the session's reset, flow control, the host's Activate, or another -/// request of the host. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ActivationInput<'a> { +/// whatever its framing: the session's reset, flow control, the host's request for the clone's +/// Ready, or another request of the host. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ActivationInput { /// The NVXS session's reset, a [`GuestEvent::Reset`](crate::protocol::nvxs::GuestEvent::Reset), /// which starts a new epoch: the one that the restore queued for the session that the guest /// attached before the capture, or a later one, such as once the host's attachment ended. @@ -467,41 +472,33 @@ pub enum ActivationInput<'a> { /// Flow control, which carries no request: bytes of a message that they do not complete, such /// as a DATA record's payload, or a credit that the adapter returned to its receive window. Flow, - /// The host's Activate, with the bytes of its activation record, the canonical encoding of an - /// [`ActivationRecord`]. - Activate(&'a [u8]), - /// The host's Activate, whose record the adapter's framing refused before it took any byte of - /// it, since its header announced this many bytes, more than - /// [`MAX_RECORD_BYTES`](crate::activation::MAX_RECORD_BYTES): the activation takes it as an - /// Activate whose record's bytes are that long, which [`ActivationRecord::decode`] refuses - /// with [`ActivationError::TooLong`]. - TooLong(usize), + /// The host's request for the clone's Ready, the first request of each of the host's + /// sessions: OpenVMM's broker refuses the guest's records while no host is attached, which the + /// guest cannot tell, so the host asks for Ready rather than the guest sending it unasked, as + /// the host asks a guest agent for its readiness with NVXC's `PING`. + Await, /// Any other request of the host, such as a part of the mapping table, the start of the /// sandbox, an execution, or a probe of the adapter's framing. Request, } /// What the adapter does next, which [`Activation::input`] returns. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ActivationAction { /// The activation took the session's reset: the adapter acknowledges it with its receive /// window, unless its transport acknowledged it already, and abandons the work of the earlier - /// epoch. + /// epoch, the host's requests for Ready that joined the activation in it included, since the + /// host reads no answer of an earlier session. Acknowledge, /// The activation took the input, which needs nothing more of the adapter. Continue, - /// The activation accepted the host's Activate, whose record decoded and validated, and - /// which nothing before had any side effect of: the adapter applies the record with - /// `apply`, which activates the root and the workload with its settings, passes - /// the activation what the session received meanwhile, which came before Ready, then reports - /// the result to [`Activation::complete`]. - Apply(Box), - /// A retry of the accepted Activate joined the activation that runs: what - /// [`Activation::complete`] returns answers it too, as the adapter's framing answers each - /// request. + /// The host's request for Ready came before the activation was ready, and joined it: what + /// [`Activation::complete`] returns answers it, as the adapter's framing answers each request, + /// unless a reset of the session abandons it first. Join, - /// A retry of the accepted Activate, once the activation is ready: the adapter answers it with - /// Ready, which reports this cached result, without activating again. + /// The host's request for Ready, once the activation applied the record: the adapter answers + /// it with Ready, which reports this result, without activating again. The first, once the + /// activation is applied, makes it ready. Ready(ReadyActivation), /// A request once the activation is ready: the adapter serves it in its order, as after a /// cold boot. @@ -513,16 +510,18 @@ pub enum ActivationAction { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum ActivationState { - /// The transport bootstrap: the clone awaits the reset of its NVXS session that the restore - /// queued, which the adapter acknowledges. - Bootstrap, - /// The restricted session: it takes only flow control and one Activate. - Restricted, - /// The adapter applies the accepted Activate's record. + /// The activation record of the clone's restore packet awaits its acceptance, which decodes + /// and validates it. + Pending, + /// The adapter applies the accepted record. Activating, - /// Ready, with the mappings pending: the activation completed, which opened the fence, and - /// returned what Ready reports. The host's mapping table, which the record announced, and the - /// start of the sandbox follow in the adapter's order, as after a cold boot. + /// The adapter applied the accepted record before the host asked for Ready in its current + /// session: the activation awaits the host's request for Ready, which makes it ready. + Applied, + /// Ready, with the mappings pending: the activation completed, and the host asked for Ready, + /// which opened the fence, and got what Ready reports. The host's mapping table, which the + /// record announced, and the start of the sandbox follow in the adapter's order, as after a + /// cold boot. Ready, /// The adapter's order started the sandbox. Started, @@ -531,12 +530,12 @@ pub enum ActivationState { } impl fmt::Display for ActivationState { - /// Writes the state's name, such as `restricted`. + /// Writes the state's name, such as `activating`. fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { - Self::Bootstrap => "transport bootstrap", - Self::Restricted => "restricted", + Self::Pending => "pending", Self::Activating => "activating", + Self::Applied => "applied", Self::Ready => "ready", Self::Started => "started", Self::Failed => "failed", @@ -550,17 +549,17 @@ impl fmt::Display for ActivationState { pub enum ActivationCause { /// The deadline passed before the activation was ready. Expired, - /// The session was reset after its bootstrap, before the activation was ready. - Reset, - /// The host sent a request that the state does not take, before the activation was ready: - /// any before the session's bootstrap, and any but one Activate, and its retries, after it. + /// The clone's restore packet carried no activation record. + Missing, + /// The host sent a request before the activation was ready, other than its request for + /// Ready. Request, - /// The Activate's record does not decode, or breaks a limit of `nvx::activation`, which - /// refused it with this error. + /// The record does not decode, or breaks a limit of `nvx::activation`, which refused it with + /// this error. Record(ActivationError), - /// The Activate's record is a cold boot's, without the restore section of a clone's. + /// The record is a cold boot's, without the restore section of a clone's. Cold, - /// The Activate's record expects other resources than the clone has. + /// The record expects other resources than the clone has. Resources { /// The CPU count and the memory size that the record expects. expected: ExpectedResources, @@ -568,21 +567,12 @@ pub enum ActivationCause { /// number of its online CPUs, and the size of its online memory blocks. actual: ExpectedResources, }, - /// The Activate's record names another transaction than the accepted Activate's. - Transaction { - /// The transaction of the accepted Activate. - accepted: TransactionId, - /// The transaction that the retry names. - offered: TransactionId, - }, - /// The Activate's record names the accepted Activate's transaction, but is not byte-equal to - /// its record. - Changed, /// The adapter's application of the accepted record failed, wholly or partway, with this /// error. Failed(Box), - /// The adapter completed an activation that did not run: before the activation accepted an - /// Activate, or once it completed one. + /// The adapter accepted the record of an activation that had accepted it already, or + /// completed an activation that did not run: before it accepted the record, or once it + /// completed. Completion, /// The adapter's order started the sandbox before the activation was ready, or again. Start, @@ -594,31 +584,33 @@ impl fmt::Display for ActivationCause { Self::Expired => { formatter.write_str("the deadline passed before the activation was ready") } - Self::Reset => { - formatter.write_str("the session was reset again before the activation was ready") + Self::Missing => { + formatter.write_str("the clone's restore packet carried no activation record") } Self::Request => { formatter.write_str("the host sent a request before the activation was ready") } - Self::Record(error) => write!(formatter, "the Activate's record is refused: {error}"), - Self::Cold => formatter - .write_str("the Activate's record is a cold boot's, without a restore section"), + Self::Record(error) => { + write!( + formatter, + "the restore packet's activation record is refused: {error}" + ) + } + Self::Cold => formatter.write_str( + "the restore packet's activation record is a cold boot's, without a restore \ + section", + ), Self::Resources { expected, actual } => write!( formatter, - "the Activate's record expects {} CPUs and {} bytes of memory, and the clone has \ + "the activation record expects {} CPUs and {} bytes of memory, and the clone has \ {} CPUs and {} bytes", expected.cpus, expected.memory, actual.cpus, actual.memory ), - Self::Transaction { accepted, offered } => write!( - formatter, - "the Activate names the transaction {offered}, after an Activate of the \ - transaction {accepted}" - ), - Self::Changed => formatter.write_str( - "the Activate changes the record of the accepted Activate's transaction", - ), Self::Failed(error) => write!(formatter, "the record's application failed: {error}"), - Self::Completion => formatter.write_str("an activation that did not run completed"), + Self::Completion => formatter.write_str( + "the activation's record was accepted again, or an activation that did not run \ + completed", + ), Self::Start => { formatter.write_str("the sandbox started before the activation was ready, or again") } @@ -628,7 +620,7 @@ impl fmt::Display for ActivationCause { /// Why an [`Activation`] failed, finally: the state in which it failed, and its cause. /// -/// Its `Display` form names both, such as `the activation failed in its restricted state: the +/// Its `Display` form names both, such as `the activation failed in its activating state: the /// host sent a request before the activation was ready`. An activation keeps its failure, which /// every later call returns again. The failure has no exit status: the adapter that ends the VM /// documents its own. @@ -673,19 +665,20 @@ impl Error for ActivationFailure { } } -/// Where an [`Activation`] is, with the Activate that it accepted, from the activating state on. +/// Where an [`Activation`] is, with what Ready reports about its record, from the activating state +/// on. #[derive(Clone, Debug)] enum State { - /// The transport bootstrap. - Bootstrap, - /// The restricted session. - Restricted, - /// The adapter applies the accepted Activate's record. - Activating(Accepted), + /// The record awaits its acceptance. + Pending, + /// The adapter applies the accepted record. + Activating(ReadyActivation), + /// The adapter applied the accepted record, which awaits the host's request for Ready. + Applied(ReadyActivation), /// Ready, with the mappings pending. - Ready(Accepted), + Ready(ReadyActivation), /// The adapter's order started the sandbox. - Started(Accepted), + Started(ReadyActivation), /// The activation failed, finally, with this failure. Failed(ActivationFailure), } @@ -694,46 +687,12 @@ impl State { /// Returns the state's name. fn name(&self) -> ActivationState { match self { - Self::Bootstrap => ActivationState::Bootstrap, - Self::Restricted => ActivationState::Restricted, + Self::Pending => ActivationState::Pending, Self::Activating(_) => ActivationState::Activating, + Self::Applied(_) => ActivationState::Applied, Self::Ready(_) => ActivationState::Ready, Self::Started(_) => ActivationState::Started, Self::Failed(_) => ActivationState::Failed, } } } - -/// The Activate that an [`Activation`] accepted, against which it checks each retry. -#[derive(Clone, Debug)] -struct Accepted { - /// The bytes of its record, to which a retry's must be equal. - record: Box<[u8]>, - /// The transaction that its record names. - transaction: TransactionId, - /// What Ready reports about its activation. - ready: ReadyActivation, -} - -impl Accepted { - /// Returns why a retry whose record's bytes are `record`, which are not the accepted record's, - /// is refused: its record does not decode, or is a cold boot's, as the first Activate's would - /// have been refused, or it names another transaction, or it changes the record of the same - /// one. - fn refusal(&self, record: &[u8]) -> ActivationCause { - let decoded = match ActivationRecord::decode(record) { - Ok(decoded) => decoded, - Err(error) => return ActivationCause::Record(error), - }; - match decoded.restore() { - None => ActivationCause::Cold, - Some(restore) if restore.transaction != self.transaction => { - ActivationCause::Transaction { - accepted: self.transaction, - offered: restore.transaction, - } - } - Some(_) => ActivationCause::Changed, - } - } -} diff --git a/nvx-rs/src/guest/clone/activation/apply.rs b/nvx-rs/src/guest/clone/activation/apply.rs index 96247341b..c78f5dcea 100644 --- a/nvx-rs/src/guest/clone/activation/apply.rs +++ b/nvx-rs/src/guest/clone/activation/apply.rs @@ -1,4 +1,4 @@ -//! The application of the record of the Activate that a clone's [`Activation`] accepted: +//! The application of the activation record that a clone's [`Activation`] accepted: //! [`apply`](fn@apply), which activates the clone's root and workload with the record's settings, //! as at a cold boot, as the [module](super::super#activation) describes. //! @@ -291,7 +291,7 @@ impl fmt::Display for AppliedTeardownError { impl std::error::Error for AppliedTeardownError {} -/// Applies `record`, the record of the Activate that a clone's [`Activation`] accepted, to the +/// Applies `record`, the activation record that a clone's [`Activation`] accepted, to the /// clone, as at a cold boot, by `deadline`, the activation's [deadline](Activation::deadline), if /// any, and returns the root and the workload that it activated, which the adapter keeps once it /// reported the result to [`Activation::complete`]. @@ -364,10 +364,10 @@ impl std::error::Error for AppliedTeardownError {} /// [`ApplyError::View`], once the binds that it made were removed, the application stopped /// the announcer that it started, and tore the root down. /// -/// The activation answers a retry of the Activate without applying the record again, so every -/// execution, and every retry, sees the same machine ID and resolver file. A failure after the -/// generation, or after the resolver file's write, leaves that file, since the activation fails -/// finally, and the adapter ends the VM. +/// The activation applies its record once, and answers every request for Ready without applying +/// it again, so every execution, and every session, sees the same machine ID and resolver file. A +/// failure after the generation, or after the resolver file's write, leaves that file, since the +/// activation fails finally, and the adapter ends the VM. /// /// It returns the root of the host export, the [`SourceRoot`] from which the adapter maps the /// host's mapping table once the clone is ready, as at a cold boot, the endpoint of the network's diff --git a/nvx-rs/src/guest/clone/activation/apply/tests.rs b/nvx-rs/src/guest/clone/activation/apply/tests.rs index b3b02609d..d431884a3 100644 --- a/nvx-rs/src/guest/clone/activation/apply/tests.rs +++ b/nvx-rs/src/guest/clone/activation/apply/tests.rs @@ -4,9 +4,9 @@ //! leaves alone, and which generates no machine ID, writes no resolver file, starts //! no announcer, and mounts, binds, and unmounts nothing; and, with the clone's activation, //! through an adapter of -//! the tests' own, which takes what the host sends, applies the record of the Activate that the -//! activation accepted through the fake system, and spawns an execution for each request that -//! the activation serves. +//! the tests' own, which applies the record of the restore packet that the activation accepted +//! through the fake system, takes what the host sends, and spawns an execution for each request +//! that the activation serves. //! //! One test runs the live system's generation of a machine ID, its write of a resolver file, and //! their binds with the kernel itself, over a tmpfs that stands for the root, which needs the @@ -52,9 +52,9 @@ const CPUS: u32 = 2; const MEMORY: u64 = 8 << 27; /// How long the tests' activation may take after the restore's acknowledgement. const TIMEOUT: Duration = Duration::from_secs(10); -/// The transaction of the tests' Activate. +/// The transaction of the tests' restore. const TRANSACTION: TransactionId = TransactionId::from_bytes([0x22; TransactionId::LEN]); -/// The UUID that the record of the tests' Activate expects of its EROFS image. +/// The UUID that the record of the tests' restore packet expects of its EROFS image. const IMAGE_UUID: Uuid = Uuid::from_bytes([0x5a; 16]); /// The GUID that a record of the tests expects of its GPT disk. const DISK_GUID: Uuid = Uuid::from_bytes([0xd1; 16]); @@ -416,10 +416,11 @@ impl AdmissionFence for FakeFence { } } -/// A clone's adapter in the tests, which takes what the host sends, as its session receives it, -/// through the clone's activation, behind a fake fence: it applies the record of the Activate that -/// the activation accepted through a fake system, with the tests' template, reports the result, -/// and spawns an execution for each request that the activation serves, once the fence is open. +/// A clone's adapter in the tests, which activates the clone with the record of its restore packet +/// and takes what the host sends, as its session receives it, through the clone's activation, +/// behind a fake fence: it applies the record that the activation accepted through a fake system, +/// with the tests' template, reports the result, and spawns an execution for each request that +/// the activation serves, once the fence is open. struct Adapter { /// The clone's activation. activation: Activation, @@ -436,9 +437,10 @@ struct Adapter { } impl Adapter { - /// Returns the adapter of a clone, in its transport bootstrap, which applies records through - /// `system`, and whose repair acknowledged its restore at `acknowledged`. - fn new(system: FakeSystem, acknowledged: Instant) -> Self { + /// Returns the adapter of a clone, pending, whose restore packet carried `record`, if any, + /// which it applies through `system`, and whose repair acknowledged its restore at + /// `acknowledged`. + fn new(system: FakeSystem, record: Option<&[u8]>, acknowledged: Instant) -> Self { let fence = FakeFence::default(); let resources = ExpectedResources { cpus: CPUS, @@ -446,6 +448,7 @@ impl Adapter { }; let activation = Activation::with_resources( resources, + record, Some(acknowledged), fence.clone(), TIMEOUT, @@ -460,33 +463,31 @@ impl Adapter { } } - /// Returns the adapter of [`new`](Self::new) once the session's reset at `at` restricted its - /// activation. - fn restricted(system: FakeSystem, at: Instant) -> Self { - let mut adapter = Self::new(system, at); - adapter.take(at, ActivationInput::Reset).unwrap(); - assert_eq!(adapter.activation.state(), ActivationState::Restricted); - adapter + /// Activates the clone at `now`, as an adapter does: accepts the record of its restore + /// packet, takes the host's request for Ready, which joins the activation, applies the record, + /// and reports the result, whose Ready it keeps, or returns the activation's failure. + fn activate(&mut self, now: Instant) -> Result<(), ActivationFailure> { + let record = self.activation.accept(now)?; + self.take(now, ActivationInput::Await)?; + let deadline = self.activation.deadline(); + let applied = apply_with(&mut self.system, &record, &template(), deadline); + let result = match applied { + Ok(applied) => { + self.applied = Some(applied); + Ok(()) + } + Err(error) => Err(error.into()), + }; + self.ready = Some(self.activation.complete(now, result)?); + Ok(()) } /// Takes `input`, which the session received at `now`, as an adapter does, or returns the /// activation's failure. - fn take(&mut self, now: Instant, input: ActivationInput<'_>) -> Result<(), ActivationFailure> { + fn take(&mut self, now: Instant, input: ActivationInput) -> Result<(), ActivationFailure> { match self.activation.input(now, input)? { ActivationAction::Acknowledge | ActivationAction::Continue | ActivationAction::Join => { } - ActivationAction::Apply(record) => { - let deadline = self.activation.deadline(); - let applied = apply_with(&mut self.system, &record, &template(), deadline); - let result = match applied { - Ok(applied) => { - self.applied = Some(applied); - Ok(()) - } - Err(error) => Err(error.into()), - }; - self.ready = Some(self.activation.complete(now, result)?); - } ActivationAction::Ready(ready) => self.ready = Some(ready), ActivationAction::Serve => { assert!( @@ -610,7 +611,7 @@ fn clone_record(configuration: Configuration) -> ActivationRecord { ActivationRecord::new(Some(restore), configuration).unwrap() } -/// Returns the record of the tests' Activate: an EROFS image, whose UUID it expects, over a +/// Returns the record of the tests' restore packet: an EROFS image, whose UUID it expects, over a /// tmpfs, with untrusted lowers, which names the tests' workload identity. fn accepted() -> ActivationRecord { clone_record(configuration( @@ -621,9 +622,9 @@ fn accepted() -> ActivationRecord { )) } -/// Returns the assembly of the root of the tests' Activate: its EROFS image, with the UUID that -/// the record expects of it, over a tmpfs, with untrusted lowers, with the template's runtime -/// directory, devices, and timeout. +/// Returns the assembly of the root of the tests' restore packet's record: its EROFS image, with +/// the UUID that the record expects of it, over a tmpfs, with untrusted lowers, with the template's +/// runtime directory, devices, and timeout. fn accepted_assembly() -> Assembly { let template = template(); Assembly { @@ -638,8 +639,8 @@ fn accepted_assembly() -> Assembly { } } -/// Returns the record of the tests' Activate, as [`accepted`] returns it, but whose workload sees -/// a machine ID that the guest generates. +/// Returns the record of the tests' restore packet, as [`accepted`] returns it, but whose workload +/// sees a machine ID that the guest generates. fn with_generated_id() -> ActivationRecord { let mut configuration = accepted().configuration().clone(); configuration.workload.machine_id = MachineIdMode::Generated; @@ -673,8 +674,8 @@ fn unpublished(dns: Option, proxy: Option<&str>) -> Network { } } -/// Returns the record of the tests' Activate, as [`with_generated_id`] returns it, but with an -/// aggregate export, of three mappings, and a network that names [`DNS`] and [`PROXY`], and +/// Returns the record of the tests' restore packet, as [`with_generated_id`] returns it, but with +/// an aggregate export, of three mappings, and a network that names [`DNS`] and [`PROXY`], and /// publishes the workload's listeners. fn with_everything() -> ActivationRecord { let mut configuration = with_generated_id().configuration().clone(); @@ -783,50 +784,38 @@ fn no_mount_spawn_or_id_generation_happens_before_the_gate() { }) .encode(); - // Before the session's reset, the host can send nothing. - let mut adapter = Adapter::new(FakeSystem::new(), base); + // Before the clone is ready, the host can send nothing but its request for Ready, and resets. + let mut adapter = Adapter::new(FakeSystem::new(), Some(&record), base); adapter.take(base, ActivationInput::Flow).unwrap(); - let failure = adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap_err(); - adapter.assert_untouched(&failure, |cause| matches!(cause, ActivationCause::Request)); - let mut adapter = Adapter::new(FakeSystem::new(), base); + adapter.take(base, ActivationInput::Reset).unwrap(); + adapter.take(base, ActivationInput::Await).unwrap(); let failure = adapter.take(base, ActivationInput::Request).unwrap_err(); adapter.assert_untouched(&failure, |cause| matches!(cause, ActivationCause::Request)); - // Once the session is restricted, a request or another reset fails, and so does an Activate - // whose record does not decode, is a cold boot's, or expects other resources. - let refused: [(ActivationInput<'_>, Expected); 5] = [ - (ActivationInput::Request, |cause| { - matches!(cause, ActivationCause::Request) - }), - (ActivationInput::Reset, |cause| { - matches!(cause, ActivationCause::Reset) + // A restore packet without a record fails the activation, and so does a record that does not + // decode, is a cold boot's, or expects other resources. + let truncated = &record[..record.len() - 1]; + let refused: [(Option<&[u8]>, Expected); 4] = [ + (None, |cause| matches!(cause, ActivationCause::Missing)), + (Some(truncated), |cause| { + matches!(cause, ActivationCause::Record(_)) }), - ( - ActivationInput::Activate(&record[..record.len() - 1]), - |cause| matches!(cause, ActivationCause::Record(_)), - ), - (ActivationInput::Activate(&cold), |cause| { - matches!(cause, ActivationCause::Cold) - }), - (ActivationInput::Activate(&larger), |cause| { + (Some(&cold), |cause| matches!(cause, ActivationCause::Cold)), + (Some(&larger), |cause| { matches!(cause, ActivationCause::Resources { .. }) }), ]; - for (input, expected) in refused { - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - adapter.take(base, ActivationInput::Flow).unwrap(); - let failure = adapter.take(base, input).unwrap_err(); + for (packet, expected) in refused { + let mut adapter = Adapter::new(FakeSystem::new(), packet, base); + adapter.take(base, ActivationInput::Reset).unwrap(); + let failure = adapter.activate(base).unwrap_err(); adapter.assert_untouched(&failure, expected); } // A GPT disk, which the activation accepted, whose partition table the kernel refuses to // reread, fails the application before anything is mounted: the reread is its one call. - let mut adapter = Adapter::restricted(FakeSystem::refusing(Errno::EBUSY), base); - let failure = adapter - .take(base, ActivationInput::Activate(&disk)) - .unwrap_err(); + let mut adapter = Adapter::new(FakeSystem::refusing(Errno::EBUSY), Some(&disk), base); + let failure = adapter.activate(base).unwrap_err(); assert!( matches!( apply_error(&failure), @@ -844,22 +833,18 @@ fn no_mount_spawn_or_id_generation_happens_before_the_gate() { assert_eq!(adapter.ready, None); assert!(!adapter.fence.is_open()); - // Past the deadline, even a valid Activate does nothing. - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - let failure = adapter - .take(base + TIMEOUT, ActivationInput::Activate(&record)) - .unwrap_err(); + // Past the deadline, even a valid record does nothing. + let mut adapter = Adapter::new(FakeSystem::new(), Some(&record), base); + let failure = adapter.activate(base + TIMEOUT).unwrap_err(); adapter.assert_untouched(&failure, |cause| matches!(cause, ActivationCause::Expired)); - // Only a valid Activate mounts the root, generates the machine ID, mounts the export, writes + // Only a valid record mounts the root, generates the machine ID, mounts the export, writes // the resolver file, starts the announcer, and binds the runtime's files, before Ready, and // an execution spawns only once Ready opened the fence. - let mut adapter = Adapter::restricted(FakeSystem::new(), base); + let mut adapter = Adapter::new(FakeSystem::new(), Some(&record), base); adapter.take(base, ActivationInput::Flow).unwrap(); assert!(adapter.system.effects.is_empty()); - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + adapter.activate(base).unwrap(); assert_eq!(adapter.system.effects, applied_everything()); assert!(adapter.fence.is_open()); adapter.take(base, ActivationInput::Request).unwrap(); @@ -867,13 +852,11 @@ fn no_mount_spawn_or_id_generation_happens_before_the_gate() { } #[test] -fn a_valid_activate_activates_the_root_and_the_workload_once_before_ready() { +fn a_valid_record_activates_the_root_and_the_workload_once_before_ready() { let base = Instant::now(); let record = accepted().encode(); - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + let mut adapter = Adapter::new(FakeSystem::new(), Some(&record), base); + adapter.activate(base).unwrap(); // The application assembled the root with the record's settings, before Ready, which reports // that the activation applied, with the record's transaction. assert_eq!(adapter.activation.state(), ActivationState::Ready); @@ -889,14 +872,12 @@ fn a_valid_activate_activates_the_root_and_the_workload_once_before_ready() { ); assert!(adapter.fence.is_open()); - // A later session, and a retry of the Activate in it, get the established activation + // A later session, and the host's request for Ready in it, get the established activation // without applying the record again; requests spawn executions. adapter.take(base, ActivationInput::Request).unwrap(); adapter.take(base, ActivationInput::Reset).unwrap(); adapter.ready = None; - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + adapter.take(base, ActivationInput::Await).unwrap(); assert_eq!(adapter.ready, Some(ReadyActivation::of(&accepted()))); adapter.take(base, ActivationInput::Request).unwrap(); assert_eq!( @@ -1034,10 +1015,9 @@ fn a_gpt_disk_s_partition_table_is_reread_before_its_root_is_assembled() { activation::OverlayMode::UntrustedLowers, Some(WORKLOAD), )); - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - adapter - .take(base, ActivationInput::Activate(&record.encode())) - .unwrap(); + let encoded = record.encode(); + let mut adapter = Adapter::new(FakeSystem::new(), Some(&encoded), base); + adapter.activate(base).unwrap(); let template = template(); let assembly = Assembly { runtime: template.runtime.clone(), @@ -1088,10 +1068,8 @@ fn the_application_checks_no_account_of_the_record_s_workload_identity() { without_passwd, ]; for system in systems { - let mut adapter = Adapter::restricted(system, base); - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + let mut adapter = Adapter::new(system, Some(&record), base); + adapter.activate(base).unwrap(); assert_eq!(adapter.activation.state(), ActivationState::Ready); assert_eq!(adapter.ready, Some(ReadyActivation::of(&accepted()))); assert_eq!( @@ -1113,10 +1091,8 @@ fn the_application_checks_no_account_of_the_record_s_workload_identity() { fn a_failed_assembly_is_final_and_reads_and_tears_down_nothing() { let base = Instant::now(); let record = accepted().encode(); - let mut adapter = Adapter::restricted(FakeSystem::failing(), base); - let failure = adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap_err(); + let mut adapter = Adapter::new(FakeSystem::failing(), Some(&record), base); + let failure = adapter.activate(base).unwrap_err(); let ApplyError::Assembly(error) = apply_error(&failure) else { panic!("{failure}"); }; @@ -1132,11 +1108,9 @@ fn a_failed_assembly_is_final_and_reads_and_tears_down_nothing() { assert_eq!(adapter.applied, None); assert_eq!(adapter.ready, None); assert!(!adapter.fence.is_open()); - assert!( - adapter - .take(base, ActivationInput::Activate(&record)) - .is_err() - ); + // Neither another acceptance nor a request for Ready applies the record again. + assert!(adapter.activate(base).is_err()); + assert!(adapter.take(base, ActivationInput::Await).is_err()); assert_eq!(adapter.system.effects.len(), 1); } @@ -1148,10 +1122,8 @@ fn a_generated_machine_id_is_generated_once_after_the_gate_for_every_execution() let base = Instant::now(); let record = with_generated_id(); let bytes = record.encode(); - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - adapter - .take(base, ActivationInput::Activate(&bytes)) - .unwrap(); + let mut adapter = Adapter::new(FakeSystem::new(), Some(&bytes), base); + adapter.activate(base).unwrap(); let expected = [ Effect::Assemble(accepted_assembly()), Effect::Generate(machine_id_path()), @@ -1165,17 +1137,13 @@ fn a_generated_machine_id_is_generated_once_after_the_gate_for_every_execution() assert_eq!(adapter.ready, Some(ReadyActivation::of(&record))); assert_eq!(machine_id_path(), Path::new("/run/nvx/workload-machine-id")); - // Every execution, a retry of the Activate, and a retry in a later session reuse it: nothing - // generates or binds it again. + // Every execution, the host's request for Ready, and its request in a later session reuse it: + // nothing generates or binds it again. adapter.take(base, ActivationInput::Request).unwrap(); - adapter - .take(base, ActivationInput::Activate(&bytes)) - .unwrap(); + adapter.take(base, ActivationInput::Await).unwrap(); adapter.take(base, ActivationInput::Reset).unwrap(); adapter.ready = None; - adapter - .take(base, ActivationInput::Activate(&bytes)) - .unwrap(); + adapter.take(base, ActivationInput::Await).unwrap(); assert_eq!(adapter.ready, Some(ReadyActivation::of(&record))); adapter.take(base, ActivationInput::Request).unwrap(); assert_eq!(adapter.system.generated(), 1); @@ -1248,10 +1216,8 @@ fn a_machine_id_or_a_view_that_fails_fails_the_activation_once_the_root_is_torn_ ), ]; for (system, effects) in cases { - let mut adapter = Adapter::restricted(system, base); - let failure = adapter - .take(base, ActivationInput::Activate(&bytes)) - .unwrap_err(); + let mut adapter = Adapter::new(system, Some(&bytes), base); + let failure = adapter.activate(base).unwrap_err(); assert_eq!(failure.state(), ActivationState::Activating); match apply_error(&failure) { ApplyError::MachineId { @@ -1273,12 +1239,9 @@ fn a_machine_id_or_a_view_that_fails_fails_the_activation_once_the_root_is_torn_ assert_eq!(adapter.applied, None); assert_eq!(adapter.ready, None); assert!(!adapter.fence.is_open()); - // The failure is final: no retry applies the record again. - assert!( - adapter - .take(base, ActivationInput::Activate(&bytes)) - .is_err() - ); + // The failure is final: nothing applies the record again. + assert!(adapter.activate(base).is_err()); + assert!(adapter.take(base, ActivationInput::Await).is_err()); assert_eq!(adapter.system.effects, expected); } } @@ -1345,10 +1308,8 @@ fn an_export_is_mounted_through_the_template_s_device_and_handed_to_the_adapter( configuration.export = Some(export(kind)); configuration.mappings = 2; let record = clone_record(configuration).encode(); - let mut adapter = Adapter::restricted(FakeSystem::new(), base); - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + let mut adapter = Adapter::new(FakeSystem::new(), Some(&record), base); + adapter.activate(base).unwrap(); let expected = [ Effect::Assemble(accepted_assembly()), Effect::MountExport(TAG.to_owned(), Access::ReadWrite), @@ -1362,10 +1323,8 @@ fn an_export_is_mounted_through_the_template_s_device_and_handed_to_the_adapter( ..activated(&adapter.system, MachineId::Image) }) ); - // A retry and an execution mount nothing again. - adapter - .take(base, ActivationInput::Activate(&record)) - .unwrap(); + // A request for Ready and an execution mount nothing again. + adapter.take(base, ActivationInput::Await).unwrap(); adapter.take(base, ActivationInput::Request).unwrap(); assert_eq!(adapter.system.effects[expected.len()..], [Effect::Spawn]); } @@ -1583,10 +1542,8 @@ fn an_export_or_a_network_that_fails_fails_the_activation_once_the_root_is_torn_ ), ]; for (system, effects, expected) in cases { - let mut adapter = Adapter::restricted(system, base); - let failure = adapter - .take(base, ActivationInput::Activate(&bytes)) - .unwrap_err(); + let mut adapter = Adapter::new(system, Some(&bytes), base); + let failure = adapter.activate(base).unwrap_err(); let error = apply_error(&failure); assert!(expected(error), "{error}"); assert_eq!(adapter.system.effects, effects, "{error}"); @@ -1976,13 +1933,9 @@ fn application_errors_name_their_cause() { // The adapter hands the error to the activation, whose failure names it. let base = Instant::now(); let generating = FakeSystem::generating(io::ErrorKind::AlreadyExists); - let mut adapter = Adapter::restricted(generating, base); - let failure = adapter - .take( - base, - ActivationInput::Activate(&with_generated_id().encode()), - ) - .unwrap_err(); + let record = with_generated_id().encode(); + let mut adapter = Adapter::new(generating, Some(&record), base); + let failure = adapter.activate(base).unwrap_err(); assert_eq!( failure.to_string(), format!( diff --git a/nvx-rs/src/guest/clone/activation/tests.rs b/nvx-rs/src/guest/clone/activation/tests.rs index 7554724aa..0cdb936ae 100644 --- a/nvx-rs/src/guest/clone/activation/tests.rs +++ b/nvx-rs/src/guest/clone/activation/tests.rs @@ -11,7 +11,7 @@ use std::rc::Rc; use super::*; use crate::activation::{ Configuration, Identity, Image, MAX_RECORD_BYTES, MachineIdMode, OverlayMode, RestoreSection, - StoreGeneration, TemplateReference, Upper, WorkloadView, + StoreGeneration, TemplateReference, TransactionId, Upper, WorkloadView, }; /// The number of online CPUs of the clone. @@ -20,12 +20,10 @@ const CPUS: u32 = 2; const MEMORY: u64 = 8 << 27; /// How long the activation may take after the restore's acknowledgement. const TIMEOUT: Duration = Duration::from_secs(10); -/// The transaction of the host's Activate. +/// The transaction of the clone's restore. const TRANSACTION: TransactionId = TransactionId::from_bytes([0x22; TransactionId::LEN]); -/// The transaction of another restore. -const OTHER_TRANSACTION: TransactionId = TransactionId::from_bytes([0x33; TransactionId::LEN]); -/// The kind of the host's Activate in the tests' framing. -const ACTIVATE: u8 = 1; +/// The kind of the host's request for Ready in the tests' framing. +const AWAIT: u8 = 1; /// The kind of any other request of the host in the tests' framing. const PING: u8 = 2; @@ -96,15 +94,11 @@ fn configuration(hostname: &[u8], resources: ExpectedResources) -> Configuration } } -/// Returns the record of a clone's restore of `transaction`, of a root named `hostname` that +/// Returns the record of a clone's restore of [`TRANSACTION`], of a root named `hostname` that /// expects `resources`. -fn clone_record( - transaction: TransactionId, - hostname: &[u8], - resources: ExpectedResources, -) -> ActivationRecord { +fn clone_record(hostname: &[u8], resources: ExpectedResources) -> ActivationRecord { let restore = RestoreSection { - transaction, + transaction: TRANSACTION, template: TemplateReference { generation: StoreGeneration::new(NonZeroU64::MIN), key: Identity::from_bytes([0x44; Identity::LEN]), @@ -113,27 +107,31 @@ fn clone_record( ActivationRecord::new(Some(restore), configuration(hostname, resources)).unwrap() } -/// Returns the record of the host's Activate: the clone's restore of [`TRANSACTION`], which -/// expects the clone's resources. +/// Returns the record of the clone's restore packet, which expects the clone's resources. fn accepted() -> ActivationRecord { - clone_record(TRANSACTION, b"clone", resources()) + clone_record(b"clone", resources()) } -/// Returns the bytes of the record of the host's Activate. +/// Returns the bytes of the record of the clone's restore packet. fn record() -> Vec { accepted().encode() } -/// Returns what Ready reports about the activation of the host's Activate. +/// Returns what Ready reports about the activation of the record of the clone's restore packet. fn reported() -> ReadyActivation { ReadyActivation::of(&accepted()) } -/// Returns the activation, in its transport bootstrap, of the clone behind `fence`, whose repair -/// acknowledged its restore at `acknowledged`. -fn bootstrap(fence: &FakeFence, acknowledged: Instant) -> Activation { +/// Returns the activation, pending, of the clone behind `fence`, whose restore packet carried +/// `record`, if any, and whose repair acknowledged its restore at `acknowledged`. +fn restored( + record: Option<&[u8]>, + fence: &FakeFence, + acknowledged: Instant, +) -> Activation { Activation::with_resources( resources(), + record, Some(acknowledged), fence.clone(), TIMEOUT, @@ -141,31 +139,37 @@ fn bootstrap(fence: &FakeFence, acknowledged: Instant) -> Activation ) } -/// Returns the activation of [`bootstrap`] once the session's reset at `at` restricted it. -fn restricted(fence: &FakeFence, at: Instant) -> Activation { - let mut activation = bootstrap(fence, at); - let action = activation.input(at, ActivationInput::Reset).unwrap(); - assert_eq!(action, ActivationAction::Acknowledge); - assert_eq!(activation.state(), ActivationState::Restricted); +/// Returns the activation, pending, of the clone behind `fence`, whose restore packet carried the +/// record of [`record`], and whose repair acknowledged its restore at `acknowledged`. +fn pending(fence: &FakeFence, acknowledged: Instant) -> Activation { + let activation = restored(Some(&record()), fence, acknowledged); + assert_eq!(activation.state(), ActivationState::Pending); activation } -/// Returns the activation of [`restricted`] once it accepted the host's Activate at `at`. +/// Returns the activation of [`pending`] once it accepted the record at `at`. fn activating(fence: &FakeFence, at: Instant) -> Activation { - let mut activation = restricted(fence, at); - let action = activation - .input(at, ActivationInput::Activate(&record())) - .unwrap(); - assert_eq!(action, ActivationAction::Apply(Box::new(accepted()))); + let mut activation = pending(fence, at); + assert_eq!(*activation.accept(at).unwrap(), accepted()); assert_eq!(activation.state(), ActivationState::Activating); activation } /// Returns the activation of [`activating`] once its application of the record completed at -/// `at`. -fn ready(fence: &FakeFence, at: Instant) -> Activation { +/// `at`, before the host asked for Ready. +fn applied(fence: &FakeFence, at: Instant) -> Activation { let mut activation = activating(fence, at); assert_eq!(activation.complete(at, Ok(())).unwrap(), reported()); + assert_eq!(activation.state(), ActivationState::Applied); + assert_eq!(fence.opened(), 0); + activation +} + +/// Returns the activation of [`applied`] once the host asked for Ready at `at`. +fn ready(fence: &FakeFence, at: Instant) -> Activation { + let mut activation = applied(fence, at); + let asked = activation.input(at, ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); assert_eq!(activation.state(), ActivationState::Ready); activation } @@ -192,14 +196,12 @@ fn assert_final( assert_eq!(activation.state(), ActivationState::Failed); assert_eq!(activation.deadline(), None); assert_eq!(activation.established(), None); - let record = record(); let later = [ activation.input(now, ActivationInput::Flow).unwrap_err(), activation.input(now, ActivationInput::Reset).unwrap_err(), - activation - .input(now, ActivationInput::Activate(&record)) - .unwrap_err(), + activation.input(now, ActivationInput::Await).unwrap_err(), activation.input(now, ActivationInput::Request).unwrap_err(), + activation.accept(now).unwrap_err(), activation.complete(now, Ok(())).unwrap_err(), activation.start().unwrap_err(), activation.tick(now).unwrap_err(), @@ -212,44 +214,41 @@ fn assert_final( } #[test] -fn a_clone_bootstraps_takes_one_activate_and_opens_the_fence_once_ready() { +fn a_clone_activates_with_its_packet_s_record_and_opens_the_fence_once_ready() { let t0 = Instant::now(); let fence = FakeFence::default(); - let mut activation = bootstrap(&fence, t0); - assert_eq!(activation.state(), ActivationState::Bootstrap); + let mut activation = pending(&fence, t0); assert_eq!(activation.deadline(), Some(after(t0, 10))); - // Bytes of the session's reset arrive before it completes, and then the reset. - let flow = activation.input(after(t0, 1), ActivationInput::Flow); - assert_eq!(flow.unwrap(), ActivationAction::Continue); + assert_eq!(activation.established(), None); + // The record of the restore packet, which the activation decoded, goes to the adapter, which + // applies it, while the session takes the reset that the restore queued. + let record = activation.accept(after(t0, 1)).unwrap(); + assert_eq!(*record, accepted()); + assert_eq!(activation.state(), ActivationState::Activating); let reset = activation.input(after(t0, 1), ActivationInput::Reset); assert_eq!(reset.unwrap(), ActivationAction::Acknowledge); - assert_eq!(activation.state(), ActivationState::Restricted); let flow = activation.input(after(t0, 2), ActivationInput::Flow); assert_eq!(flow.unwrap(), ActivationAction::Continue); - // The Activate's record, which the activation decoded, goes to the adapter, which applies it. - let action = activation.input(after(t0, 2), ActivationInput::Activate(&record())); - assert_eq!( - action.unwrap(), - ActivationAction::Apply(Box::new(accepted())) - ); - assert_eq!(activation.state(), ActivationState::Activating); + // The host's request for Ready comes while the application runs, and joins it. + let asked = activation.input(after(t0, 2), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Join); assert_eq!(activation.deadline(), Some(after(t0, 10))); assert_eq!(activation.established(), None); assert_eq!(fence.opened(), 0); - let flow = activation.input(after(t0, 3), ActivationInput::Flow); - assert_eq!(flow.unwrap(), ActivationAction::Continue); // Once the application succeeded, the fence opens, and Ready reports the activation, with the // record's transaction. - let ready = activation.complete(after(t0, 4), Ok(())).unwrap(); + let ready = activation.complete(after(t0, 3), Ok(())).unwrap(); assert_eq!(ready, reported()); assert_eq!(ready.transaction, Some(TRANSACTION)); assert_eq!(fence.opened(), 1); assert_eq!(activation.state(), ActivationState::Ready); assert_eq!(activation.established(), Some(reported())); - // No deadline holds a ready clone: the host's mapping table and its start go to the adapter's - // order, whenever they come. + // No deadline holds a ready clone: the host's request for Ready, its mapping table, and its + // start go to the adapter whenever they come. assert_eq!(activation.deadline(), None); activation.tick(after(t0, 3600)).unwrap(); + let asked = activation.input(after(t0, 3600), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); let request = activation.input(after(t0, 3600), ActivationInput::Request); assert_eq!(request.unwrap(), ActivationAction::Serve); activation.start().unwrap(); @@ -263,46 +262,57 @@ fn a_clone_bootstraps_takes_one_activate_and_opens_the_fence_once_ready() { } #[test] -fn no_request_is_accepted_before_activate() { +fn an_activation_that_applied_before_the_host_asked_is_ready_once_it_asks() { let t0 = Instant::now(); - let record = record(); - // Before the session's bootstrap, the host can send nothing, Activate included; while the - // session is restricted, and while the activation runs, nothing but the Activate. - let cases: [(Setup, ActivationInput<'_>); 4] = [ - (bootstrap, ActivationInput::Request), - (bootstrap, ActivationInput::Activate(&record)), - (restricted, ActivationInput::Request), - (activating, ActivationInput::Request), - ]; - for (activation, input) in cases { - let fence = FakeFence::default(); - let mut activation = activation(&fence, t0); - let state = activation.state(); - let failure = activation.input(after(t0, 1), input).unwrap_err(); - assert_eq!(failure.state(), state, "{input:?}"); - assert!(matches!(failure.cause(), ActivationCause::Request)); - assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); - } + // The application completed before the host's request for Ready: the activation is applied, + // its fence closed, and its deadline holds, through a reset. + let fence = FakeFence::default(); + let mut activation = applied(&fence, t0); + assert_eq!(activation.deadline(), Some(after(t0, 10))); + assert_eq!(activation.established(), None); + activation.tick(after(t0, 9)).unwrap(); + let reset = activation.input(after(t0, 9), ActivationInput::Reset); + assert_eq!(reset.unwrap(), ActivationAction::Acknowledge); + assert_eq!(activation.state(), ActivationState::Applied); + assert_eq!(fence.opened(), 0); + // The host's request for Ready makes it ready, which opens the fence, and gets its Ready, as + // does every later one, without activating again. + let asked = activation.input(after(t0, 9), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); + assert_eq!(activation.state(), ActivationState::Ready); + assert_eq!(activation.established(), Some(reported())); + assert_eq!(activation.deadline(), None); + assert_eq!(fence.opened(), 1); + let asked = activation.input(after(t0, 60), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); + assert_eq!(fence.opened(), 1); + // A clone whose host never asks fails at the deadline, with its fence closed. + let fence = FakeFence::default(); + let mut activation = applied(&fence, t0); + let failure = activation.tick(after(t0, 10)).unwrap_err(); + assert_eq!(failure.state(), ActivationState::Applied); + assert!(matches!(failure.cause(), ActivationCause::Expired)); + assert_final(&mut activation, &fence, &failure, after(t0, 11), 0); } #[test] -fn a_reset_before_ready_is_final_and_one_after_it_starts_a_later_session() { +fn the_host_s_requests_for_ready_join_the_activation_or_get_its_result() { let t0 = Instant::now(); - // Only the reset that the restore queued bootstraps the session: another, such as once the - // host's attachment ended, fails the activation before it is ready. - for activation in [restricted, activating] { + // Before the clone is ready, a request for Ready joins the activation, whatever its state: + // the record came with the restore, so the host's request starts nothing. + for activation in [pending, activating] { let fence = FakeFence::default(); let mut activation = activation(&fence, t0); let state = activation.state(); - let failure = activation - .input(after(t0, 1), ActivationInput::Reset) - .unwrap_err(); - assert_eq!(failure.state(), state); - assert!(matches!(failure.cause(), ActivationCause::Reset)); - assert_final(&mut activation, &fence, &failure, after(t0, 1), 0); + for _ in 0..3 { + let asked = activation.input(after(t0, 1), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Join); + assert_eq!(activation.state(), state); + } + assert_eq!(fence.opened(), 0); } - // Once the clone is ready, or started, a reset starts a later session, which gets the - // established activation and transaction without activating again. + // Once the clone is ready, or started, each request gets the established activation, in + // every session, without activating again. for activation in [ready, started] { let fence = FakeFence::default(); let mut activation = activation(&fence, t0); @@ -311,160 +321,74 @@ fn a_reset_before_ready_is_final_and_one_after_it_starts_a_later_session() { let now = after(t0, 60 * session); let reset = activation.input(now, ActivationInput::Reset); assert_eq!(reset.unwrap(), ActivationAction::Acknowledge); + let asked = activation.input(now, ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); assert_eq!(activation.state(), state); assert_eq!(activation.established(), Some(reported())); - let retry = activation.input(now, ActivationInput::Activate(&record())); - assert_eq!(retry.unwrap(), ActivationAction::Ready(reported())); - let request = activation.input(now, ActivationInput::Request); - assert_eq!(request.unwrap(), ActivationAction::Serve); } assert_eq!(fence.opened(), 1); } } #[test] -fn retries_join_the_running_activation_and_get_its_cached_result() { +fn a_reset_abandons_only_the_session_s_requests_for_ready() { let t0 = Instant::now(); + // A reset, such as once the host's attachment ended, starts a later session, which the + // adapter acknowledges: the activation goes on with the record of the restore packet, which + // no session can change. + for activation in [pending, activating, applied, ready, started] { + let fence = FakeFence::default(); + let mut activation = activation(&fence, t0); + let state = activation.state(); + let opened = fence.opened(); + for session in 1..=3 { + let reset = activation.input(after(t0, session), ActivationInput::Reset); + assert_eq!(reset.unwrap(), ActivationAction::Acknowledge); + assert_eq!(activation.state(), state); + } + assert_eq!(fence.opened(), opened); + } + // A clone that the host reset while it activated completes its application all the same, and + // the reset abandons the host's request for Ready of the earlier session: the activation is + // applied, until the host asks again. let fence = FakeFence::default(); let mut activation = activating(&fence, t0); - // A retry of the same transaction, with a byte-equal record, joins the activation that runs, - // whose deadline stays where it was. - for second in 1..=3 { - let retry = activation.input(after(t0, second), ActivationInput::Activate(&record())); - assert_eq!(retry.unwrap(), ActivationAction::Join); - assert_eq!(activation.state(), ActivationState::Activating); - assert_eq!(activation.deadline(), Some(after(t0, 10))); - } - assert_eq!(fence.opened(), 0); + let asked = activation.input(after(t0, 1), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Join); + activation + .input(after(t0, 1), ActivationInput::Reset) + .unwrap(); assert_eq!( - activation.complete(after(t0, 4), Ok(())).unwrap(), + activation.complete(after(t0, 2), Ok(())).unwrap(), reported() ); - // Afterwards, one gets the cached result, before and after the start, without applying the - // record again or opening the fence again. - let retry = activation.input(after(t0, 5), ActivationInput::Activate(&record())); - assert_eq!(retry.unwrap(), ActivationAction::Ready(reported())); - activation.start().unwrap(); - let retry = activation.input(after(t0, 6), ActivationInput::Activate(&record())); - assert_eq!(retry.unwrap(), ActivationAction::Ready(reported())); - assert_eq!(activation.state(), ActivationState::Started); + assert_eq!(activation.state(), ActivationState::Applied); + assert_eq!(fence.opened(), 0); + let asked = activation.input(after(t0, 3), ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Ready(reported())); assert_eq!(fence.opened(), 1); } #[test] -fn a_changed_record_or_a_new_transaction_is_final() { +fn no_other_request_is_accepted_before_ready() { let t0 = Instant::now(); - let renamed = clone_record(TRANSACTION, b"renamed", resources()).encode(); - let other = clone_record(OTHER_TRANSACTION, b"clone", resources()).encode(); - let cold = ActivationRecord::new(None, configuration(b"clone", resources())) - .unwrap() - .encode(); - let whole = record(); - let truncated = &whole[..whole.len() - 1]; - let retries: [(&[u8], Expected); 4] = [ - (&renamed, |cause| matches!(cause, ActivationCause::Changed)), - (&other, |cause| { - matches!( - cause, - ActivationCause::Transaction { - accepted: TRANSACTION, - offered: OTHER_TRANSACTION, - } - ) - }), - (&cold, |cause| matches!(cause, ActivationCause::Cold)), - (truncated, |cause| { - matches!(cause, ActivationCause::Record(ActivationError::Truncated)) - }), - ]; - // While the activation runs, and once it is ready, or started, whose fence stays open. - let states: [(Setup, u32); 3] = [(activating, 0), (ready, 1), (started, 1)]; - for (activation, opened) in states { - for (retry, cause) in retries { - let fence = FakeFence::default(); - let mut activation = activation(&fence, t0); - let state = activation.state(); - let failure = activation - .input(after(t0, 1), ActivationInput::Activate(retry)) - .unwrap_err(); - assert_eq!(failure.state(), state); - assert!(cause(failure.cause()), "{state}: {failure}"); - assert_final(&mut activation, &fence, &failure, after(t0, 2), opened); - } + // The host sends its mapping table, its start, and its probes only once Ready came, even + // once the record applied. + for activation in [pending, activating, applied] { + let fence = FakeFence::default(); + let mut activation = activation(&fence, t0); + let state = activation.state(); + let failure = activation + .input(after(t0, 1), ActivationInput::Request) + .unwrap_err(); + assert_eq!(failure.state(), state); + assert!(matches!(failure.cause(), ActivationCause::Request)); + assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); } } #[test] -fn an_activate_too_long_to_read_fails_as_a_record_that_long() { - let t0 = Instant::now(); - // In the transport bootstrap, as any Activate. - let fence = FakeFence::default(); - let mut activation = bootstrap(&fence, t0); - let failure = activation - .input(after(t0, 1), ActivationInput::TooLong(MAX_RECORD_BYTES + 1)) - .unwrap_err(); - assert_eq!(failure.state(), ActivationState::Bootstrap); - assert!(matches!(failure.cause(), ActivationCause::Request)); - assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); - // From then on, as an Activate whose record has that many bytes, which no record has: while - // restricted, while the activation runs, and once it is ready, or started, whose fence stays - // open. - let states: [(Setup, u32); 4] = [(restricted, 0), (activating, 0), (ready, 1), (started, 1)]; - for (activation, opened) in states { - for length in [MAX_RECORD_BYTES + 1, usize::MAX] { - let fence = FakeFence::default(); - let mut activation = activation(&fence, t0); - let state = activation.state(); - let failure = activation - .input(after(t0, 1), ActivationInput::TooLong(length)) - .unwrap_err(); - assert_eq!(failure.state(), state); - assert!( - matches!( - failure.cause(), - ActivationCause::Record(ActivationError::TooLong(long)) if *long == length - ), - "{state}: {failure}" - ); - assert_final(&mut activation, &fence, &failure, after(t0, 2), opened); - } - } - // Past the deadline, before the clone is ready, it fails as every input does. - let fence = FakeFence::default(); - let mut activation = restricted(&fence, t0); - let failure = activation - .input( - after(t0, 10), - ActivationInput::TooLong(MAX_RECORD_BYTES + 1), - ) - .unwrap_err(); - assert!(matches!(failure.cause(), ActivationCause::Expired)); - assert_final(&mut activation, &fence, &failure, after(t0, 11), 0); -} - -#[test] -fn a_failed_application_is_final_and_no_retry_applies_the_record_again() { - let t0 = Instant::now(); - let fence = FakeFence::default(); - let mut activation = activating(&fence, t0); - let failure = activation - .complete(after(t0, 1), Err("the root's assembly failed".into())) - .unwrap_err(); - assert_eq!(failure.state(), ActivationState::Activating); - let ActivationCause::Failed(error) = failure.cause() else { - panic!("unexpected cause: {failure}"); - }; - assert_eq!(error.to_string(), "the root's assembly failed"); - assert_eq!( - failure.source().unwrap().to_string(), - "the root's assembly failed" - ); - // A byte-equal retry neither joins nor applies the record again: it gets the failure. - assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); -} - -#[test] -fn an_activate_is_decoded_and_validated_before_any_side_effect() { +fn the_packet_s_record_is_decoded_and_validated_before_any_side_effect() { let t0 = Instant::now(); let mut long = record(); long.resize(MAX_RECORD_BYTES + 1, 0); @@ -486,88 +410,99 @@ fn an_activate_is_decoded_and_validated_before_any_side_effect() { cpus: CPUS, memory: MEMORY + (64 << 20), }; - let cases: [(Vec, Expected); 8] = [ - (Vec::new(), |cause| { + let cases: [(Option>, Expected); 8] = [ + (None, |cause| matches!(cause, ActivationCause::Missing)), + (Some(Vec::new()), |cause| { matches!(cause, ActivationCause::Record(ActivationError::Truncated)) }), - (long, |cause| { + (Some(long), |cause| { matches!( cause, ActivationCause::Record(ActivationError::TooLong(1025)) ) }), - (version, |cause| { + (Some(version), |cause| { matches!(cause, ActivationCause::Record(ActivationError::Version(2))) }), - (cold, |cause| matches!(cause, ActivationCause::Cold)), - ( - clone_record(TRANSACTION, b"clone", more_cpus).encode(), - |cause| { - matches!( - cause, - ActivationCause::Resources { - expected: ExpectedResources { cpus: 4, .. }, - actual: ExpectedResources { - cpus: CPUS, - memory: MEMORY - }, - } - ) - }, - ), + (Some(cold), |cause| matches!(cause, ActivationCause::Cold)), + (Some(clone_record(b"clone", more_cpus).encode()), |cause| { + matches!( + cause, + ActivationCause::Resources { + expected: ExpectedResources { cpus: 4, .. }, + actual: ExpectedResources { + cpus: CPUS, + memory: MEMORY + }, + } + ) + }), ( - clone_record(TRANSACTION, b"clone", less_memory).encode(), + Some(clone_record(b"clone", less_memory).encode()), |cause| matches!(cause, ActivationCause::Resources { expected, .. } if expected.memory == MEMORY - (128 << 20)), ), ( - clone_record(TRANSACTION, b"clone", partial_block).encode(), + Some(clone_record(b"clone", partial_block).encode()), |cause| matches!(cause, ActivationCause::Resources { expected, .. } if expected.memory == MEMORY + (64 << 20)), ), - ( - clone_record(OTHER_TRANSACTION, b"clone", more_cpus).encode(), - |cause| matches!(cause, ActivationCause::Resources { .. }), - ), ]; for (bytes, cause) in cases { let fence = FakeFence::default(); - let mut activation = restricted(&fence, t0); - let failure = activation - .input(after(t0, 1), ActivationInput::Activate(&bytes)) - .unwrap_err(); - assert_eq!(failure.state(), ActivationState::Restricted); + let mut activation = restored(bytes.as_deref(), &fence, t0); + let failure = activation.accept(after(t0, 1)).unwrap_err(); + assert_eq!(failure.state(), ActivationState::Pending); assert!(cause(failure.cause()), "{failure}"); assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); } // A valid record goes to the adapter exactly as it decodes, and opens nothing yet. let fence = FakeFence::default(); - let mut activation = restricted(&fence, t0); - let action = activation.input(after(t0, 1), ActivationInput::Activate(&record())); + let mut activation = pending(&fence, t0); + let accepted = activation.accept(after(t0, 1)).unwrap(); + assert_eq!(*accepted, ActivationRecord::decode(&record()).unwrap()); + assert_eq!(fence.opened(), 0); +} + +#[test] +fn a_failed_application_is_final_and_nothing_applies_the_record_again() { + let t0 = Instant::now(); + let fence = FakeFence::default(); + let mut activation = activating(&fence, t0); + let failure = activation + .complete(after(t0, 1), Err("the root's assembly failed".into())) + .unwrap_err(); + assert_eq!(failure.state(), ActivationState::Activating); + let ActivationCause::Failed(error) = failure.cause() else { + panic!("unexpected cause: {failure}"); + }; + assert_eq!(error.to_string(), "the root's assembly failed"); assert_eq!( - action.unwrap(), - ActivationAction::Apply(Box::new(ActivationRecord::decode(&record()).unwrap())) + failure.source().unwrap().to_string(), + "the root's assembly failed" ); - assert_eq!(fence.opened(), 0); + // Neither an acceptance nor a request for Ready applies the record again: each gets the + // failure. + assert_final(&mut activation, &fence, &failure, after(t0, 2), 0); } #[test] -fn the_deadline_starts_at_the_acknowledgement_and_retries_never_extend_it() { +fn the_deadline_starts_at_the_acknowledgement_and_nothing_extends_it() { let t0 = Instant::now(); let fence = FakeFence::default(); // The repair acknowledged the restore at `t0`, and the adapter made the activation later. - let mut activation = - Activation::with_resources(resources(), Some(t0), fence.clone(), TIMEOUT, after(t0, 3)); - assert_eq!(activation.deadline(), Some(after(t0, 10))); - activation - .input(after(t0, 8), ActivationInput::Reset) - .unwrap(); - let action = activation.input(after(t0, 9), ActivationInput::Activate(&record())); - assert_eq!( - action.unwrap(), - ActivationAction::Apply(Box::new(accepted())) + let mut activation = Activation::with_resources( + resources(), + Some(&record()), + Some(t0), + fence.clone(), + TIMEOUT, + after(t0, 3), ); + assert_eq!(activation.deadline(), Some(after(t0, 10))); + assert_eq!(*activation.accept(after(t0, 8)).unwrap(), accepted()); let late = after(t0, 9) + Duration::from_millis(999); - let retry = activation.input(late, ActivationInput::Activate(&record())); - assert_eq!(retry.unwrap(), ActivationAction::Join); + let asked = activation.input(late, ActivationInput::Await); + assert_eq!(asked.unwrap(), ActivationAction::Join); + activation.input(late, ActivationInput::Reset).unwrap(); activation.tick(late).unwrap(); assert_eq!(activation.deadline(), Some(after(t0, 10))); // At the deadline, the activation fails, however its application ends after it. @@ -575,17 +510,45 @@ fn the_deadline_starts_at_the_acknowledgement_and_retries_never_extend_it() { assert_eq!(failure.state(), ActivationState::Activating); assert!(matches!(failure.cause(), ActivationCause::Expired)); assert_final(&mut activation, &fence, &failure, after(t0, 11), 0); - // An application that completes at the deadline completes too late, as does any input. - for complete in [true, false] { + // An acceptance, an application that completes, a request for Ready of an applied + // activation, and any input at the deadline come too late. + let late: [(Setup, Failing); 5] = [ + ( + pending, + Box::new(move |activation| activation.accept(after(t0, 10)).map(drop)), + ), + ( + activating, + Box::new(move |activation| activation.complete(after(t0, 10), Ok(())).map(drop)), + ), + ( + activating, + Box::new(move |activation| { + let input = ActivationInput::Await; + activation.input(after(t0, 10), input).map(drop) + }), + ), + ( + applied, + Box::new(move |activation| { + let input = ActivationInput::Await; + activation.input(after(t0, 10), input).map(drop) + }), + ), + ( + pending, + Box::new(move |activation| { + let input = ActivationInput::Flow; + activation.input(after(t0, 10), input).map(drop) + }), + ), + ]; + for (activation, call) in late { let fence = FakeFence::default(); - let mut activation = activating(&fence, t0); - let failure = if complete { - activation.complete(after(t0, 10), Ok(())).unwrap_err() - } else { - let retry = ActivationInput::Activate(&record()); - activation.input(after(t0, 10), retry).unwrap_err() - }; - assert_eq!(failure.state(), ActivationState::Activating); + let mut activation = activation(&fence, t0); + let state = activation.state(); + let failure = call(&mut activation).unwrap_err(); + assert_eq!(failure.state(), state); assert!(matches!(failure.cause(), ActivationCause::Expired)); assert_final(&mut activation, &fence, &failure, after(t0, 11), 0); } @@ -596,32 +559,54 @@ fn an_ungated_restore_s_deadline_starts_at_the_activation() { let t0 = Instant::now(); let fence = FakeFence::default(); // Without an acknowledgement, the deadline starts when the adapter makes the activation. - let mut activation = - Activation::with_resources(resources(), None, fence.clone(), TIMEOUT, after(t0, 5)); + let mut activation = Activation::with_resources( + resources(), + Some(&record()), + None, + fence.clone(), + TIMEOUT, + after(t0, 5), + ); assert_eq!(activation.deadline(), Some(after(t0, 15))); activation.tick(after(t0, 14)).unwrap(); let failure = activation .input(after(t0, 15), ActivationInput::Reset) .unwrap_err(); - assert_eq!(failure.state(), ActivationState::Bootstrap); + assert_eq!(failure.state(), ActivationState::Pending); assert!(matches!(failure.cause(), ActivationCause::Expired)); assert_final(&mut activation, &fence, &failure, after(t0, 16), 0); // A deadline that no instant can hold never passes. - let mut activation = - Activation::with_resources(resources(), Some(t0), fence.clone(), Duration::MAX, t0); + let mut activation = Activation::with_resources( + resources(), + Some(&record()), + Some(t0), + fence.clone(), + Duration::MAX, + t0, + ); assert_eq!(activation.deadline(), None); activation.tick(after(t0, 1 << 30)).unwrap(); let reset = activation.input(after(t0, 1 << 30), ActivationInput::Reset); assert_eq!(reset.unwrap(), ActivationAction::Acknowledge); + assert_eq!(*activation.accept(after(t0, 1 << 30)).unwrap(), accepted()); } #[test] -fn out_of_order_completions_and_starts_are_final() { +fn out_of_order_acceptances_completions_and_starts_are_final() { let t0 = Instant::now(); - // An activation completes only while it runs, and the sandbox starts only once it is ready, - // and once: the fence opens no sooner, and no more than once. - let completions: [(Setup, u32); 4] = - [(bootstrap, 0), (restricted, 0), (ready, 1), (started, 1)]; + // An activation accepts its record once, completes only while it runs, and the sandbox starts + // only once it is ready, and once: the fence opens no sooner, and no more than once. + let acceptances: [(Setup, u32); 4] = [(activating, 0), (applied, 0), (ready, 1), (started, 1)]; + for (activation, opened) in acceptances { + let fence = FakeFence::default(); + let mut activation = activation(&fence, t0); + let state = activation.state(); + let failure = activation.accept(after(t0, 1)).unwrap_err(); + assert_eq!(failure.state(), state); + assert!(matches!(failure.cause(), ActivationCause::Completion)); + assert_final(&mut activation, &fence, &failure, after(t0, 2), opened); + } + let completions: [(Setup, u32); 4] = [(pending, 0), (applied, 0), (ready, 1), (started, 1)]; for (activation, opened) in completions { let fence = FakeFence::default(); let mut activation = activation(&fence, t0); @@ -631,12 +616,7 @@ fn out_of_order_completions_and_starts_are_final() { assert!(matches!(failure.cause(), ActivationCause::Completion)); assert_final(&mut activation, &fence, &failure, after(t0, 2), opened); } - let starts: [(Setup, u32); 4] = [ - (bootstrap, 0), - (restricted, 0), - (activating, 0), - (started, 1), - ]; + let starts: [(Setup, u32); 4] = [(pending, 0), (activating, 0), (applied, 0), (started, 1)]; for (activation, opened) in starts { let fence = FakeFence::default(); let mut activation = activation(&fence, t0); @@ -669,10 +649,10 @@ enum Reply { /// An adapter of the tests' framing, which drives an activation as an adapter's loop does: it /// takes the host's bytes as its transport delivers them, passes the activation each message -/// that they complete, and flow control for each byte that completes none, keeps the records -/// that the activation accepts, whose application the test completes when it chooses, and queues -/// its replies for an output that takes nothing, as a host that reads nothing leaves it, and -/// which it never waits for. +/// that they complete, and flow control for each byte that completes none, counts the host's +/// requests for Ready that joined the activation in the current session, which it answers once +/// the test completed the activation, and queues its replies for an output that takes nothing, +/// as a host that reads nothing leaves it, and which it never waits for. struct Adapter { /// The activation. activation: Activation, @@ -680,8 +660,8 @@ struct Adapter { message: Vec, /// The replies that wait for the output. output: Vec, - /// The records whose application the activation handed over. - applied: Vec, + /// The host's requests for Ready of the current session that joined the activation. + joined: usize, } impl Adapter { @@ -691,7 +671,7 @@ impl Adapter { activation, message: Vec::new(), output: Vec::new(), - applied: Vec::new(), + joined: 0, } } @@ -716,7 +696,7 @@ impl Adapter { let action = if complete { let message = std::mem::take(&mut self.message); let input = match message.as_slice() { - [ACTIVATE, _, _, record @ ..] => ActivationInput::Activate(record), + [AWAIT, _, _] => ActivationInput::Await, _ => ActivationInput::Request, }; self.activation.input(now, input)? @@ -728,13 +708,25 @@ impl Adapter { Ok(()) } - /// Does what `action` says, except that it leaves an accepted record's application to the - /// test. + /// Completes the activation at `now` with an application that succeeded, and answers each + /// request for Ready that joined it. + fn complete(&mut self, now: Instant) -> Result<(), ActivationFailure> { + let ready = self.activation.complete(now, Ok(()))?; + for _ in 0..std::mem::take(&mut self.joined) { + self.output.push(Reply::Ready(ready)); + } + Ok(()) + } + + /// Does what `action` says. fn act(&mut self, action: ActivationAction) { match action { - ActivationAction::Acknowledge => self.output.push(Reply::Acknowledgement), - ActivationAction::Continue | ActivationAction::Join | ActivationAction::Serve => {} - ActivationAction::Apply(record) => self.applied.push(*record), + ActivationAction::Acknowledge => { + self.joined = 0; + self.output.push(Reply::Acknowledgement); + } + ActivationAction::Continue | ActivationAction::Serve => {} + ActivationAction::Join => self.joined += 1, ActivationAction::Ready(ready) => self.output.push(Reply::Ready(ready)), } } @@ -743,15 +735,16 @@ impl Adapter { #[test] fn the_deadline_stays_live_under_trickled_input_and_blocked_output() { let t0 = Instant::now(); - let activate = message(ACTIVATE, &record()); - assert!(activate.len() > 10); - // A host that trickles its Activate, a byte a second, never completes it by the deadline: - // the byte at the deadline fails the activation, while the output took nothing. + let asked = message(AWAIT, b""); + let ping = message(PING, b"0123456789"); + // A host that trickles a message, a byte a second, while the record's application runs past + // the deadline gets no Ready: the byte at the deadline fails the activation, while the output + // took nothing. let fence = FakeFence::default(); - let mut adapter = Adapter::new(bootstrap(&fence, t0)); + let mut adapter = Adapter::new(activating(&fence, t0)); adapter.reset(t0).unwrap(); let mut failure = None; - for (second, byte) in (1..).zip(&activate) { + for (second, byte) in (1..).zip(ping.iter().cycle()) { if let Err(error) = adapter.receive(after(t0, second), &[*byte]) { failure = Some((second, error)); break; @@ -759,15 +752,14 @@ fn the_deadline_stays_live_under_trickled_input_and_blocked_output() { } let (second, failure) = failure.unwrap(); assert_eq!(second, 10); - assert_eq!(failure.state(), ActivationState::Restricted); + assert_eq!(failure.state(), ActivationState::Activating); assert!(matches!(failure.cause(), ActivationCause::Expired)); - assert!(adapter.applied.is_empty()); assert_eq!(adapter.output, [Reply::Acknowledgement]); assert_final(&mut adapter.activation, &fence, &failure, after(t0, 11), 0); - // A host that sends nothing leaves the adapter only its wake at the deadline, which fails the - // activation as well. + // A host that sends nothing leaves the adapter only its wake at the deadline, which fails an + // activation whose application did not complete. let fence = FakeFence::default(); - let mut adapter = Adapter::new(bootstrap(&fence, t0)); + let mut adapter = Adapter::new(activating(&fence, t0)); adapter.reset(after(t0, 1)).unwrap(); let deadline = adapter.activation.deadline().unwrap(); assert_eq!(deadline, after(t0, 10)); @@ -776,38 +768,56 @@ fn the_deadline_stays_live_under_trickled_input_and_blocked_output() { .tick(deadline - Duration::from_millis(1)) .unwrap(); let failure = adapter.activation.tick(deadline).unwrap_err(); - assert_eq!(failure.state(), ActivationState::Restricted); + assert_eq!(failure.state(), ActivationState::Activating); assert!(matches!(failure.cause(), ActivationCause::Expired)); assert_final(&mut adapter.activation, &fence, &failure, after(t0, 11), 0); - // A host whose Activate arrives at once, and that trickles a retry while the record's - // application runs, gets the retry joined, without another application or a later deadline, - // and an application that completes after the deadline fails. + // A host whose request for Ready arrives while the record's application runs gets Ready once + // the application completed, and only in the session in which it asked: a reset abandons the + // requests of the earlier session, and a host that asks again in the later one gets one + // Ready. let fence = FakeFence::default(); - let mut adapter = Adapter::new(bootstrap(&fence, t0)); + let mut adapter = Adapter::new(activating(&fence, t0)); adapter.reset(t0).unwrap(); - adapter.receive(after(t0, 1), &activate).unwrap(); - assert_eq!(adapter.applied, [accepted()]); - for (second, byte) in (2..9).zip(&activate) { + for (second, byte) in (1..).zip(&asked) { adapter.receive(after(t0, second), &[*byte]).unwrap(); } - adapter.receive(after(t0, 9), &activate[7..]).unwrap(); - assert_eq!(adapter.applied, [accepted()]); - assert_eq!(adapter.activation.deadline(), Some(after(t0, 10))); - let failure = adapter - .activation - .complete(after(t0, 11), Ok(())) - .unwrap_err(); - assert_eq!(failure.state(), ActivationState::Activating); - assert!(matches!(failure.cause(), ActivationCause::Expired)); + assert_eq!(adapter.joined, 1); + adapter.reset(after(t0, 4)).unwrap(); + adapter.receive(after(t0, 5), &asked).unwrap(); + adapter.receive(after(t0, 5), &asked).unwrap(); + adapter.complete(after(t0, 6)).unwrap(); + assert_eq!( + adapter.output, + [ + Reply::Acknowledgement, + Reply::Acknowledgement, + Reply::Ready(reported()), + Reply::Ready(reported()) + ] + ); + assert_eq!(fence.opened(), 1); + // Once ready, a request for Ready gets its answer at once. + adapter.receive(after(t0, 7), &asked).unwrap(); + assert_eq!(adapter.output.last(), Some(&Reply::Ready(reported()))); + // A host that asks for Ready only once the application completed gets it then, which opens + // the fence. + let fence = FakeFence::default(); + let mut adapter = Adapter::new(activating(&fence, t0)); + adapter.reset(t0).unwrap(); + adapter.complete(after(t0, 1)).unwrap(); assert_eq!(adapter.output, [Reply::Acknowledgement]); - assert_final(&mut adapter.activation, &fence, &failure, after(t0, 12), 0); + assert_eq!(fence.opened(), 0); + adapter.receive(after(t0, 2), &asked).unwrap(); + assert_eq!( + adapter.output, + [Reply::Acknowledgement, Reply::Ready(reported())] + ); + assert_eq!(fence.opened(), 1); // Another message of the framing, whole, while the record's application runs fails the - // activation, as it would before the Activate. + // activation. let fence = FakeFence::default(); let mut adapter = Adapter::new(activating(&fence, t0)); - let failure = adapter - .receive(after(t0, 1), &message(PING, b"")) - .unwrap_err(); + let failure = adapter.receive(after(t0, 1), &ping).unwrap_err(); assert_eq!(failure.state(), ActivationState::Activating); assert!(matches!(failure.cause(), ActivationCause::Request)); assert_final(&mut adapter.activation, &fence, &failure, after(t0, 2), 0); @@ -819,8 +829,6 @@ type Failing = Box) -> Result<(), ActivationFa #[test] fn strict_failures_are_returned_and_never_power_off() { let t0 = Instant::now(); - let renamed = clone_record(TRANSACTION, b"renamed", resources()).encode(); - let other = clone_record(OTHER_TRANSACTION, b"clone", resources()).encode(); let cold = ActivationRecord::new(None, configuration(b"clone", resources())) .unwrap() .encode(); @@ -828,42 +836,34 @@ fn strict_failures_are_returned_and_never_power_off() { cpus: CPUS + 1, memory: MEMORY, }; - let unexpected = clone_record(TRANSACTION, b"clone", more_cpus).encode(); - let activate = move |record: Vec| -> Failing { - Box::new(move |activation| { - let input = ActivationInput::Activate(&record); - activation.input(after(t0, 1), input).map(drop) - }) - }; + let unexpected = clone_record(b"clone", more_cpus).encode(); // Each failure, in a state in which it comes: the activation returns it, keeps it, and changes // nothing else. It has no system through which it could power the VM off: the adapter reports // the failure, and ends the VM. - let failures: [(Setup, Failing); 11] = [ + let records: [Option>; 4] = [None, Some(Vec::new()), Some(cold), Some(unexpected)]; + let mut causes = Vec::new(); + for record in records { + let fence = FakeFence::default(); + let mut activation = restored(record.as_deref(), &fence, t0); + let failure = activation.accept(after(t0, 1)).unwrap_err(); + let cause = std::mem::discriminant(failure.cause()); + assert!(!causes.contains(&cause), "{failure}"); + causes.push(cause); + assert_final(&mut activation, &fence, &failure, after(t0, 11), 0); + } + let failures: [(Setup, Failing); 5] = [ ( - restricted, + activating, Box::new(move |activation| activation.tick(after(t0, 10))), ), ( activating, - Box::new(move |activation| { - activation - .input(after(t0, 1), ActivationInput::Reset) - .map(drop) - }), - ), - ( - restricted, Box::new(move |activation| { activation .input(after(t0, 1), ActivationInput::Request) .map(drop) }), ), - (restricted, activate(Vec::new())), - (restricted, activate(cold)), - (restricted, activate(unexpected)), - (ready, activate(other)), - (started, activate(renamed)), ( activating, Box::new(move |activation| { @@ -873,12 +873,11 @@ fn strict_failures_are_returned_and_never_power_off() { }), ), ( - bootstrap, + pending, Box::new(move |activation| activation.complete(after(t0, 1), Ok(())).map(drop)), ), (activating, Box::new(|activation| activation.start())), ]; - let mut causes = Vec::new(); for (activation, fail) in failures { let fence = FakeFence::default(); let mut activation = activation(&fence, t0); @@ -894,9 +893,9 @@ fn strict_failures_are_returned_and_never_power_off() { #[test] fn failures_name_their_state_and_their_cause() { let states = [ - (ActivationState::Bootstrap, "transport bootstrap"), - (ActivationState::Restricted, "restricted"), + (ActivationState::Pending, "pending"), (ActivationState::Activating, "activating"), + (ActivationState::Applied, "applied"), (ActivationState::Ready, "ready"), (ActivationState::Started, "started"), (ActivationState::Failed, "failed"), @@ -910,36 +909,41 @@ fn failures_name_their_state_and_their_cause() { }; let cases = [ ( - failure(ActivationState::Bootstrap, ActivationCause::Expired), - "the activation failed in its transport bootstrap state: the deadline passed before \ - the activation was ready", + failure(ActivationState::Pending, ActivationCause::Expired), + "the activation failed in its pending state: the deadline passed before the \ + activation was ready", + ), + ( + failure(ActivationState::Pending, ActivationCause::Missing), + "the activation failed in its pending state: the clone's restore packet carried no \ + activation record", ), ( - failure(ActivationState::Activating, ActivationCause::Reset), - "the activation failed in its activating state: the session was reset again before \ - the activation was ready", + failure(ActivationState::Activating, ActivationCause::Request), + "the activation failed in its activating state: the host sent a request before the \ + activation was ready", ), ( - failure(ActivationState::Restricted, ActivationCause::Request), - "the activation failed in its restricted state: the host sent a request before the \ + failure(ActivationState::Applied, ActivationCause::Expired), + "the activation failed in its applied state: the deadline passed before the \ activation was ready", ), ( failure( - ActivationState::Restricted, + ActivationState::Pending, ActivationCause::Record(ActivationError::Truncated), ), - "the activation failed in its restricted state: the Activate's record is refused: \ - the record ends inside its version or one of its fields", + "the activation failed in its pending state: the restore packet's activation record \ + is refused: the record ends inside its version or one of its fields", ), ( - failure(ActivationState::Restricted, ActivationCause::Cold), - "the activation failed in its restricted state: the Activate's record is a cold \ - boot's, without a restore section", + failure(ActivationState::Pending, ActivationCause::Cold), + "the activation failed in its pending state: the restore packet's activation record \ + is a cold boot's, without a restore section", ), ( failure( - ActivationState::Restricted, + ActivationState::Pending, ActivationCause::Resources { expected: ExpectedResources { cpus: 4, @@ -948,26 +952,9 @@ fn failures_name_their_state_and_their_cause() { actual: resources(), }, ), - "the activation failed in its restricted state: the Activate's record expects 4 CPUs \ + "the activation failed in its pending state: the activation record expects 4 CPUs \ and 2147483648 bytes of memory, and the clone has 2 CPUs and 1073741824 bytes", ), - ( - failure( - ActivationState::Ready, - ActivationCause::Transaction { - accepted: TRANSACTION, - offered: OTHER_TRANSACTION, - }, - ), - "the activation failed in its ready state: the Activate names the transaction \ - 33333333333333333333333333333333, after an Activate of the transaction \ - 22222222222222222222222222222222", - ), - ( - failure(ActivationState::Started, ActivationCause::Changed), - "the activation failed in its started state: the Activate changes the record of the \ - accepted Activate's transaction", - ), ( failure( ActivationState::Activating, @@ -977,9 +964,9 @@ fn failures_name_their_state_and_their_cause() { the root's assembly failed", ), ( - failure(ActivationState::Restricted, ActivationCause::Completion), - "the activation failed in its restricted state: an activation that did not run \ - completed", + failure(ActivationState::Ready, ActivationCause::Completion), + "the activation failed in its ready state: the activation's record was accepted \ + again, or an activation that did not run completed", ), ( failure(ActivationState::Activating, ActivationCause::Start), @@ -1004,13 +991,14 @@ fn failures_name_their_state_and_their_cause() { } #[test] -fn an_activation_debugs_its_state_resources_and_deadline_without_its_fence() { +fn an_activation_debugs_its_state_resources_and_deadline_without_its_fence_or_record() { let t0 = Instant::now(); let activation = Activation::with_resources( ExpectedResources { cpus: 4, memory: 2 << 30, }, + Some(&record()), Some(t0), FakeFence::default(), TIMEOUT, @@ -1019,7 +1007,7 @@ fn an_activation_debugs_its_state_resources_and_deadline_without_its_fence() { assert_eq!( format!("{activation:?}"), format!( - "Activation {{ state: Bootstrap, resources: ExpectedResources {{ cpus: 4, memory: \ + "Activation {{ state: Pending, resources: ExpectedResources {{ cpus: 4, memory: \ 2147483648 }}, deadline: {:?}, .. }}", Some(after(t0, 10)) ) diff --git a/nvx-rs/src/guest/clone/repair.rs b/nvx-rs/src/guest/clone/repair.rs index ce2193660..fabedb2c7 100644 --- a/nvx-rs/src/guest/clone/repair.rs +++ b/nvx-rs/src/guest/clone/repair.rs @@ -168,7 +168,7 @@ impl Repair { self.state = State::Finish(resources); } State::Finish(resources) => return self.finish(*resources), - State::Repaired(repaired) => return Ok(*repaired), + State::Repaired(repaired) => return Ok(repaired.clone()), State::Failed(error) => return Err(error.clone()), } } @@ -344,8 +344,9 @@ impl Repair { cpus: resources.cpus, memory: resources.memory, acknowledged_at, + activation: self.restore.activation_record().map(Box::from), }; - self.state = State::Repaired(repaired); + self.state = State::Repaired(repaired.clone()); Ok(repaired) } } @@ -412,9 +413,10 @@ fn first_found(log: &mut BTreeMap, block: u64) -> bool { } /// A clone that its [`Repair`] repaired: its new VM generation ID, the resources that it has, -/// which an activation checks against the resources that it expects, and when its repair -/// acknowledged its restore. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +/// which an activation checks against the resources that it expects, when its repair +/// acknowledged its restore, and the activation record that its restore packet carried, from +/// which its activation takes its configuration. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct Repaired { /// The new VM generation ID. generation_id: GenerationId, @@ -425,6 +427,8 @@ pub struct Repaired { /// An instant no later than the repair's acknowledgement of the restore, taken right before /// the finish, if the restore asked for the acknowledgement. acknowledged_at: Option, + /// The bytes of the activation record of the restore packet, if it carried one, undecoded. + activation: Option>, } impl Repaired { @@ -453,6 +457,14 @@ impl Repaired { pub fn acknowledged_at(&self) -> Option { self.acknowledged_at } + + /// Returns the bytes of the activation record that the host handed the clone in its restore + /// packet, as [`Restore::activation_record`] returned them, undecoded, or `None` for a restore + /// whose packet carried none: the clone's [`Activation`](super::Activation) decodes and + /// validates them. + pub fn activation_record(&self) -> Option<&[u8]> { + self.activation.as_deref() + } } /// How far a clone's repair got, across its attempts, which its [`RepairError`] reports: the diff --git a/nvx-rs/src/guest/mod.rs b/nvx-rs/src/guest/mod.rs index c065259c8..0640ca4ae 100644 --- a/nvx-rs/src/guest/mod.rs +++ b/nvx-rs/src/guest/mod.rs @@ -13,8 +13,8 @@ //! nothing mounted from the devices that its clones replace, and through which each clone that a //! restore of the snapshot starts repairs its restore: it activates the restore's memory target, //! reseeds the kernel's random number generator, and finishes the restore, with its -//! acknowledgement, and then takes its own configuration from the host, in one Activate on a -//! restricted control session. +//! acknowledgement, and then takes its own configuration from the activation record that its +//! restore packet carried, and reports its readiness on its control session. //! //! The guest's space, the root that an init assembles and what it mounts in it, is each init's //! own. NVX's own space is internal reference code, which NVX's guest programs use: it is hidden diff --git a/nvx-rs/src/guest/time/api.rs b/nvx-rs/src/guest/time/api.rs index 976c33763..dabc230cf 100644 --- a/nvx-rs/src/guest/time/api.rs +++ b/nvx-rs/src/guest/time/api.rs @@ -663,7 +663,8 @@ impl Restore { /// Returns the activation record that the restore packet ends with: the bytes, 1 to /// [`MAX_RECORD_BYTES`](crate::activation::MAX_RECORD_BYTES) of them, that the host handed a /// platform clone with its restore, which the time ABI does not decode, or `None` for a - /// restore that handed over none. A clone decodes them with + /// restore that handed over none. A clone's repair hands them to the repaired clone, whose + /// activation decodes them with /// [`ActivationRecord::decode`](crate::activation::ActivationRecord::decode). pub fn activation_record(&self) -> Option<&[u8]> { self.activation.as_deref() diff --git a/nvx-rs/src/host/template/clone/failure.rs b/nvx-rs/src/host/template/clone/failure.rs index f2df25512..a1672f877 100644 --- a/nvx-rs/src/host/template/clone/failure.rs +++ b/nvx-rs/src/host/template/clone/failure.rs @@ -27,7 +27,7 @@ use crate::protocol::status; /// /// | Evidence | Outcome | Quarantine | /// | --- | --- | --- | -/// | An authorization that the image's binding denies; a stale image binding; invalid targets, format, settings, Activate, or mappings; an image that the template can't serve; a scratch that isn't ext4 or isn't fresh; a root that does not assemble from them | [`Request`](Outcome::Request) | No | +/// | An authorization that the image's binding denies; a stale image binding; invalid targets, format, settings, activation record, or mappings; an image that the template can't serve; a scratch that isn't ext4 or isn't fresh; a root that does not assemble from them | [`Request`](Outcome::Request) | No | /// | A supplied file with the wrong identity, length, or geometry | [`Request`](Outcome::Request), or the host's [`Staging`](Outcome::Staging) once the caller proves it | No | /// | An endpoint or forward that cannot be bound; resource exhaustion; a destination that cannot restore the template, such as an incompatible CPU or TSC; another runtime; another failure of the host's own operations | [`Host`](Outcome::Host) | No | /// | A lost status; a timeout; a reset; a forced kill; a broken protocol | [`Unknown`](Outcome::Unknown): the host, the protocol, or neither | No | @@ -95,9 +95,10 @@ pub enum Evidence { /// apply, as its [`ACTIVATION_WORKLOAD`](crate::protocol::status::ACTIVATION_WORKLOAD) and /// [`ACTIVATION_NETWORK`](crate::protocol::status::ACTIVATION_NETWORK) tell. InvalidSettings, - /// The request's Activate is invalid: settings that no activation record holds, or a record - /// that the guest refused, as its - /// [`ACTIVATION_RECORD`](crate::protocol::status::ACTIVATION_RECORD) tells. + /// The request's activation record is invalid: settings that no activation record holds, or + /// a record that the guest refused, as its + /// [`ACTIVATION_RECORD`](crate::protocol::status::ACTIVATION_RECORD) tells. Its name, which + /// records spell `invalid-activate`, predates the record's delivery in the restore packet. InvalidActivate, /// The request's mappings are invalid: an export that no clone takes, such as one that /// overlaps the template store; an export whose read-only mappings the host cannot keep @@ -381,7 +382,8 @@ pub enum Progress { /// A session of the adapter's framing attached: the guest's repair acknowledged the restore, /// before which OpenVMM reads none of the host's input. Acknowledged, - /// The Activate of the clone's record was sent. + /// The host asked for the clone's Ready, with which the clone answers once it activated with + /// the record of its restore packet. Activating, /// The clone's Ready was checked: the clone is activated, with the record's transaction. A /// clone with an export gets its mapping table at this step, before the start. diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs index c484c11da..bdb0faf5a 100644 --- a/nvx-rs/src/host/template/clone/launch.rs +++ b/nvx-rs/src/host/template/clone/launch.rs @@ -100,8 +100,9 @@ impl ClonePlan { /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox's /// directory, in place of one that an earlier launch left, before the VMM starts: the /// record, with the transaction ID of its restore section, is the one that the clone - /// activates with, which the caller, or a caller that takes the sandbox over after a - /// crash, reads there again and resends unchanged. + /// activates with, which OpenVMM reads there with `--restore-activation-record` before any + /// restored vCPU runs, and seals in the clone's restore packet, and which a caller that + /// takes the sandbox over after a crash reads there again, to check the clone's Ready. /// 3. Listens at the endpoint of the VMM's restore-ready event, which OpenVMM connects to once /// it has prepared the restore, before the restored VM runs, and which must exist before /// the VMM starts: on Linux, the socket @@ -153,7 +154,9 @@ impl ClonePlan { /// generation's snapshot, [`ClonePlan::snapshot`], for a template that records its processors /// online at boot, [`TemplateKey::boot_processors`], `--restore-processors` with the processor /// target, which brings online those of the target beyond them, `--restore-memory` with the - /// memory target, + /// memory target, `--restore-activation-record` with the sandbox's activation record, which + /// OpenVMM hands the clone's guest at the end of its restore packet, gating the host's input + /// until the guest acknowledged the restore, /// `--restore-ready-path` with the endpoint of the restore-ready event, the boot console: one /// that discards its output, as cold boots have, `--virtio-console none`, or for a plan that /// asks for one that the caller reads, a listener at its endpoint, @@ -388,6 +391,7 @@ impl ClonePlan { snapshot, processors, memory, + activation: &record, ready: &ready, console: console.as_deref(), report: locked.outcome_path(), @@ -450,7 +454,6 @@ impl ClonePlan { console, image, activation, - record, capability, tracker, table, @@ -709,6 +712,9 @@ struct Restore<'a> { processors: u32, /// The memory target, in bytes. memory: u64, + /// The path of the sandbox's activation record, which OpenVMM hands the clone's guest in its + /// restore packet. + activation: &'a Path, /// The endpoint of the restore-ready event. ready: &'a str, /// The endpoint at which the boot console that the caller reads listens, or `None` for a boot @@ -750,6 +756,8 @@ impl Restore<'_> { options.extend([ OsStr::new("--restore-memory"), OsStr::new(&memory), + OsStr::new("--restore-activation-record"), + self.activation.as_os_str(), OsStr::new("--restore-ready-path"), OsStr::new(self.ready), OsStr::new("--virtio-console"), @@ -793,8 +801,6 @@ pub struct Restoring<'s> { pub(super) image: CheckedDistro, /// The clone's activation record. pub(super) activation: ActivationRecord, - /// The path of the activation record in the sandbox, which the launch wrote. - pub(super) record: PathBuf, /// The capability of the launch. pub(super) capability: [u8; CAPABILITY_LEN], /// How far the clone got, and how its failures are recorded. diff --git a/nvx-rs/src/host/template/clone/reattach.rs b/nvx-rs/src/host/template/clone/reattach.rs index 61aa70aeb..06bf0b154 100644 --- a/nvx-rs/src/host/template/clone/reattach.rs +++ b/nvx-rs/src/host/template/clone/reattach.rs @@ -26,10 +26,9 @@ use crate::host::template::Lease; #[derive(Debug)] #[non_exhaustive] pub enum ReattachedClone { - /// The clone's VMM runs under the caller's runtime identity, and the clone answered the - /// Activate of its record, byte for byte, in a new session of the adapter's framing, with the - /// Ready of the record's activation: the caller holds the VM, and the session, in which it - /// drives the clone. + /// The clone's VMM runs under the caller's runtime identity, and the clone answered the request + /// for its Ready in a new session of the adapter's framing with the Ready of the activation of + /// its record: the caller holds the VM, and the session, in which it drives the clone. Live { /// The clone's VM. vmm: Box, @@ -77,15 +76,16 @@ pub enum ReattachedClone { /// /// - A VMM that runs under `runtime`, whose launch is the one that the clone's lease reserves, is /// the clone's, which the reattach checks in a new session, as the restore sequence of the -/// clone's launch does, [`Restoring::activate`](super::Restoring::activate), sending the -/// sandbox's activation record, byte for byte, which must still be the canonical encoding of the -/// record of a restore: the clone answers with the Ready that it answered once, and the reattach -/// checks the transaction ID that it reports, which [`ReattachedClone::Live`] returns with the -/// session. The clone's launch committed once the clone applied its mapping table, if it has one, -/// and started its sandbox, so the reattach sends neither again, and no launch that it ends gets -/// either. A check that fails, by the evidence that the restore sequence gives it, ends the VMM -/// through its verified identity, as [`Vmm::terminate`] does within `bound`, and the clone fails -/// at [`Stage::Reattach`]. +/// clone's launch does, [`Restoring::activate`](super::Restoring::activate): it asks for the +/// clone's Ready, which reports the transaction of the sandbox's activation record, which must +/// still be the canonical encoding of the record of a restore, the record of the clone's +/// restore packet: the clone answers with the Ready that it answers in every session, and the +/// reattach checks the transaction ID that it reports, which [`ReattachedClone::Live`] returns +/// with the session. The clone's launch committed once the clone applied its mapping table, if +/// it has one, and started its sandbox, so the reattach sends neither again, and no launch that +/// it ends gets either. A check that fails, by the evidence that the restore sequence gives it, +/// ends the VMM through its verified identity, as [`Vmm::terminate`] does within `bound`, and +/// the clone fails at [`Stage::Reattach`]. /// - A VMM that runs under another runtime identity, which might not speak the caller's /// protocol, is ended the same way, without a word to its guest, and the clone fails at /// [`Stage::Reattach`] with [`Evidence::RuntimeMismatch`], and @@ -263,7 +263,7 @@ pub(super) fn reattach_clone_syncing( let transaction = activation .as_ref() .ok() - .and_then(|(_, record)| record.restore().map(|restore| restore.transaction)); + .and_then(|record| record.restore().map(|restore| restore.transaction)); let recorder = Recorder::new(lease.clone(), transaction, locked).uncertain(uncertain); let fresh = earlier.is_none(); // A record of an earlier failure may be in place without being durable, as a write whose sync @@ -474,13 +474,14 @@ fn exit_fault( } /// Reads the clone's activation record at `path`, which must be the canonical encoding of the -/// record of a restore, as the clone's launch wrote it, and returns its bytes and the record, or -/// the fault, the host's, of one that is not. +/// record of a restore, as the clone's launch wrote it, and from which OpenVMM read the record of +/// the clone's restore packet, and returns the record, or the fault, the host's, of one that is +/// not. /// /// It compares nothing of the record with the clone's lease: the reattach matches the clone's /// records with the lease as [`CloneRecords`](super::CloneRecords) does, through /// [`read_failure_of`], which refuses the record of the failure of another template's clone. -fn read_activation(path: &Path) -> std::result::Result<(Vec, ActivationRecord), Fault> { +fn read_activation(path: &Path) -> std::result::Result { let bytes = open_regular(path) .and_then(|file| read_at_most(file, MAX_RECORD_BYTES as u64, "an activation record")) .map_err(|error| { @@ -513,11 +514,11 @@ fn read_activation(path: &Path) -> std::result::Result<(Vec, ActivationRecor malformed("is not the record of a restore"), )); } - Ok((bytes, record)) + Ok(record) } /// Checks the clone of `lease`, whose VM `vmm` runs in the sandbox of `locked`, in a new session -/// of `control`, with the sandbox's capability and its activation record, `activation`, by +/// of `control`, with the sandbox's capability, against its activation record, `record`, by /// `deadline`, as the reattach that `started` describes, and returns the session in which the /// clone answered. fn check_activation( @@ -525,7 +526,7 @@ fn check_activation( lease: &Lease, vmm: &Vmm, control: &mut C, - (encoded, record): (Vec, ActivationRecord), + record: ActivationRecord, deadline: Instant, started: Instant, ) -> std::result::Result { @@ -542,12 +543,9 @@ fn check_activation( )); } let capability = locked.read_capability().on_host()?; - let path = locked.clone_activation_path(); let session = ActivationSession { capability: &capability, - record: &path, expected: ReadyActivation::of(&record), - encoded, deadline, timeout: deadline.saturating_duration_since(started), since: "its reattach", diff --git a/nvx-rs/src/host/template/clone/records.rs b/nvx-rs/src/host/template/clone/records.rs index f25f6943e..46a042a4a 100644 --- a/nvx-rs/src/host/template/clone/records.rs +++ b/nvx-rs/src/host/template/clone/records.rs @@ -16,9 +16,9 @@ //! written during another boot, as by one that is damaged or missing: a workload may have run in //! the clone. The launch appends the spawn before it returns, although the launch's journal //! records it too, since a teardown of the launch removes the journal. A step that the next step -//! follows at once, the acknowledgment, which the Activate follows, is appended with that next -//! step, in its one write, so that the log takes a write for the spawn and for each step from the -//! restore-ready event on. A reader ignores a last line that a crash cut short. A sandbox +//! follows at once, the acknowledgment, which the request for Ready follows, is appended with that +//! next step, in its one write, so that the log takes a write for the spawn and for each step from +//! the restore-ready event on. A reader ignores a last line that a crash cut short. A sandbox //! that boots cold writes no log. //! //! # The failure record diff --git a/nvx-rs/src/host/template/clone/restore.rs b/nvx-rs/src/host/template/clone/restore.rs index fa7cdf421..1e82efd67 100644 --- a/nvx-rs/src/host/template/clone/restore.rs +++ b/nvx-rs/src/host/template/clone/restore.rs @@ -7,21 +7,19 @@ use std::fmt; use std::io; -use std::path::Path; use std::thread; use std::time::{Duration, Instant}; use super::failure::{Attribute, CloneFailure, Evidence, Fault, Progress, Stage, host_evidence}; use super::launch::Restoring; use super::records::Tracker; -use crate::activation::{ActivationRecord, MAX_RECORD_BYTES, ReadyActivation}; +use crate::activation::{ActivationRecord, ReadyActivation}; use crate::error::{Error, Result}; use crate::host::platform::{ReadyListener, Transport}; use crate::host::process::ProcessRef; use crate::host::sandbox::distro::CheckedDistro; use crate::host::session::{self, SessionError}; use crate::host::store::Recovery; -use crate::host::store::regular::{open_regular, read_at_most}; use crate::host::supervisor::{Starting, Vmm}; use crate::protocol::{CAPABILITY_LEN, HostMap}; @@ -45,8 +43,8 @@ pub const RESTORE_TO_READY_TIMEOUT: Duration = Duration::from_secs(60); /// It is the test adapter's deadline of a session's answers. pub const READY_TO_START_TIMEOUT: Duration = Duration::from_secs(30); -/// The most sessions in which [`Restoring::activate`] sends a clone's Activate: its first session, -/// and those that replace a session that the adapter lost. +/// The most sessions in which [`Restoring::activate`] asks for a clone's Ready: its first +/// session, and those that replace a session that the adapter lost. pub const MAX_ACTIVATION_SESSIONS: usize = 3; /// How often the wait for the restore-ready event checks whether the VMM still runs. @@ -66,7 +64,7 @@ const SESSION_RETRY_INTERVAL: Duration = Duration::from_millis(50); pub struct RestoreDeadlines { /// The restore-to-Ready deadline: how long after the clone's launch its Ready may come, which /// bounds the wait for the restore-ready event, the connections to the clone's control - /// endpoint, the adapter's attachments, and the Activates. + /// endpoint, the adapter's attachments, and the waits for Ready. pub ready: Duration, /// The Ready-to-Start deadline: how long after the check of the clone's Ready its sandbox may /// start. @@ -104,15 +102,20 @@ pub trait CloneControl { deadline: Instant, ) -> Result; - /// Sends `record`, the clone's activation record in its canonical encoding, in the framing's - /// Activate, as the first request of `session`, and waits until `deadline` for the clone's - /// Ready. Returns the activation that Ready reports, or `None` for a Ready that reports - /// none, once the adapter checked its own identities in Ready, such as its build, and - /// refused a Ready that does not match them. - fn activate( + /// Asks for the clone's Ready in the framing's request for it, as the first request of + /// `session`, and waits until `deadline` for the clone's Ready, which the clone sends once it + /// activated with the record of its restore packet, in whichever session asks. Returns the + /// activation that Ready reports, or `None` for a Ready that reports none, once the adapter + /// checked its own identities in Ready, such as its build, and refused a Ready that does not + /// match them. + /// + /// The host asks, rather than the clone sending Ready unasked, since OpenVMM's broker refuses + /// the guest's records while no host is attached, which the guest cannot tell: the request + /// carries nothing, as the host asks a guest agent for its readiness with NVXC's `PING`, since + /// the record reached the clone with its restore. + fn ready( &mut self, session: &mut Self::Session, - record: &[u8], deadline: Instant, ) -> Result, Self::Error>; @@ -137,11 +140,11 @@ pub trait CloneControl { /// table if it has one: sends the start, and waits until `deadline` for its confirmation. fn start(&mut self, session: &mut Self::Session, deadline: Instant) -> Result<(), Self::Error>; - /// Returns whether `error`, with which [`attach`](Self::attach) or - /// [`activate`](Self::activate) failed, lost the session, as a reset of the session, the end - /// of its stream, or a failure of its transport do, rather than another failure, such as the - /// guest's refusal or the passing of the deadline: a clone that answers no more in a lost - /// session may still answer in another. + /// Returns whether `error`, with which [`attach`](Self::attach) or [`ready`](Self::ready) + /// failed, lost the session, as a reset of the session, the end of its stream, or a failure of + /// its transport do, rather than another failure, such as the guest's refusal or the passing + /// of the deadline: a clone that answers no more in a lost session may still answer in + /// another. fn lost(&self, error: &Self::Error) -> bool; /// Returns whether `error`, with which [`map`](Self::map) failed, is the guest's refusal of @@ -172,20 +175,20 @@ impl<'s> Restoring<'s> { /// event once the restored VM has started, after which, on Windows, it waits until the /// host has read the event, and then closes its end, which carries nothing else. The wait /// refuses a connection of another process than the VMM, and bytes that differ from the - /// event, reads nothing past the event, and ends once the VMM has exited. - /// 2. Reads the clone's activation record again where the launch wrote it, - /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox, - /// which must still be, byte for byte, the record of the launch, - /// [`activation`](Self::activation), in its canonical encoding: the only record that the - /// clone's restore sends. - /// 3. Connects to the clone's control endpoint, [`Starting::endpoint`], which the VMM serves, + /// event, reads nothing past the event, and ends once the VMM has exited. By then, the + /// clone holds its activation record, [`activation`](Self::activation), which OpenVMM read + /// from the sandbox's [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) + /// at the launch and sealed in the restore packet before the first restored vCPU ran, and + /// activates with it, whatever the host does. + /// 2. Connects to the clone's control endpoint, [`Starting::endpoint`], which the VMM serves, /// as [`session::connect`] does, only after the event: the VMM's control console bounds the /// host's authentication from its connection on, which the restore may hold up until its /// event. - /// 4. Attaches the adapter's session over the connection with the launch's capability, - /// [`capability`](Self::capability), which the VMM authenticates, and sends the record in - /// the adapter's Activate, as the session's first request. - /// 5. Checks the activation that the clone's Ready reports, after the adapter checked its own + /// 3. Attaches the adapter's session over the connection with the launch's capability, + /// [`capability`](Self::capability), which the VMM authenticates, and asks for the clone's + /// Ready in the adapter's request for it, as the session's first request, + /// [`CloneControl::ready`]. + /// 4. Checks the activation that the clone's Ready reports, after the adapter checked its own /// identities in it: the transaction ID of the record's restore section, as /// [`ReadyActivation::of`] returns it. /// @@ -193,7 +196,7 @@ impl<'s> Restoring<'s> { /// [`CloneRecords`](super::CloneRecords) describes: the event; its acknowledgment, which the /// attachment of a session shows, since OpenVMM reads none of the host's input, its /// authentication included, until the guest's repair acknowledged a restore that gates the - /// input; the Activate; and the Ready. + /// input; the request for Ready; and the Ready. /// /// Returns the [`Activated`] clone, whose Ready-to-Start deadline, `deadlines.start`, starts /// at the check of its Ready, and whose sandbox [`Activated::start`] starts, once the clone @@ -203,18 +206,15 @@ impl<'s> Restoring<'s> { /// /// # Retries /// - /// A clone answers an Activate of its record, byte for byte, with the same Ready, without - /// activating again, also in a later session once it is ready, and refuses any other Activate - /// for good, as does the reset of its session before it is ready. So once the adapter lost the - /// session, as [`CloneControl::lost`] tells, while it attached it or awaited the clone's - /// Ready, the sequence waits briefly, reads the record again, connects again, and attaches a - /// new session, in which it sends the record again, byte for byte, and checks the Ready again: - /// a clone that was ready answers it, and a clone that took the reset of the lost session - /// before then has ended, so that the connection fails once its VMM has exited. It does so - /// until the restore-to-Ready deadline, which no retry extends, in at most - /// [`MAX_ACTIVATION_SESSIONS`] sessions in all, and never after another failure: a refusal, a - /// Ready that does not match the record, a record that changed, and the passing of the - /// deadline are final. + /// A clone activates with the record of its restore packet, which no session changes, and + /// answers the request for its Ready in every session, once it is ready, with the same Ready, + /// whatever resets of its session came before. So once the adapter lost the session, as + /// [`CloneControl::lost`] tells, while it attached it or awaited the clone's Ready, the + /// sequence waits briefly, connects again, attaches a new session, in which it asks for Ready + /// again, and checks the Ready again: a clone that ended meanwhile fails the connection once + /// its VMM has exited. It does so until the restore-to-Ready deadline, which no retry extends, + /// in at most [`MAX_ACTIVATION_SESSIONS`] sessions in all, and never after another failure: a + /// refusal, a Ready that does not match the record, and the passing of the deadline are final. /// /// # Failures /// @@ -243,10 +243,9 @@ impl<'s> Restoring<'s> { /// adapter's framing, and a Ready that reports another activation than the record's, or /// none, [`Evidence::Protocol`], and for a session that the adapter lost in the last of the /// sessions, [`Evidence::Reset`]; - /// - [`ErrorKind::Io`](crate::ErrorKind::Io) if the endpoints fail otherwise, if the sandbox's - /// activation record cannot be read, or is no longer the launch's, and if the progress log - /// cannot be appended to, the host's evidence, as [`ClonePlan::new`](super::ClonePlan::new) - /// describes it; + /// - [`ErrorKind::Io`](crate::ErrorKind::Io) if the endpoints fail otherwise, and if the + /// progress log cannot be appended to, the host's evidence, as + /// [`ClonePlan::new`](super::ClonePlan::new) describes it; /// - [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a deadline beyond the /// host's clock, [`Evidence::HostFailure`]. /// @@ -267,7 +266,6 @@ impl<'s> Restoring<'s> { console: _, image, activation, - record, capability, mut tracker, table, @@ -286,8 +284,6 @@ impl<'s> Restoring<'s> { } let activation_session = ActivationSession { capability: &capability, - record: &record, - encoded: activation.encode(), expected: ReadyActivation::of(&activation), deadline, timeout: deadlines.ready, @@ -319,9 +315,10 @@ impl<'s> Restoring<'s> { } /// A clone that its restore sequence activated, as [`Restoring::activate`] returns it: its guest -/// answered the Activate of its record with a Ready that reports the record's activation, and -/// awaits its mapping table, if it has an export, and the start of its sandbox, which -/// [`start`](Self::start) sends, by the Ready-to-Start deadline, [`deadline`](Self::deadline). +/// answered the request for its Ready with a Ready that reports the activation of the record of +/// its restore packet, and awaits its mapping table, if it has an export, and the start of its +/// sandbox, which [`start`](Self::start) sends, by the Ready-to-Start deadline, +/// [`deadline`](Self::deadline). /// /// It holds the adapter's session in which the clone answered, and the launch, uncommitted, /// through which it borrows the sandbox's locked store exclusively, as [`Starting`] does. @@ -329,7 +326,7 @@ impl<'s> Restoring<'s> { /// of the caller leaves the launch's journal instead, for /// [`LockedStore::recover`](crate::host::store::LockedStore::recover). pub struct Activated<'s, S> { - /// The adapter's session, in which the clone answered its Activate. + /// The adapter's session, in which the clone answered with its Ready. session: S, /// The launch, whose journal records the VMM. starting: Starting<'s>, @@ -352,7 +349,7 @@ impl<'s, S> Activated<'s, S> { &self.starting } - /// Returns the adapter's session, in which the clone answered its Activate. + /// Returns the adapter's session, in which the clone answered with its Ready. pub fn session(&mut self) -> &mut S { &mut self.session } @@ -673,16 +670,12 @@ impl fmt::Debug for Started<'_, S> { } } -/// What every session of a clone's activation takes: the launch's capability, the sandbox's -/// activation record, and the activation that its Ready must report, by the restore-to-Ready +/// What every session of a clone's activation takes: the launch's capability, and the activation +/// that its Ready must report, that of the sandbox's activation record, by the restore-to-Ready /// deadline, or by the deadline of a reattach. pub(super) struct ActivationSession<'a> { /// The capability of the launch. pub(super) capability: &'a [u8; CAPABILITY_LEN], - /// The path of the activation record in the sandbox. - pub(super) record: &'a Path, - /// The canonical encoding of the launch's activation record. - pub(super) encoded: Vec, /// The activation that the clone's Ready must report. pub(super) expected: ReadyActivation, /// The deadline by which the clone must be ready. @@ -702,11 +695,11 @@ enum Attempt { } impl ActivationSession<'_> { - /// Activates the clone whose VMM, `reference`, serves its control endpoint at `endpoint`, in - /// sessions of `control`, as [`Restoring::activate`] describes: in a new session after a - /// session that the adapter lost, in [`MAX_ACTIVATION_SESSIONS`] at most, and returns the - /// session in which the clone answered with its Ready. A launch records the clone's progress - /// in its `tracker`. + /// Asks for the Ready of the clone whose VMM, `reference`, serves its control endpoint at + /// `endpoint`, in sessions of `control`, as [`Restoring::activate`] describes: in a new session + /// after a session that the adapter lost, in [`MAX_ACTIVATION_SESSIONS`] at most, and returns + /// the session in which the clone answered with its Ready. A launch records the clone's + /// progress in its `tracker`. pub(super) fn activate( &self, control: &mut C, @@ -728,7 +721,7 @@ impl ActivationSession<'_> { } } - /// Activates the clone in a new session of `control`, as [`activate`](Self::activate) + /// Asks for the clone's Ready in a new session of `control`, as [`activate`](Self::activate) /// describes, records its progress in `tracker`, if it has one, and returns the session once /// the clone is ready. fn run( @@ -738,23 +731,22 @@ impl ActivationSession<'_> { reference: &ProcessRef, mut tracker: Option<&mut Tracker>, ) -> std::result::Result { - let record = self.read_record().map_err(Attempt::Failed)?; let transport = self.connect(endpoint, reference).map_err(Attempt::Failed)?; let attached = control.attach(transport, self.capability, self.deadline); let mut session = attached.map_err(|error| self.failed(control, error, "attach its control session"))?; if let Some(tracker) = &mut tracker { // OpenVMM authenticated the session, which it does only once the guest acknowledged - // a restore that gates the host's input. The Activate follows at once, with which - // the progress log records the acknowledgment. + // a restore that gates the host's input. The request for Ready follows at once, with + // which the progress log records the acknowledgment. tracker.advance(Progress::Acknowledged); tracker .reach(Progress::Activating) .map_err(Attempt::Failed)?; } let reported = control - .activate(&mut session, &record, self.deadline) - .map_err(|error| self.failed(control, error, "activate it"))?; + .ready(&mut session, self.deadline) + .map_err(|error| self.failed(control, error, "get its Ready"))?; self.check(reported).map_err(Attempt::Failed)?; if let Some(tracker) = &mut tracker { tracker.reach(Progress::Ready).map_err(Attempt::Failed)?; @@ -794,34 +786,6 @@ impl ActivationSession<'_> { ) } - /// Reads the activation record in the sandbox, which must be, byte for byte, the encoding of - /// the launch's record, and returns it. - fn read_record(&self) -> std::result::Result, Fault> { - let path = self.record; - let bytes = open_regular(path) - .and_then(|file| read_at_most(file, MAX_RECORD_BYTES as u64, "an activation record")) - .map_err(|error| { - Fault::host( - Error::io(format!( - "cannot read the clone's activation record {}", - path.display() - )) - .with_source(error), - ) - })?; - if bytes != self.encoded { - return Err(Fault::new( - Evidence::HostFailure, - Error::io(format!( - "{} no longer holds the activation record that the clone's launch wrote, the \ - only record that its restore sends", - path.display() - )), - )); - } - Ok(bytes) - } - /// Checks `reported`, the activation that the clone's Ready reports, against the record's. fn check(&self, reported: Option) -> std::result::Result<(), Fault> { check_ready(&self.expected, reported).attribute(Evidence::Protocol) diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs index fb3507c05..7f0b2773d 100644 --- a/nvx-rs/src/host/template/clone/tests/launch.rs +++ b/nvx-rs/src/host/template/clone/tests/launch.rs @@ -61,6 +61,10 @@ const EXIT_MARKER: &str = "exit"; /// File in which [`fake_vmm`] writes the arguments that a launch gave its VMM, as JSON, for the /// [`restore_vmm_role`] that runs instead. const ARGUMENTS: &str = "arguments.json"; +/// File in which [`fake_vmm`] keeps the activation record that a launch hands its VMM with +/// `--restore-activation-record`, as OpenVMM reads it before the restored VM runs: the record of +/// the clone's restore packet, with which the clone activates. +const PACKET: &str = "packet"; /// File in which a [`restore_vmm_role`] writes, once it has tried to send its restore-ready /// event, the capability that it read, in hexadecimal, and whether it sent the event. const STARTED: &str = "started"; @@ -278,8 +282,8 @@ pub(super) struct Made { /// Returns the seam of the command of a launch whose VMM is a [`restore_vmm_role`] in `vmm`: it /// keeps the command that the launch made in `made`, writes its arguments in [`ARGUMENTS`] in -/// `vmm`, and runs the role there instead, with the descriptors or handles that the command -/// passes. +/// `vmm`, and the activation record that they hand the VMM in [`PACKET`] there, as OpenVMM reads +/// it, and runs the role there instead, with the descriptors or handles that the command passes. pub(super) fn fake_vmm<'a>( vmm: &'a Path, made: &'a RefCell>, @@ -291,6 +295,9 @@ pub(super) fn fake_vmm<'a>( .map(|argument| argument.to_str().unwrap().to_owned()) .collect(); fs::write(vmm.join(ARGUMENTS), serde_json::to_vec(&arguments).unwrap()).unwrap(); + // OpenVMM reads the record before the restored VM runs, and seals it in the packet. + let record = argument(&arguments, "--restore-activation-record").unwrap(); + fs::write(vmm.join(PACKET), fs::read(record).unwrap()).unwrap(); // The VMM reports its outcome in the sandbox, wherever the VMM runs. let report = argument(&arguments, "--microvm-report").unwrap(); let sandbox = Path::new(&report).parent().unwrap(); @@ -337,6 +344,12 @@ impl VmmDir { self.dir.path() } + /// Returns the path of the file that holds the record of the clone's restore packet, which + /// [`fake_vmm`] wrote at the launch. + pub(super) fn packet(&self) -> PathBuf { + self.path().join(PACKET) + } + /// Waits until the VMM has tried to send its restore-ready event, and returns what it wrote /// in [`STARTED`]. pub(super) fn started(&self) -> String { @@ -854,6 +867,7 @@ fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() let (profile, processors) = (key.cpu_profile(), key.processors().to_string()); let listen = format!("listen={endpoint}"); let report = sandbox.join(OUTCOME_NAME); + let record = sandbox.join(CLONE_ACTIVATION_NAME); let expected = [ "--machine", "microvm", @@ -871,6 +885,8 @@ fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() snapshot.to_str().unwrap(), "--restore-memory", "512M", + "--restore-activation-record", + record.to_str().unwrap(), "--restore-ready-path", &ready, "--virtio-console", @@ -883,9 +899,11 @@ fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() ]; assert_eq!(made.arguments, expected); // The sandbox holds the clone's activation record, in its canonical encoding, which the - // launch keeps. - let recorded = fs::read(sandbox.join(CLONE_ACTIVATION_NAME)).unwrap(); + // launch keeps, and which the VMM reads, at the path that the arguments name, to hand it to + // the clone's guest in its restore packet. + let recorded = fs::read(&record).unwrap(); assert_eq!(recorded, activation.encode()); + assert_eq!(fs::read(vmm.packet()).unwrap(), recorded); assert_eq!(ActivationRecord::decode(&recorded).unwrap(), activation); assert_eq!(restoring.activation(), &activation); assert_eq!(restoring.capability(), &capability); @@ -2082,8 +2100,12 @@ fn launch_role() { if step == "launched" { crash(&step); } - // The restore sequence, through a framing that crashes within the Activate or the start. - let mut control = RoleControl { step: &step }; + // The restore sequence, through a framing that crashes as it awaits the clone's Ready or + // within the start. + let mut control = RoleControl { + step: &step, + packet: vmm.join(PACKET), + }; let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -2111,12 +2133,14 @@ fn launch_role() { panic!("the clone's launch did not crash at {step}"); } -/// The adapter's framing of a [`launch_role`], whose clone answers as the record asks, and which -/// crashes as it awaits the clone's Ready, at the step `activating`, or the start of its sandbox, -/// at the step `starting`. +/// The adapter's framing of a [`launch_role`], whose clone answers with the activation of the +/// record of its restore packet, and which crashes as it awaits the clone's Ready, at the step +/// `activating`, or the start of its sandbox, at the step `starting`. struct RoleControl<'a> { /// The step at which the role crashes. step: &'a str, + /// The file that holds the record of the clone's restore packet. + packet: PathBuf, } impl CloneControl for RoleControl<'_> { @@ -2132,16 +2156,15 @@ impl CloneControl for RoleControl<'_> { Ok(transport) } - fn activate( + fn ready( &mut self, _session: &mut Box, - record: &[u8], _deadline: Instant, ) -> io::Result> { if self.step == "activating" { crash(self.step); } - let record = ActivationRecord::decode(record).unwrap(); + let record = ActivationRecord::decode(&fs::read(&self.packet)?).unwrap(); Ok(Some(ReadyActivation::of(&record))) } @@ -2303,15 +2326,17 @@ impl Crashed { } } - /// Takes the clone of the sandbox of `locked` over, under `runtime`, through `control`, with + /// Takes the clone of the sandbox of `locked` over, under `runtime`, through `control`, whose + /// clone answers with the record of the restore packet that its VMM kept, with /// [`reattach_clone`], until the result is determinate, as a caller retries an indeterminate /// reattach, and returns it. - fn take_over( + fn take_over( &self, locked: &mut LockedStore, runtime: &RuntimeIdentity, - control: &mut C, - ) -> ReattachedClone { + control: &mut ReattachControl, + ) -> ReattachedClone> { + control.packet = Some(self.vmm.packet()); let deadline = Instant::now() + TIMEOUT; loop { let session = Instant::now() + TIMEOUT; @@ -2588,28 +2613,32 @@ fn a_restore_whose_caller_crashes_once_it_committed_keeps_its_lease_while_its_vm crashed.releases(&mut locked); } -/// The Ready of a clone that a [`ReattachControl`] answers, from the activation of the record that -/// the clone's Activate carries. +/// The Ready of a clone that a [`ReattachControl`] answers, from the activation of the record of +/// the clone's restore packet. type Answered = fn(ReadyActivation) -> Option; /// The adapter's framing of the sessions of a reattach: it attaches a session to each connection -/// to the clone's control endpoint, keeps the capability of each, and the record of each Activate, -/// which it answers as its `answer` says. +/// to the clone's control endpoint, keeps the capability of each, and counts the requests for +/// Ready, which it answers as its `answer` says, from the record of the clone's restore packet, +/// which the clone's VMM keeps in its `packet`. struct ReattachControl { /// The capabilities of the sessions, in order. capabilities: Vec<[u8; CAPABILITY_LEN]>, - /// The records of the Activates, in order. - records: Vec>, - /// How the clone answers an Activate. + /// The number of requests for Ready. + asked: usize, + /// The file that holds the record of the clone's restore packet, once the test names it. + packet: Option, + /// How the clone answers a request for Ready. answer: Answered, } impl ReattachControl { - /// Returns a framing whose clone answers each Activate as `answer` says. + /// Returns a framing whose clone answers each request for Ready as `answer` says. fn answering(answer: Answered) -> Self { Self { capabilities: Vec::new(), - records: Vec::new(), + asked: 0, + packet: None, answer, } } @@ -2629,14 +2658,17 @@ impl CloneControl for ReattachControl { Ok(transport) } - fn activate( + fn ready( &mut self, _session: &mut Box, - record: &[u8], _deadline: Instant, ) -> io::Result> { - self.records.push(record.to_vec()); - let record = ActivationRecord::decode(record).unwrap(); + self.asked += 1; + let packet = self + .packet + .as_ref() + .expect("the test names the clone's restore packet"); + let record = ActivationRecord::decode(&fs::read(packet)?).unwrap(); Ok((self.answer)(ReadyActivation::of(&record))) } @@ -2680,11 +2712,12 @@ fn a_clone_whose_caller_crashed_once_it_committed_is_taken_over_in_a_new_session ReattachedClone::Live { vmm, session } => (vmm, session), other => panic!("the reattach found {}", describe(&other)), }; - // The VM is the one that the launch committed, whose clone answered the Activate of the - // sandbox's record, byte for byte, in a session that the launch's capability authenticated, - // and runs on, with its lease. + // The VM is the one that the launch committed, whose clone answered the request for its Ready + // with the activation of the sandbox's record, the record of its restore packet, in a session + // that the launch's capability authenticated, and runs on, with its lease. assert_eq!(vmm.process().reference(), &reference); - assert_eq!(control.records, [recorded]); + assert_eq!(control.asked, 1); + assert_eq!(fs::read(crashed.vmm.packet()).unwrap(), recorded); let attached: Vec = control.capabilities.iter().map(|key| hex(key)).collect(); assert_eq!(attached, [capability]); drop(session); @@ -2738,7 +2771,7 @@ fn a_journal_that_a_clone_s_commit_left_is_no_failure_of_the_clone() { ); live = Some(vmm); } - assert_eq!(control.records.len(), 2); + assert_eq!(control.asked, 2); crashed.refuses_release(&mut locked); let vmm = live.unwrap(); assert_eq!( @@ -2781,7 +2814,7 @@ fn a_reattach_ends_a_clone_that_it_may_not_keep_through_its_verified_identity() Evidence::Protocol, terminated, ); - assert_eq!(control.records.len(), 1); + assert_eq!(control.asked, 1); assert!(locked.runtime().unwrap().is_none()); crashed.releases(&mut locked); } @@ -3908,6 +3941,7 @@ fn a_reattach_that_cannot_record_a_failure_ends_and_forgets_nothing_of_the_clone Some(other) => (other, &mut control), None => (&crashed.runtime, &mut refusing), }; + control.packet = Some(crashed.vmm.packet()); unrecorded(&unrecorded_take_over(&mut locked, runtime, control)); let record = locked .runtime() diff --git a/nvx-rs/src/host/template/clone/tests/restore.rs b/nvx-rs/src/host/template/clone/tests/restore.rs index e44b2fc03..7a9eb0c16 100644 --- a/nvx-rs/src/host/template/clone/tests/restore.rs +++ b/nvx-rs/src/host/template/clone/tests/restore.rs @@ -1,13 +1,14 @@ //! Tests of the restore sequence of clones, [`Restoring::activate`], [`Activated::start`], and //! [`Started::commit`], with a [`restore_vmm_role`](super::launch) as the clone's VMM, which //! sends its restore-ready event as its plan says, and a [`FakeControl`] in place of the adapter's -//! framing: the wait for the event, and the connection only once it came; the Activate of the -//! sandbox's record, byte for byte, as the first request of each session; the check of the -//! activation that Ready reports; the mapping table of a clone with an export, and the end of the -//! launch at a partial or failed table; the start; the sessions that replace those that the -//! adapter lost, after a reset of the session, say, and the clone that ended once it took that -//! reset before it was ready; and the deadlines. The crashes of the sequence's caller are among -//! the launch's tests. +//! framing, whose clone reports the activation of the record that its VMM read with +//! `--restore-activation-record` at the launch, the record of its restore packet: the wait for the +//! event, and the connection only once it came; the request for Ready as the first request of +//! each session; the check of the activation that Ready reports; the mapping table of a clone with +//! an export, and the end of the launch at a partial or failed table; the start; the sessions that +//! replace those that the adapter lost, after a reset of the session, say, and the clone whose VMM +//! ended meanwhile; and the deadlines. The crashes of the sequence's caller are among the launch's +//! tests. use std::cell::RefCell; use std::collections::VecDeque; @@ -35,8 +36,8 @@ const DELAY: Duration = Duration::from_millis(300); /// A short deadline, which a test expects to pass. const SHORT: Duration = Duration::from_secs(2); -/// The restore-to-Ready deadline of a clone that never answers its Activate, which leaves its VMM -/// time to start on a busy host. +/// The restore-to-Ready deadline of a clone that never answers the request for its Ready, which +/// leaves its VMM time to start on a busy host. const SILENT: Duration = Duration::from_secs(10); /// A call that the restore sequence made to a [`FakeControl`]. @@ -50,12 +51,8 @@ enum Call { at: Instant, deadline: Instant, }, - /// An Activate of the record in a session, by the deadline. - Activate { - session: usize, - record: Vec, - deadline: Instant, - }, + /// A request for the clone's Ready in a session, by the deadline. + Ready { session: usize, deadline: Instant }, /// A mapping table in a session, by the deadline, as the guest decodes the `MAPS` payloads in /// which the framing encodes it. Map { @@ -67,10 +64,10 @@ enum Call { Start { session: usize, deadline: Instant }, } -/// How a [`FakeControl`] answers an Activate or a start. +/// How a [`FakeControl`] answers a request for Ready or a start. enum Answer { - /// With the clone's Ready, which reports the activation of the record that the Activate - /// carries, or with the start's confirmation. + /// With the clone's Ready, which reports the activation of the record of its restore packet, + /// or with the start's confirmation. Done, /// With a Ready that reports this activation, or none. Report(Option), @@ -132,19 +129,22 @@ struct FakeSession { } /// The adapter's framing of the tests: it attaches a session to each connection that the sequence -/// makes, numbered from 1, answers each Activate as the next of its `activates` says, each mapping -/// table, which it encodes in `MAPS` payloads that its guest decodes, as the next of its `maps` -/// says, and each start as the next of its `starts` says, as [`Answer::Done`] does once they run -/// out, and records each call. Its `on_activate` runs at each Activate, with the session's number, -/// before the answer. +/// makes, numbered from 1, answers each request for Ready as the next of its `readies` says, with +/// the activation of the record that the clone's VMM read at the launch, its `packet`, each +/// mapping table, which it encodes in `MAPS` payloads that its guest decodes, as the next of its +/// `maps` says, and each start as the next of its `starts` says, as [`Answer::Done`] does once +/// they run out, and records each call. Its `on_ready` runs at each request for Ready, with the +/// session's number, before the answer. #[derive(Default)] struct FakeControl<'a> { /// The calls, in order. calls: Vec, /// The sessions that it attached. sessions: usize, - /// How it answers the next Activates. - activates: VecDeque, + /// The file in which the clone's VMM keeps the record of the clone's restore packet. + packet: PathBuf, + /// How it answers the next requests for Ready. + readies: VecDeque, /// How its guest answers the next mapping tables: how many of a table's mappings it applies, /// in order, at most all, and its answer. maps: VecDeque<(usize, Answer)>, @@ -152,44 +152,43 @@ struct FakeControl<'a> { applied: Vec, /// How it answers the next starts. starts: VecDeque, - /// What runs at each Activate. - on_activate: Option>, + /// What runs at each request for Ready. + on_ready: Option>, } impl<'a> FakeControl<'a> { - /// Returns a framing that answers the Activates as `activates` say. - fn activating(activates: impl IntoIterator) -> Self { + /// Returns a framing of the clone whose VMM runs in `vmm`, which answers every request as + /// [`Answer::Done`] does. + fn new(vmm: &VmmDir) -> Self { Self { - activates: activates.into_iter().collect(), + packet: vmm.packet(), ..Self::default() } } - /// Returns the records of the Activates, in order. - fn records(&self) -> Vec<&[u8]> { - self.calls - .iter() - .filter_map(|call| match call { - Call::Activate { record, .. } => Some(record.as_slice()), - _ => None, - }) - .collect() + /// Returns a framing of the clone whose VMM runs in `vmm`, which answers the requests for + /// Ready as `readies` say. + fn answering(vmm: &VmmDir, readies: impl IntoIterator) -> Self { + Self { + readies: readies.into_iter().collect(), + ..Self::new(vmm) + } } - /// Returns the deadlines of the attachments and Activates, in order. + /// Returns the deadlines of the attachments and the requests for Ready, in order. fn ready_deadlines(&self) -> Vec { self.calls .iter() .filter_map(|call| match call { - Call::Attach { deadline, .. } | Call::Activate { deadline, .. } => Some(*deadline), + Call::Attach { deadline, .. } | Call::Ready { deadline, .. } => Some(*deadline), Call::Map { .. } | Call::Start { .. } => None, }) .collect() } - /// Checks that each session of the framing attached, then sent an Activate first, in its + /// Checks that each session of the framing attached, then asked for Ready first, in its /// order, and that the mapping tables and the starts came last, in the last session. - fn assert_activates_first(&self) { + fn assert_ready_first(&self) { let mut session = 0; let mut pending = false; for call in &self.calls { @@ -202,7 +201,7 @@ impl<'a> FakeControl<'a> { assert_eq!(*number, session, "{:?}", self.calls); pending = true; } - Call::Activate { + Call::Ready { session: number, .. } => { assert!(pending && *number == session, "{:?}", self.calls); @@ -260,22 +259,23 @@ impl CloneControl for FakeControl<'_> { }) } - fn activate( + fn ready( &mut self, session: &mut FakeSession, - record: &[u8], deadline: Instant, ) -> Result, Failure> { - self.calls.push(Call::Activate { + self.calls.push(Call::Ready { session: session.number, - record: record.to_vec(), deadline, }); - if let Some(on_activate) = &mut self.on_activate { - on_activate(session.number); + if let Some(on_ready) = &mut self.on_ready { + on_ready(session.number); } - let ready = ReadyActivation::of(&ActivationRecord::decode(record).unwrap()); - let answer = self.activates.pop_front().unwrap_or(Answer::Done); + // The clone activated with the record of its restore packet, which its VMM read at the + // launch, whatever the sandbox holds since. + let packet = fs::read(&self.packet).unwrap(); + let ready = ReadyActivation::of(&ActivationRecord::decode(&packet).unwrap()); + let answer = self.readies.pop_front().unwrap_or(Answer::Done); answered(answer, Some(ready), deadline) } @@ -471,7 +471,7 @@ fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { let activation = restoring.activation().clone(); let capability = *restoring.capability(); let endpoint = restoring.starting().endpoint().to_owned(); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let deadlines = RestoreDeadlines::default(); assert_eq!(deadlines.ready, RESTORE_TO_READY_TIMEOUT); assert_eq!(deadlines.start, READY_TO_START_TIMEOUT); @@ -479,11 +479,12 @@ fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { let mut activated = restoring.activate(&mut control, deadlines).unwrap(); let ready = Instant::now(); // The host read the event, and only then connected, and attached one session with the - // launch's capability, whose first request was the Activate of the sandbox's record, byte for - // byte the launch's, by the restore-to-Ready deadline, which started with the launch. + // launch's capability, whose first request asked for the clone's Ready, by the + // restore-to-Ready deadline, which started with the launch. The clone's VMM got the sandbox's + // record, byte for byte the launch's, which it hands the clone in its restore packet. assert!(vmm.started().ends_with(" true")); assert!(vmm.flushed()); - control.assert_activates_first(); + control.assert_ready_first(); let [ Call::Attach { session: 1, @@ -491,10 +492,9 @@ fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { at, deadline, }, - Call::Activate { + Call::Ready { session: 1, - record: sent, - deadline: activated_by, + deadline: asked_by, }, ] = control.calls.as_slice() else { @@ -506,9 +506,9 @@ fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { "the host connected before the event" ); launched.assert_deadline(*deadline, RESTORE_TO_READY_TIMEOUT); - assert_eq!(activated_by, deadline); - assert_eq!(sent, &activation.encode()); - assert_eq!(&fs::read(&record).unwrap(), sent); + assert_eq!(asked_by, deadline); + assert_eq!(fs::read(&record).unwrap(), activation.encode()); + assert_eq!(fs::read(vmm.packet()).unwrap(), activation.encode()); // The Ready-to-Start deadline started at the check of the clone's Ready. let start = activated.deadline(); assert!( @@ -594,7 +594,7 @@ fn a_ready_that_does_not_report_the_record_s_activation_ends_the_launch() { let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); let reported = reported(other_transaction); - let mut control = FakeControl::activating([Answer::Report(reported)]); + let mut control = FakeControl::answering(&vmm, [Answer::Report(reported)]); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -613,25 +613,23 @@ fn a_ready_that_does_not_report_the_record_s_activation_ends_the_launch() { } #[test] -fn a_session_that_the_adapter_lost_is_replaced_by_one_that_resends_the_same_record_first() { +fn a_session_that_the_adapter_lost_is_replaced_by_one_that_asks_for_ready_first() { let (fixture, image) = fixture(); let mut acquired = fixture.acquire("sandbox"); - let record = acquired.locked.clone_activation_path(); let mut vmm = vmm_with("flush"); let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); // The adapter loses its first session as it awaits the clone's Ready, as a reset of the - // session does: the clone, which was ready, answers the Activate of a new session. - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Done]); + // session does: the clone, which activated with the record of its restore packet, answers the + // request for Ready of a new session. + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::LOST), Answer::Done]); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); - control.assert_activates_first(); + control.assert_ready_first(); assert_eq!(control.sessions, 2); - // Both Activates carry the sandbox's record, byte for byte, by the same deadline, which the - // retry did not extend. - let recorded = fs::read(&record).unwrap(); - assert_eq!(control.records(), [&recorded[..], &recorded[..]]); + // Both requests for Ready have the same deadline, which the retry did not extend. let deadlines = control.ready_deadlines(); + assert_eq!(deadlines.len(), 4); assert!(deadlines.iter().all(|deadline| *deadline == deadlines[0])); launched.assert_deadline(deadlines[0], RESTORE_TO_READY_TIMEOUT); // The sandbox starts in the new session. @@ -648,11 +646,12 @@ fn a_session_that_the_adapter_lost_is_replaced_by_one_that_resends_the_same_reco } #[test] -fn a_retry_never_sends_another_record_nor_takes_another_activation() { +fn a_retry_takes_the_activation_of_the_restore_packet_s_record_alone() { let (fixture, image) = fixture(); - // The sandbox's record changed once the first session was lost: the retry sends nothing, and - // connects no more. + // The sandbox's record changed once the first session was lost: the clone activated with the + // record that its VMM read at the launch, and its Ready, in the next session, reports the + // launch's transaction, which the host checks against the record of the launch. let mut acquired = fixture.acquire("changed"); let record = acquired.locked.clone_activation_path(); let mut vmm = vmm_with("flush"); @@ -668,24 +667,17 @@ fn a_retry_never_sends_another_record_nor_takes_another_activation() { ) .unwrap(); let path = record.clone(); - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| { + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::LOST)]); + control.on_ready = Some(Box::new(move |_| { fs::write(&path, other.encode()).unwrap(); })); - let error = restoring + let activated = restoring .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - assert!(error.message().contains("no longer holds"), "{error}"); - assert_eq!(control.sessions, 1); - assert_eq!(control.records(), [&original.encode()[..]]); - // The host's own record changed: the record of the failure names the launch's transaction. - let record = error.record(); - assert_eq!( - (record.evidence, record.outcome, record.stage), - (Evidence::HostFailure, Outcome::Host, Stage::Activation) - ); - assert_eq!(record.transaction, Some(restore.transaction)); + .unwrap(); + assert_eq!(control.sessions, 2); + assert_eq!(activated.activation(), &original); + assert_eq!(fs::read(vmm.packet()).unwrap(), original.encode()); + drop(activated); assert_launch_ended(&fixture, &mut acquired, &vmm); // The Ready of the retry reports another transaction. @@ -695,8 +687,10 @@ fn a_retry_never_sends_another_record_nor_takes_another_activation() { let diverged = ReadyActivation { transaction: Some(TransactionId::from_bytes([0x78; TransactionId::LEN])), }; - let mut control = - FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Report(Some(diverged))]); + let mut control = FakeControl::answering( + &vmm, + [Answer::Fail(Failure::LOST), Answer::Report(Some(diverged))], + ); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -721,12 +715,12 @@ fn only_lost_sessions_are_replaced_and_only_so_many_times() { let mut acquired = fixture.acquire("refused"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::activating([Answer::Fail(Failure::REFUSED)]); + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::REFUSED)]); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - assert_eq!(error.message(), "cannot activate it"); + assert_eq!(error.message(), "cannot get its Ready"); assert_eq!(control.sessions, 1); let ready = (Stage::Activation, Progress::Activating); assert_failed( @@ -743,13 +737,13 @@ fn only_lost_sessions_are_replaced_and_only_so_many_times() { let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); let lost = (0..MAX_ACTIVATION_SESSIONS).map(|_| Answer::Fail(Failure::LOST)); - let mut control = FakeControl::activating(lost); + let mut control = FakeControl::answering(&vmm, lost); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); assert!(error.message().contains("was lost"), "{error}"); - control.assert_activates_first(); + control.assert_ready_first(); assert_eq!(control.sessions, MAX_ACTIVATION_SESSIONS); assert_failed( &error, @@ -762,16 +756,16 @@ fn only_lost_sessions_are_replaced_and_only_so_many_times() { } #[test] -fn a_clone_that_took_the_reset_of_a_lost_session_before_its_ready_ends_the_retries() { +fn a_clone_whose_vmm_ended_with_a_lost_session_ends_the_retries() { let (fixture, image) = fixture(); let mut acquired = fixture.acquire("sandbox"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - // The reset of the lost session ends a clone that was not ready yet, and with it its VMM: the + // The clone's VMM ends with the lost session, as once its guest's activation failed: the // retry cannot connect any more. let ending = &vmm; - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| ending.end_and_wait())); + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::LOST)]); + control.on_ready = Some(Box::new(move |_| ending.end_and_wait())); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -802,7 +796,7 @@ fn the_restore_to_ready_deadline_runs_from_the_launch_through_every_step() { let mut acquired = fixture.acquire("no-event"); let mut vmm = vmm_with("none"); let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let error = restoring.activate(&mut control, deadlines).unwrap_err(); assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); assert!(launched.before.elapsed() >= SHORT); @@ -817,12 +811,12 @@ fn the_restore_to_ready_deadline_runs_from_the_launch_through_every_step() { ); assert_launch_ended(&fixture, &mut acquired, &vmm); - // A clone that never answers its Activate, after a late event: the attachment and the - // Activate have the deadline that started with the launch. + // A clone that never answers the request for its Ready, after a late event: the attachment + // and the request have the deadline that started with the launch. let mut acquired = fixture.acquire("no-ready"); let mut vmm = vmm_with(&format!("delay:{} flush", DELAY.as_millis())); let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::activating([Answer::Silent]); + let mut control = FakeControl::answering(&vmm, [Answer::Silent]); let deadlines = RestoreDeadlines { ready: SILENT, ..RestoreDeadlines::default() @@ -857,7 +851,7 @@ fn the_ready_to_start_deadline_runs_from_the_ready_and_a_start_is_never_resent() let mut acquired = fixture.acquire("silent"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); control.starts.push_back(Answer::Silent); let ready = Instant::now(); let activated = restoring.activate(&mut control, deadlines).unwrap(); @@ -887,7 +881,7 @@ fn the_ready_to_start_deadline_runs_from_the_ready_and_a_start_is_never_resent() let mut acquired = fixture.acquire("lost"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); control.starts.push_back(Answer::Fail(Failure::LOST)); let activated = restoring.activate(&mut control, deadlines).unwrap(); let error = activated.start(&mut control).unwrap_err(); @@ -977,7 +971,7 @@ fn a_vmm_that_exits_or_sends_anything_but_its_event_ends_the_launch() { let mut acquired = fixture.acquire(name); let mut vmm = vmm_with(plan); let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -1017,7 +1011,7 @@ fn a_vmm_that_exits_or_sends_anything_but_its_event_ends_the_launch() { let _intruder = std::os::unix::net::UnixStream::connect(&ready).unwrap(); #[cfg(windows)] let _intruder = fs::OpenOptions::new().write(true).open(&ready).unwrap(); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -1085,7 +1079,7 @@ fn a_status_with_which_the_guest_ended_the_clone_s_vm_attributes_its_failure() { let mut vmm = vmm_with(&format!("flush report:{report}")); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); let error = if at == starting { - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); control.starts.push_back(Answer::Fail(Failure::LOST)); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) @@ -1094,8 +1088,8 @@ fn a_status_with_which_the_guest_ended_the_clone_s_vm_attributes_its_failure() { activated.start(&mut control).unwrap_err() } else { let ending = &vmm; - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| ending.end_and_wait())); + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::LOST)]); + control.on_ready = Some(Box::new(move |_| ending.end_and_wait())); restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err() @@ -1136,7 +1130,7 @@ fn a_failure_is_recorded_before_its_teardown_and_again_once_the_teardown_ended() &mut vmm, ended_once_recorded, ); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -1178,7 +1172,7 @@ fn a_launch_that_cannot_be_ended_keeps_its_records_and_its_lease_until_a_reattac let mut acquired = fixture.acquire("sandbox"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch_ending(&fixture, &mut acquired, &image, &mut vmm, kept); - let mut control = FakeControl::activating([Answer::Fail(Failure::REFUSED)]); + let mut control = FakeControl::answering(&vmm, [Answer::Fail(Failure::REFUSED)]); let error = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap_err(); @@ -1197,7 +1191,7 @@ fn a_launch_that_cannot_be_ended_keeps_its_records_and_its_lease_until_a_reattac assert!(Process::open(&reference).unwrap().is_some()); // A reattach ends the launch through its VMM's verified identity: the record keeps the // failure's stage, evidence, and original outcome, with the teardown that resolved it. - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let reattached = reattach_clone( locked, &runtime_of(&fixture), @@ -1231,7 +1225,7 @@ fn a_caller_fails_a_ready_or_started_clone_with_its_own_evidence_as_the_sequence let mut acquired = fixture.acquire("ready"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1247,7 +1241,7 @@ fn a_caller_fails_a_ready_or_started_clone_with_its_own_evidence_as_the_sequence let mut acquired = fixture.acquire("started"); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1263,7 +1257,7 @@ fn a_caller_fails_a_ready_or_started_clone_with_its_own_evidence_as_the_sequence let mut acquired = fixture.acquire("repair"); let mut vmm = vmm_with("flush report:guest-exit:195"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1280,7 +1274,7 @@ fn a_caller_fails_a_ready_or_started_clone_with_its_own_evidence_as_the_sequence let mut acquired = fixture.acquire("defect"); let mut vmm = vmm_with("flush report:guest-exit:195"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1309,7 +1303,7 @@ fn a_failure_that_cannot_be_recorded_before_its_teardown_keeps_its_launch_for_a_ let path = acquired.locked.clone_failure_path(); let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1345,7 +1339,7 @@ fn a_failure_that_cannot_be_recorded_before_its_teardown_keeps_its_launch_for_a_ // Once the record can be written, a reattach, as after a crash of the caller, finds that the // clone failed, though not how, rather than a clone that ended. fs::remove_dir(&path).unwrap(); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let reattached = reattach_clone( &mut acquired.locked, &runtime_of(&fixture), @@ -1367,7 +1361,7 @@ fn a_diagnosis_too_long_for_a_record_keeps_its_ends_and_its_evidence_through_a_r let mut acquired = fixture.acquire("diagnosis"); let mut vmm = vmm_with("flush report:guest-exit:195"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1399,7 +1393,7 @@ fn a_diagnosis_too_long_for_a_record_keeps_its_ends_and_its_evidence_through_a_r assert_eq!(records.failure.as_ref(), Some(record)); // A reattach finds the failure as it was recorded, with the template's evidence, which // quarantines the generation once the sandbox's lock is released. - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let reattached = reattach_clone( &mut acquired.locked, &runtime_of(&fixture), @@ -1433,7 +1427,7 @@ fn a_clone_with_an_export_gets_its_mapping_table_once_ready_and_its_start_once_i &mut vmm, Hooks::LAUNCH.end, ); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1443,7 +1437,7 @@ fn a_clone_with_an_export_gets_its_mapping_table_once_ready_and_its_start_once_i // start follows once the guest applied every mapping, both by the Ready-to-Start deadline. // The table names each path by its child's index, as a cold launch's does, through the // `MAPS` payloads that carry it. - control.assert_activates_first(); + control.assert_ready_first(); assert_eq!(data_table().to_vec(), export.guest_table()); assert_eq!( control.calls[2..], @@ -1476,7 +1470,7 @@ fn a_clone_with_an_export_gets_its_mapping_table_once_ready_and_its_start_once_i let mut vmm = vmm_with("flush"); let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); assert!(restoring.table.is_empty()); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) .unwrap(); @@ -1560,7 +1554,7 @@ fn a_partial_or_failed_mapping_table_ends_the_launch_and_its_sandbox_never_start &mut vmm, Hooks::LAUNCH.end, ); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); let source = match &answer { Answer::Fail(failure) => failure.to_string(), _ => Failure::REFUSED.to_string(), @@ -1606,7 +1600,7 @@ fn a_partial_or_failed_mapping_table_ends_the_launch_and_its_sandbox_never_start &mut vmm, Hooks::LAUNCH.end, ); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); control.maps.push_back((0, Answer::Fail(Failure::TABLE))); let passed = RestoreDeadlines { start: Duration::ZERO, @@ -1639,7 +1633,7 @@ fn a_partial_or_failed_mapping_table_ends_the_launch_and_its_sandbox_never_start &mut vmm, Hooks::LAUNCH.end, ); - let mut control = FakeControl::default(); + let mut control = FakeControl::new(&vmm); control.maps.push_back((0, Answer::Fail(Failure::LOST))); let activated = restoring .activate(&mut control, RestoreDeadlines::default()) diff --git a/nvx-rs/src/host/template/key.rs b/nvx-rs/src/host/template/key.rs index 4f08e9d0c..758bbb0a0 100644 --- a/nvx-rs/src/host/template/key.rs +++ b/nvx-rs/src/host/template/key.rs @@ -36,7 +36,7 @@ const HOST_GENERATION: &str = "host"; /// The versions of the contracts of a template's clones that its runtime identity does not /// record: the snapshot format, the saved device state, the time ABI, the restore packet, and the -/// activation record. +/// activation contract. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] struct Contracts { /// [`contract::SNAPSHOT_FORMAT_VERSION`]. @@ -47,7 +47,7 @@ struct Contracts { time: u64, /// [`contract::RESTORE_PACKET_VERSION`]. restore_packet: u64, - /// [`activation::VERSION`], as a `u64`, as the other versions are. + /// [`activation::CONTRACT_VERSION`]. activation: u64, } @@ -58,8 +58,7 @@ impl Contracts { device_state: contract::DEVICE_STATE_SCHEMA_VERSION, time: contract::TIME_ABI_VERSION, restore_packet: contract::RESTORE_PACKET_VERSION, - // `u64::from` is not `const`, and widening a `u16` is lossless. - activation: activation::VERSION as u64, + activation: activation::CONTRACT_VERSION, }; /// Writes the versions. diff --git a/nvx-rs/src/host/template/key/tests.rs b/nvx-rs/src/host/template/key/tests.rs index 862fdfeb2..08eef2d82 100644 --- a/nvx-rs/src/host/template/key/tests.rs +++ b/nvx-rs/src/host/template/key/tests.rs @@ -285,6 +285,22 @@ fn each_contract_version_changes_the_key() { } } +#[test] +fn a_template_whose_clones_take_their_record_in_an_activate_speaks_no_current_contract() { + // A template that a build of activation contract 1 captured holds a guest that takes its + // clone's activation record in the adapter's Activate, which no host of this build sends, or + // that refuses the restore packet's record, so its key speaks no current contract, and no + // clone of this build restores it. The record's own schema keeps its version. + assert_eq!(activation::CONTRACT_VERSION, 2); + assert_eq!(activation::VERSION, 1); + let base = network_key(); + assert_eq!(base.contracts.activation, activation::CONTRACT_VERSION); + let mut earlier = base.clone(); + earlier.contracts.activation = 1; + assert!(!earlier.speaks_current_contracts()); + assert_ne!(earlier.identity(), base.identity()); +} + #[test] fn a_generations_own_metadata_leaves_the_key_as_it_is() { let changes: &[(&str, Change)] = &[ diff --git a/nvx-rs/src/host/template/mod.rs b/nvx-rs/src/host/template/mod.rs index 5476a0c2d..d38dc0d1a 100644 --- a/nvx-rs/src/host/template/mod.rs +++ b/nvx-rs/src/host/template/mod.rs @@ -43,16 +43,17 @@ //! or one at another guest address than the template's, before any VMM starts. //! - [`ClonePlan::launch`] launches the clone, as the launch that its lease reserves, once the //! objects of its export are still those that its plan found: OpenVMM restores the generation's -//! snapshot in place with the clone's devices and export, and the clone's activation record -//! waits in its sandbox, as a [`Restoring`] describes. +//! snapshot in place with the clone's devices and export, and hands the clone's activation +//! record, which the launch wrote in its sandbox, to the clone's guest in its restore packet, as +//! a [`Restoring`] describes. //! - [`Restoring::activate`] runs the clone's restore sequence through the adapter's framing of //! the clone's control session, a [`CloneControl`]: it waits for OpenVMM's restore-ready event, -//! then connects, sends the clone's activation record in the adapter's Activate, and checks the -//! activation that the clone's Ready reports, by a deadline that starts with the launch, and -//! resends the same record in a new session once the adapter lost one. [`Activated::start`] -//! then sends the clone's mapping table, if it has an export, and starts the clone's sandbox -//! once the guest applied every mapping, by a deadline that starts at its Ready, and -//! [`Started::commit`] commits the launch. +//! then connects, asks for the clone's Ready, which the clone answers once it activated with the +//! record of its restore packet, and checks the activation that the Ready reports, by a deadline +//! that starts with the launch, and asks again in a new session once the adapter lost one. +//! [`Activated::start`] then sends the clone's mapping table, if it has an export, and starts +//! the clone's sandbox once the guest applied every mapping, by a deadline that starts at its +//! Ready, and [`Started::commit`] commits the launch. //! - The plan, the launch, and the restore sequence fail with a [`CloneFailure`], whose //! [`Evidence`] attributes it to an [`Outcome`], at a [`Stage`], once the clone got as far as //! its [`Progress`], with the [`Teardown`] that followed: a failure from the launch on ends the diff --git a/nvx-rs/src/protocol/status.rs b/nvx-rs/src/protocol/status.rs index 45c54a57d..216ec51d0 100644 --- a/nvx-rs/src/protocol/status.rs +++ b/nvx-rs/src/protocol/status.rs @@ -12,9 +12,9 @@ //! status for each class. //! - [`ACTIVATION_DEADLINE`] to [`ACTIVATION_ORDER`], 50 to 58: the failures of a clone's //! activation, which follows its repair, one status reserved for each class: the deadline, the -//! host's session, the Activate's record and the resources that it expects, the activation of -//! the root, of the workload, of the host export, and of the network settings, and the guest's -//! own order. +//! host's session, the activation record of the restore packet and the resources that it +//! expects, the activation of the root, of the workload, of the host export, and of the network +//! settings, and the guest's own order. //! //! The guest ends the VM with a status after a diagnostic that names the failure, such as a //! violation event of the time ABI, or an error line. A status alone does not tell the cause, nor @@ -56,15 +56,15 @@ pub const REPAIR_FINISH: u8 = 45; /// The clone was not activated before its deadline, which starts at its repair's /// acknowledgement. pub const ACTIVATION_DEADLINE: u8 = 50; -/// The host broke the session of a repaired clone's activation: before the clone was ready, it -/// reset the session again, or sent a request before the activation, and, before or after, it -/// sent an Activate of another transaction, or one that changes the accepted record. +/// The host broke the session of a repaired clone's activation: it sent a request other than its +/// request for the clone's Ready before the clone was ready. pub const ACTIVATION_PROTOCOL: u8 = 51; -/// The Activate's record is refused: it does not decode, breaks a limit, or is a cold boot's, -/// without the restore section of a clone's. +/// The activation record of the clone's restore packet is refused: the packet carries none, or +/// the record does not decode, breaks a limit, or is a cold boot's, without the restore section +/// of a clone's. pub const ACTIVATION_RECORD: u8 = 52; -/// The Activate's record expects other resources than the repaired clone has: another number of -/// online CPUs, or another memory size. +/// The activation record of the clone's restore packet expects other resources than the repaired +/// clone has: another number of online CPUs, or another memory size. pub const ACTIVATION_RESOURCES: u8 = 53; /// The clone's root cannot be activated with the record's settings: its image does not assemble /// over its upper, a GPT image's partition table is refused, or its scratch is not fresh. @@ -78,8 +78,8 @@ pub const ACTIVATION_EXPORT: u8 = 56; /// endpoint, or the announcer of the workload's listeners, which its network publishes, cannot /// start, or ended. pub const ACTIVATION_NETWORK: u8 = 57; -/// The guest's own order misused the activation: it completed an activation that did not run, -/// or started the sandbox before the clone was ready. +/// The guest's own order misused the activation: it accepted the record again, completed an +/// activation that did not run, or started the sandbox before the clone was ready. pub const ACTIVATION_ORDER: u8 = 58; /// The time ABI's statuses, from [`TIME_CONFORMANCE`] to [`TIME_REPAIR`]. @@ -104,9 +104,9 @@ pub fn describe(status: u8) -> Option<&'static str> { REPAIR_RESEED => "a clone's reseed failed", REPAIR_FINISH => "a clone's restore finish failed", ACTIVATION_DEADLINE => "a clone was not activated before its deadline", - ACTIVATION_PROTOCOL => "the host broke a repaired clone's restricted session", - ACTIVATION_RECORD => "a clone's Activate record is refused", - ACTIVATION_RESOURCES => "a clone's Activate record expects other resources", + ACTIVATION_PROTOCOL => "the host broke a repaired clone's session before it was ready", + ACTIVATION_RECORD => "a clone's activation record is refused", + ACTIVATION_RESOURCES => "a clone's activation record expects other resources", ACTIVATION_ROOT => "a clone's root activation failed", ACTIVATION_WORKLOAD => "a clone's workload activation failed", ACTIVATION_EXPORT => "a clone's host export cannot be mounted", diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 3c7647123..6da1f6376 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -66,27 +66,26 @@ length, a little-endian `u32`, then the message's kind: | 24 | `MapFailed`: the guest refused the part, or failed to apply one of its mappings | how many mappings it applied, then the problem, at most 4 KiB | | 25 | `Start`: asks the guest to start the sandbox's workloads once it applied the mapping table, which it confirms with `Started` | none | | 26 | `Started`: the guest's confirmation of `Start` | none | -| 27 | `Activate`: the host's activation of a clone, which a restore of a [template's](#template-mode) snapshot started, and which the guest answers with `Ready` | a clone's activation record, as `nvx::activation::ActivationRecord::encode` writes it, at most 1 KiB | +| 28 | `Await`: the host's request for the `Ready` of a clone, which a restore of a [template's](#template-mode) snapshot started, and which the guest answers once it activated the clone with the activation record of its restore packet | none | + +Kind 27 is no message's: it was the `Activate` that carried a clone's activation +record, which the clone's restore packet carries instead. The session keeps no message boundaries, so a message may span several of its records, and one record may end a message and start the next. `Decoder` reassembles the messages. It checks each header before it takes any byte of the body, and holds at most one partial message. A malformed message ends the -stream, because the framing cannot find the next message after it. A clone's -`Activate` whose header announces more than 1 KiB fails the clone's -activation instead, as a record that long does. +stream, because the framing cannot find the next message after it. The framing carries each body as bytes, within its kind's limit. The bodies of -kinds 6 to 22 and 24 have encodings of their own, which the `root`, `probe`, -and `exec` modules write and read, and that of kind 27 is `nvx::activation`'s. -A body that does not decode leaves the stream intact, at the start of the next -message. The guest answers a root configuration that does not decode with -`RootFailed`, as a configuration that it refuses, and a probe with -`ProbeFailed`, while a message about an execution that does not decode ends the -worker's session, as one that breaks the execution's rules does, and an -activation record that does not decode fails the clone's activation. The host -program's client ends on an answer that does not decode, since it cannot tell -what the guest did. +kinds 6 to 22 and 24 have encodings of their own, which the `root`, `probe`, and +`exec` modules write and read. A body that does not decode leaves the stream +intact, at the start of the next message. The guest answers a root configuration +that does not decode with `RootFailed`, as a configuration that it refuses, and +a probe with `ProbeFailed`, while a message about an execution that does not +decode ends the worker's session, as one that breaks the execution's rules does. +The host program's client ends on an answer that does not decode, since it +cannot tell what the guest did. ### Root configurations @@ -168,7 +167,7 @@ also the build that the host program ships unless it names another; see [The runtime identity](#the-runtime-identity). The host sends a guest agent of another build nothing more; see [The host's order](#the-hosts-order). -A clone's `Ready`, its answer to the host's `Activate`, also reports its +A clone's `Ready`, its answer to the host's `Await`, also reports its activation after the build, as `nvx::activation::ReadyActivation::encode` writes it: 1, then the transaction ID of the record's restore, 16 bytes. `root::Ready::with_activation` adds it, and `root::Ready::activation` returns @@ -249,13 +248,16 @@ microseconds after the repair's acknowledgement of the restore: whether the repair acknowledged the restore, `acknowledged`, `yes`, or `none` for a restore whose host did not gate its input until the acknowledgement; how long the worker waited before the repair, `repair-delay`, in microseconds, as its template's -[repair delay](#template-mode) asked, for tests, or `none` without one; and -when the clone took the reset that the restore queued, `reset`, when the host's -first message came, `message`, which must be the clone's `Activate`, when the -clone was ready, `ready`, when its sandbox started, `started`, and when the -start of the host's first execution came, `execution`. Each event is `none` -until it happened, `before` if it happened before the acknowledgement, and -`unknown` without one. The worker of a VM that boots cold runs no such probe. +[repair delay](#template-mode) asked, for tests, or `none` without one; and when +the clone took the reset that the restore queued, `reset`, at which it started +to apply the record of its restore packet, when the host's first message came, +`message`, which must be its `Await`, when the application of the record +completed, `applied`, before the host's first message or after it, when the +clone was ready, `ready`, once the record applied and the host asked for its +`Ready`, when its sandbox started, `started`, and when the start of the host's +first execution came, `execution`. Each event is `none` until it happened, +`before` if it happened before the acknowledgement, and `unknown` without one. +The worker of a VM that boots cold runs no such probe. The probes of `worker::mapping` inspect the mappings beneath the root, and map from the host export, as an adapter maps the host paths of its sandboxes. `map` @@ -828,12 +830,16 @@ after the repair's acknowledgement of the restore: 1. It resumes the session that the template attached, and whose first epoch it acknowledged, which the snapshot holds, and acknowledges the reset that the - restore queued, the only one that the clone reads, with its receive window: - it attaches no session. -2. It takes the host's `Activate`. The activation decodes its record, which - must be a clone's, with a restore section, and checks that it expects the - CPU count and the memory size that the repair found, before anything has a - side effect. + restore queued with its receive window, which it writes at once, since + OpenVMM's broker attaches the host only once the guest acknowledged that + reset, so that the host attaches while the worker activates the clone: it + attaches no session. +2. It takes the activation record of the clone's restore packet, which the host + supplied at the clone's launch, with `--restore-activation-record`, and which + OpenVMM sealed in the packet before the first restored vCPU ran, without + waiting for the host. The activation decodes the record, which must be a + clone's, with a restore section, and checks that it expects the CPU count and + the memory size that the repair found, before anything has a side effect. 3. It applies the record with `nvx::guest::clone::apply`, with `worker::capture::root_template` of the template's configuration and of its NIC's IPv4 gateway, which `template::boot` read from the template's kernel @@ -871,12 +877,18 @@ after the repair's acknowledgement of the restore: execution; with the image's own machine ID and without a DNS server, it binds nothing. 4. Once it took what the session received while the application ran, which came - before `Ready`, it completes the activation, which opens the fence, and - answers `Ready`, which reports the activation after the guest agent's build, - to the `Activate` and to each retry of it, byte for byte, that came - meanwhile, which joins the application. Anything else that came meanwhile, a - reset included, fails the activation, as before the application, so the host - waits for `Ready` before it sends the mapping table or the start. + before `Ready`, it completes the activation, and answers `Ready`, which + reports the activation after the guest agent's build, to each `Await` of the + host's current session that came meanwhile, which joins the application, as + it answers each `Await` that comes later at once. The clone is ready, which + opens the fence, once the record applied and the host asked for `Ready`: at + the completion if an `Await` joined it, and at the host's next `Await` + otherwise. The host asks for `Ready`, the first request of each of its + sessions, rather than the worker sending it unasked, since OpenVMM's broker + refuses the guest's records while no host is attached, which the guest cannot + tell. A reset abandons the `Await`s of the earlier session, and anything else + that came before `Ready` fails the activation, so the host waits for `Ready` + before it sends the mapping table or the start. 5. It then serves the host's order, as after a cold boot: the mapping table that the record announces, which it applies from the record's export, the start, the executions, the probes, and the stop. A clone has no workload identity of @@ -899,24 +911,25 @@ after the repair's acknowledgement of the restore: [`activation` probe](#probes), with when the events of its activation and order happened after the repair's acknowledgement of the restore. -Until the clone is ready, its session takes nothing but that `Activate`: -another reset, any other message, a `Stop` and a `Ping` included, an -`Activate` that the activation refuses, and the passing of the deadline fail -the activation finally, as does an application that failed, which nothing -retries. Once the clone is ready, a retry of the same `Activate`, byte for -byte, gets the same `Ready` without a second application, and any other -`Activate` fails the activation. A failed activation ends the clone with the -status of its class, which `worker::activation::failure_status` selects, and a -broker that closes the stream before the clone is ready ends the clone with -status 1, once the worker dropped its executions, and waited for its stream, -and then the tty, to take what it wrote, for at most -`worker::UNACTIVATED_DRAIN`, a second each, whatever the stream does, so that a -stream that takes nothing, as once the broker failed, does not keep the clone -from ending; the fence stays closed if the clone was not ready. Once the clone -is ready, a broker that closes the stream ends it as it ends the worker of a VM -that boots cold. The worker of a VM that boots cold refuses an `Activate` as -any message that it does not take in its state. The worker, and with it the VM, -ends with one of these statuses, after a diagnostic on standard error: +Until the clone is ready, its session takes nothing but resets and the host's +`Await`: any other message, a `Stop` and a `Ping` included, a restore packet +without a record, or with one that the activation refuses, and the passing of +the deadline, whether the record applied or not, so that a clone whose host +never asks ends, fail the activation finally, as does an application that +failed, which nothing retries. Once the clone is ready, each `Await`, in any +session, gets the same `Ready` without a second application. A failed activation +ends the clone with the status of its class, which +`worker::activation::failure_status` selects, and a broker that closes the +stream before the clone is ready ends the clone with status 1, once the worker +dropped its executions, and waited for its stream, and then the tty, to take +what it wrote, for at most `worker::UNACTIVATED_DRAIN`, a second each, whatever +the stream does, so that a stream that takes nothing, as once the broker failed, +does not keep the clone from ending; the fence stays closed if the clone was not +ready. Once the clone is ready, a broker that closes the stream ends it as it +ends the worker of a VM that boots cold. The worker of a VM that boots cold +refuses an `Await` as any message that it does not take in its state. The +worker, and with it the VM, ends with one of these statuses, after a diagnostic +on standard error: | Status | When | | --- | --- | @@ -942,14 +955,14 @@ ends with one of these statuses, after a diagnostic on standard error: | 44 | The reseed of the kernel's random number generator failed | | 45 | The restore's finish failed, other than by a violation of the time ABI | | 50 | The clone was not ready before its activation's deadline | -| 51 | The host broke the clone's session: it reset the session again, or sent another request, before the clone was ready, or an `Activate` of another transaction, or one that changes the accepted record | -| 52 | The `Activate`'s record does not decode, breaks a limit, or is a cold boot's | +| 51 | The host broke the clone's session: it sent a request other than `Await` before the clone was ready | +| 52 | The clone's restore packet carried no activation record, or its record does not decode, breaks a limit, or is a cold boot's | | 53 | The record expects other resources than the clone has | | 54 | The root's assembly failed, its upper not fresh among its causes, or the kernel did not reread the partition table of the record's GPT disk before it | | 55 | The clone's workload cannot be activated: the machine ID that the record asks for cannot be generated, or the record's hostname or that machine ID cannot be set for every execution | | 56 | The record's host export cannot be mounted | | 57 | The record's network settings cannot be applied: the resolver file that names its DNS server cannot be written, or set for every execution, or the announcer of the listeners that its network publishes cannot start, as without the template's gateway; or that announcer ended | -| 58 | The worker's own order misused the activation: it completed an activation that did not run, or started the sandbox before the clone was ready, or again | +| 58 | The worker's own order misused the activation: it accepted the record again, completed an activation that did not run, or started the sandbox before the clone was ready, or again | | 193 | A conformance check of the time ABI failed in the clone's repair, such as the finish's, which needs a running time daemon | | 195 | The time ABI's part of the clone's repair failed, such as the validation of the restore packet, or the record of the restore | @@ -991,10 +1004,10 @@ transport to the session's host endpoint, such as the one that - `configure_root` sends a root configuration, and returns how the guest answered: `Ready`, with the guest agent's build, or the stage of the root that failed; -- `activate` sends a clone's activation record in an `Activate`, and returns - the guest's `Ready`, with the guest agent's build and the clone's - activation. A clone whose activation failed answers nothing, but ends, and - its session with it; +- `await_ready` sends `Await`, which asks for a clone's `Ready`, and returns the + guest's `Ready`, with the guest agent's build and the clone's activation, once + the clone activated with the record of its restore packet. A clone whose + activation failed answers nothing, but ends, and its session with it; - `map` sends a part of the host's mapping table, and returns how the guest answered: `Mapped`, or `MapFailed`, with how many of the table's mappings it applied and the problem; `start` sends `Start`, and checks that the guest @@ -1009,7 +1022,7 @@ with a new message, in a new attachment. A failure of the session, a send that fails or times out, a malformed message, an answer whose body does not decode, and a message that only the host program sends, an execution's start, input, end of input, or cancellation, and a -clone's `Activate` included, end the client, and its later calls fail. A +clone's `Await` included, end the client, and its later calls fail. A receive that times out doesn't: it keeps what arrived. A request ends the client on any failure after the framing accepted it, a timeout included, because an answer that arrived later would answer the wrong request. @@ -1083,32 +1096,34 @@ which fails before anything is sent, or a failure of the client. The host program [tears the VM down](#commands) after any of them. A clone, which a restore of a [template's](#template-mode) snapshot started, -takes the host's activation of its record instead of a root configuration: -the guest assembles the clone's root from the record, with the inputs that the -template's capture kept, as it activates the clone. Its first step is -`host::activate` instead of `host::ready`: it sends the clone's activation -record in an `Activate`, the first request of the session, and waits for -`Ready`, whose build it checks as `host::ready` does, and which reports the -clone's activation. The table and `Start` then follow as after a cold boot, in -two steps: `host::map_table` sends the table as `host::start_sandbox` does, -and ends the order in the same way at a part that the guest fails, and -`Client::start` sends `Start`. `host::CloneOrder` runs these steps as the -framing of the restore sequence of `nvx::host::template`, which checks the -activation against the record, sends the table only for a clone with an -export, and `Start` only once the guest applied every mapping, and which -replaces a session that the order takes for lost: one that a reset of the -broker, the end of its stream, or a failure of the transport ended while the -host attached or awaited `Ready`. A clone that was ready answers the same -record, byte for byte, with the same `Ready` in the new session, and one that -took the reset before it was ready has ended. OpenVMM's broker ends the stream -both to end an attachment and to refuse one, so the order takes such a refusal -for a lost session too, whose retries the sequence's limit of sessions bounds. -Any other failure is final: a refusal that the broker sends, a deadline that -passed, another build, and a message that breaks the framing. The guest's -`MapFailed` does not tell whether the request's mappings or a failure of the -guest's own failed the table, so the sequence attributes it, as any other -failure of the table, to what the host saw of it: a broken protocol, unless the -deadline passed or the session was lost. +takes no root configuration: the guest assembles the clone's root from the +activation record that OpenVMM handed it in its restore packet, with the inputs +that the template's capture kept, as it activates the clone, before the host +attaches, or while it does. Its first step is `host::await_ready` instead of +`host::ready`: it asks for the clone's `Ready` in an `Await`, the first request +of the session, and waits for `Ready`, whose build it checks as `host::ready` +does, and which reports the clone's activation. The host asks, rather than the +guest sending `Ready` unasked, since OpenVMM's broker refuses the guest's +records while no host is attached, which the guest cannot tell. The table and +`Start` then follow as after a cold boot, in two steps: `host::map_table` sends +the table as `host::start_sandbox` does, and ends the order in the same way at a +part that the guest fails, and `Client::start` sends `Start`. `host::CloneOrder` +runs these steps as the framing of the restore sequence of +`nvx::host::template`, which checks the activation against the record, sends the +table only for a clone with an export, and `Start` only once the guest applied +every mapping, and which replaces a session that the order takes for lost: one +that a reset of the broker, the end of its stream, or a failure of the transport +ended while the host attached or awaited `Ready`. The clone answers the `Await` +of the new session with the same `Ready`, since no session changes the record of +its restore packet. OpenVMM's broker ends the stream both to end an attachment +and to refuse one, so the order takes such a refusal for a lost session too, +whose retries the sequence's limit of sessions bounds. Any other failure is +final: a refusal that the broker sends, a deadline that passed, another build, +and a message that breaks the framing. The guest's `MapFailed` does not tell +whether the request's mappings or a failure of the guest's own failed the table, +so the sequence attributes it, as any other failure of the table, to what the +host saw of it: a broken protocol, unless the deadline passed or the session was +lost. ## VMs @@ -1413,7 +1428,7 @@ The steps of a launch, in order, then that of a command that reattaches, are: | `spawned` | The VMM runs, and the launch's journal records it, but the launch has not connected to the VM. | | `ready` | The guest agent, of the build that the host ships, has assembled the layered root, and the launch has released the checked files, but has sent neither the mapping table nor `Start`. | | `commit` | The guest has started the sandbox, if the VM has a layered root, the guest agent has answered, and the launch is about to commit: its journal still records the VMM, which no runtime record names yet. | -| `found` | `reattach` or `stop` has found the VM running, and holds it, under the sandbox's lock, with the session in which a clone answered the `Activate` of its record again, if the VM is a clone's, but has sent the guest agent nothing more, and stopped nothing. `console` reaches it once it also connected to the clone's boot console. | +| `found` | `reattach` or `stop` has found the VM running, and holds it, under the sandbox's lock, with the session in which a clone answered the host's `Await` with its `Ready`, if the VM is a clone's, but has sent the guest agent nothing more, and stopped nothing. `console` reaches it once it also connected to the clone's boot console. | A value that names no step or action is a usage error. Without the variable, the hook does nothing. @@ -1712,8 +1727,8 @@ it has one, under the policy of the clone's network, which `--restore-network-policy` installs in place of the template's complete deny-all policy, and the restore sequence activates the clone through the -adapter's clone order, `host::CloneOrder`, which sends the clone's activation -record in an `Activate`, checks the guest agent's build in its `Ready`, sends +adapter's clone order, `host::CloneOrder`, which asks for the clone's `Ready` +in an `Await`, checks the guest agent's build in its `Ready`, sends the export's mapping table, which the record announces, in order, as a launch sends its own, and then `Start`. The record names the export by the identity of its plan, and the table holds its mappings, each from a source that names its @@ -1758,9 +1773,9 @@ that holds a lease must not move: they refuse one whose lease names another path, at which no sandbox holds the same lease, as the lease of a sandbox that moved does, rather than take another sandbox there for it. They then reattach to the clone with NVX's `reattach_clone`, in a new session of the adapter's -clone order, in which the guest agent answers the `Activate` of the clone's -record again, with the `Ready` that it answered once, and which they then use -as `reattach` attaches one. The reattach ends a clone that runs under another +clone order, in which the guest agent answers the host's `Await` with the +`Ready` that it answered once, and which they then use as `reattach` attaches +one. The reattach ends a clone that runs under another runtime identity, without a word to its guest, or that fails the check of its `Ready`, through its verified process identity, and finishes the record of an earlier failure of the clone, whose teardown it resolves if it can. The @@ -1993,8 +2008,8 @@ Each command of `runtime-adapter-host` but `stage`, `list`, `inspect`, and child, through its verified process identity. It reads the capability from the store, attaches a new session, in the session's next epoch, exchanges a `Ping`, and exits, leaving the VM running. It reattaches to a - [clone](#clones) in the session in which the clone answers the `Activate` of - its record again, and releases the lease of a clone that ended, through the + [clone](#clones) in the session in which the clone answers the host's `Await` + with its `Ready`, and releases the lease of a clone that ended, through the template store TEMPLATES, if given, in place of the one that the clone's record names. - `start STORE --template TEMPLATES --generation G --distro FORMAT:PATH --distro-identity UUID --upper (scratch:PATH | ram) [--processors N] [--memory MIB] [--workload-identity UID:GID] [--overlay-mode MODE] [--scratch-journal POLICY] [--first-partition-bound BYTES] [--machine-id (image | generated)] [--network FILE] [--boot-console (none | listen)] [SHARES]` @@ -2317,10 +2332,10 @@ Windows. They cover: stale and identity-less resets, record lengths, bootstrap identity, ERROR payloads, a payload beyond the session's limit, which fails at its header, and header faults; -- the messages that it refuses: malformed headers, which fail before their - body arrives, and messages that only the host program sends, a clone's - `Activate` among them; and a message beyond the framing's limit, which it - refuses to send; +- the messages that it refuses: malformed headers, which fail before their body + arrives, the retired kind 27 among them, and messages that only the host + program sends, a clone's `Await` among them; and a message beyond the + framing's limit, which it refuses to send; - the answers that its requests check, and a request whose answer arrives too late; - a layered root: the configuration that the client sends, the guest's @@ -2338,16 +2353,15 @@ Windows. They cover: teardown that failed, or as the session ends; and `Maps` or `Start` answered with another kind, or failures whose bodies do not decode, which end the client; -- a clone's order: an attachment that waits through the session's bootstrap, - the record as the first request, after which the host waits for `Ready`, - whose activation it returns, also none, then the table in parts and `Start`; - a session that a later reset, the end of the stream, or a failure of the - transport ended, which the order takes for lost, and whose successor sends - the same record first; and the failures that are final: a refusal in an - ERROR, deadlines that pass while attaching or awaiting `Ready`, another - build, after which the host sends nothing more, an `Activate` answered with - another kind or with an activation that does not decode, and a record beyond - the framing's limit, which fails before it is sent; +- a clone's order: an attachment that waits through the session's bootstrap, its + `Await` as the first request, after which the host waits for `Ready`, whose + activation it returns, also none, then the table in parts and `Start`; a + session that a later reset, the end of the stream, or a failure of the + transport ended, which the order takes for lost, and whose successor gets the + same `Ready`; and the failures that are final: a refusal in an ERROR, + deadlines that pass while attaching or awaiting `Ready`, another build, after + which the host sends nothing more, and an `Await` answered with another kind + or with an activation that does not decode; - the driver of executions, whose guest runs the runner of NVX's execution engine for each start, as the worker does, without a program: two executions at once, one with more output than its credit holds, each @@ -2391,8 +2405,8 @@ the worker reports before it ends with the stage's status, without the operations of the later stages; the account check of a sandbox's workload identity, once the root is assembled, with the default hostname; a teardown that fails; before the root, the configuration and the `Stop` that it accepts, -and the messages that it refuses, a clone's `Activate` among them; once the -root is ready, a second configuration, an `Activate`, and a broker that closes +and the messages that it refuses, a clone's `Await` among them; once the +root is ready, a second configuration, an `Await`, and a broker that closes the stream, which leave the root that it assembled to its operations; and a broker that closes only its end of the stream while the worker probes, which still receives the replies that the worker queued. @@ -2515,61 +2529,61 @@ network's DNS server, and the setting of the record's hostname and machine ID for every execution, each of which a test may fail. The tests cover: - the reset that the restore queued, of another instance, which the worker - acknowledges without an attachment, and the `Activate`, whose record it - applies once, with the template's inputs, and answers with the `Ready` that - reports the activation; -- retries of the `Activate`, in records of 7 bytes, after a later reset, and - once the sandbox started, which get the same `Ready` without a second - application, and the order that the worker serves once the clone is ready, - as after a cold boot; -- the messages before the `Activate`, a stop and a ping among them, and another - reset, which fail the activation with the status of the host's session, with - nothing answered or applied; -- a record that does not decode, a cold boot's, and one that expects other - resources, which fail the activation with nothing applied, and a GPT disk - whose partition table the kernel does not reread, a machine ID that cannot be - generated, a hostname or machine ID that cannot be set, an export that cannot - be mounted, and a resolver file that cannot be written, which fail its - application, each with the status of its class, and no `Ready`; -- an `Activate` whose header announces more than 1 KiB, which the framing - refuses before it takes any byte of its body, and which fails the activation - with the record's status, before the clone is ready and once it is; -- the activation's deadline, whether the restore's reset came or not, or part - of an `Activate`, which the broker sends before the worker runs, so that the - worker takes it at once, however late it runs: the deadline fails the - activation with nothing applied, and ends nothing once the clone is ready; + acknowledges without an attachment, at which it applies the record of its + restore packet, once, with the template's inputs, and the host's `Await`, + which it answers with the `Ready` that reports the activation; +- more `Await`s, in records of 2 bytes, after a later reset, and once the + sandbox started, which get the same `Ready` without a second application, and + the order that the worker serves once the clone is ready, as after a cold + boot; +- the messages before the host's first `Await`, a stop and a ping among them, + which fail the activation with the status of the host's session, with nothing + answered, once the record applied, and a later reset while the record applies, + which abandons the `Await` of the earlier session, after which the clone + answers the `Await` of the new session; +- a restore packet without a record, and a record that does not decode, a cold + boot's, and one that expects other resources, which fail the activation with + nothing applied, and a GPT disk whose partition table the kernel does not + reread, a machine ID that cannot be generated, a hostname or machine ID that + cannot be set, an export that cannot be mounted, and a resolver file that + cannot be written, which fail its application, each with the status of its + class, and no `Ready`; +- the activation's deadline, whether the restore's reset came or not, which the + broker sends before the worker runs, so that the worker takes it at once: the + deadline fails the activation with nothing applied, and fails a clone whose + record applied, but whose host never asks for its `Ready`, and ends nothing + once the clone is ready, which it is once the host's `Await` came, after the + application; - an application that would outlast the activation's deadline, which gets that deadline, and which ends the clone with the deadline's status once the deadline passed, with no `Ready`; -- a reset, a request, and a retry of the `Activate` that the host sends while - the application runs, which the worker takes before it completes the - activation: the reset and the request fail it, with the status of the host's - session and no `Ready`, and the retry joins it, so that both get `Ready`, - after one application; +- an `Await` and a request that the host sends while the application runs, which + the worker takes before it completes the activation: the request fails it, + with the status of the host's session and no `Ready`, and `Await`s join it, so + that each gets `Ready`, after one application; - a stream that takes nothing more from the worker, as once the broker failed, - after which the worker still ends once its activation's deadline passed, - with the deadline's status, or once the broker closed its end of the stream - before the clone was ready, with status 1, once it waited for the stream for - `worker::UNACTIVATED_DRAIN`; + after which the worker still applies the record, and ends once its + activation's deadline passed, with the deadline's status, or once the broker + closed its end of the stream before the clone was ready, with status 1, once + it waited for the stream for `worker::UNACTIVATED_DRAIN`; - a broker that closes the stream before the clone is ready, whether the - restore's reset came or not, or part of an `Activate`, which ends the clone - with status 1, with nothing applied, and once the clone is ready, which ends - it as it ends the worker of a VM that boots cold; + restore's reset came or not, or part of an `Await`, which ends the clone with + status 1, and once the clone is ready, which ends it as it ends the worker of + a VM that boots cold; - a broker that sent the restore's reset, and a ping or nothing more, and is gone before the worker runs, so that the worker cannot write its - acknowledgement: the worker still ends with the activation's status, or with - status 1, rather than with the failure of its stream; -- an `Activate` of another transaction, or of the same transaction with - another record, once the clone is ready, or its sandbox started, which fails - the activation with the status of the host's session; + acknowledgement: the worker still applies the record, and ends with the + activation's status, or with status 1, rather than with the failure of its + stream; - the identity that each start names, whether the record names a workload identity or not, while a start that names none is refused with `EINVAL`, and the record's hostname, which every execution's entry takes; - the `activation` probe, which the worker answers itself, with no root - operation, executions running or not: the restore's reset, the host's first - message, the clone's `Ready`, and its start, in that order, after the - repair's acknowledgement, and the start of the first execution after them, - and no delay of the repair, which the clone's template did not ask for; + operation, executions running or not: the restore's reset, then the host's + first message and the application of the record, in either order, then the + clone's `Ready`, and its start, after the repair's acknowledgement, and the + start of the first execution after them, and no delay of the repair, which the + clone's template did not ask for; - the mapping table of the record's export, which the host sends in parts once the clone is ready, and which the worker applies in order, from the export that the application mounted, before it starts the sandbox, and the endpoint @@ -3039,7 +3053,7 @@ directory that the starts authorize: | `a_template_s_clones_run_with_its_kernel_tuning_parameters` | Of a template whose kernel boots with the kernel tuning parameter `mitigations=off`: one | The template's key, whose kernel command line holds `mitigations=off` once; then the clone started, whose `/proc/cmdline`, which an execution reads, holds it once, and is the key's command line, token for token; a stop releases the lease | | `one_generation_serves_clones_of_either_format_over_either_upper` | Of a template whose placeholder is a GPT disk of two layers, longer than the EROFS image, four at once, each image shorter than the placeholder: the EROFS image over a scratch image and in RAM, a GPT disk of one layer, of the EROFS image's length, in RAM, and one of three layers, as short as its layout allows, over a scratch image, with a bound that its first partition meets exactly; then the disk of one layer, with a bound one byte smaller than its first partition | Each started, with its upper and the overlay's options; the EROFS image's root, without a first partition; each GPT disk's layers, ext4 file systems mounted read-only once each, from the partitions that its own table names, stacked in order, and the bytes of its own first partition; the image's read-only disk at the placeholder's length, and the scratch device at the template's scratch length, each of which the guest reads to its end; for a clone in RAM, no file of its sandbox as long as the template's scratch, and on Linux `--restore-absent-scratch` and no `scratch` block device among its VMM's arguments, which name one over a scratch image; then status 8, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of an unusable root, with the guest's status 54; nothing quarantined, and the lease released, with the records | | `a_start_refuses_what_the_template_cannot_serve_before_any_vmm_starts` | An EROFS image and a GPT disk 4 KiB longer, and an EROFS image 100 bytes longer, than the placeholder, a scratch image of twice the template's scratch, memory targets of 2 GiB and of 320 MiB, unstaged images, and a generation that the store does not hold, then the generation once its key is gone | For those that NVX's plan refuses, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of a length that the request supplied, of the image's geometry, which OpenVMM cannot attach, or of invalid targets, which quarantines nothing; for unstaged images, status 1, without a record; for the generation, status 10, with the record of a selection that names no template, which the store never published, without a lease; any lease released, and no VMM's log or record in the sandbox; for the generation without its key, status 8, and the record of a failure of the template, at the plan, by the evidence of a malformed generation, which quarantines the generation, so that the store refuses a later start's selection, with status 10, as `ineligible`, by the evidence of its tombstone, without a lease | -| `a_clone_whose_activation_fails_ends_before_it_is_ready` | A scratch image of zeros, a journal that the record forbids, and, through NVX's restore sequence, a ping, a start, or the start of an execution before the `Activate` | Status 8, no `Ready`, and the record of a failure, at the activation, once the VM exited: of the request, by the evidence of an unusable root, with the guest's status 54, or, before the `Activate`, of an unknown cause, by the protocol's, with 51, which the sandbox records until the release; nothing quarantined, and the lease released, with the records | +| `a_clone_whose_activation_fails_ends_before_it_is_ready` | A scratch image of zeros, a journal that the record forbids, and, through NVX's restore sequence, a ping, a start, or the start of an execution before the `Await` | Status 8, no `Ready`, and the record of a failure, at the activation, once the VM exited: of the request, by the evidence of an unusable root, with the guest's status 54, or, before the `Await`, whether the record applied or not, of an unknown cause, by the protocol's, with 51, which the sandbox records until the release; nothing quarantined, and the lease released, with the records | | `a_clone_refuses_a_scratch_image_whose_upper_an_earlier_clone_used` | Over a scratch image, in whose root the `contents` probe writes a file, then over a copy of that scratch image once the first clone stopped | The first started; then status 8, since the copy's upper is not fresh, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of an unusable root, with the guest's status 54; nothing quarantined, and the lease released, with the records | | `clones_at_once_keep_their_own_uppers_and_machine_ids` | Three at once: over a scratch image and in RAM, each with a generated machine ID, and in RAM with the image's own; two executions in each, as user ID 0, that write a marker in the root, and try to write `/etc/machine-id` | In every execution, the hostname `runtime-adapter`, and no marker of another clone; a generated machine ID that differs between the clones, the same in each execution of a clone, held by one bind that is read-only, `nosuid`, and `nodev`, and that the write cannot change; with the image's own machine ID, none, since the image has none, no bind, and in the later execution, the ID that the first wrote; then each stop releases its lease | | `a_clone_whose_view_cannot_be_set_ends_before_it_is_ready` | Of a template of the EROFS image, a GPT disk whose layer holds a regular file `etc`, with the image's own machine ID, then with a generated one | The first started, with the disk's root; then status 8, since no bind can hold the machine ID below a file, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of invalid settings, with the guest's status 55; nothing quarantined, and the lease released, with the records | diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs index a2b121337..a19df21b7 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs @@ -306,21 +306,20 @@ //! mismatched, and then signals nothing: the VM keeps running. //! //! The sandbox of a clone holds the clone's lease, through whose path the commands that reattach -//! take the sandbox's lock. They reattach to a clone with `nvx::host::template::reattach_clone`, -//! in a new session of the adapter's clone order, in which the clone answers the Activate of its -//! record again, with the Ready that it answered once, and which they then use as `reattach` -//! attaches one. The reattach ends a clone that runs under another runtime identity, without a -//! word to its guest, or that fails the check of its Ready, through its verified identity, and -//! finishes the record of an earlier failure of the clone, whose teardown it resolves if it can. -//! The command then releases the lease of a clone whose VM ended without a failure, and fails; -//! ends as `start` does for a clone that failed; and keeps the lease and the records, and fails, -//! for a clone whose state cannot be determined. It releases a lease through the template store -//! that the clone's record names, once the lease is of it. A sandbox whose start ended before it -//! recorded the clone holds a lease of a template store that no record names: it records no -//! clone, or an earlier clone's, which a release leaves, of another template store perhaps. -//! `reattach STORE --template TEMPLATES` and `stop STORE --template TEMPLATES` release such a -//! lease through the template store TEMPLATES, in place of the record's, which the release refuses -//! unless the lease is of it. +//! take the sandbox's lock. They reattach to a clone with `nvx::host::template::reattach_clone`, in +//! a new session of the adapter's clone order, in which the clone answers the host's `Await` with +//! the Ready that it answered once, and which they then use as `reattach` attaches one. The +//! reattach ends a clone that runs under another runtime identity, without a word to its guest, or +//! that fails the check of its Ready, through its verified identity, and finishes the record of an +//! earlier failure of the clone, whose teardown it resolves if it can. The command then releases +//! the lease of a clone whose VM ended without a failure, and fails; ends as `start` does for a +//! clone that failed; and keeps the lease and the records, and fails, for a clone whose state +//! cannot be determined. It releases a lease through the template store that the clone's record +//! names, once the lease is of it. A sandbox whose start ended before it recorded the clone holds a +//! lease of a template store that no record names: it records no clone, or an earlier clone's, +//! which a release leaves, of another template store perhaps. `reattach STORE --template TEMPLATES` +//! and `stop STORE --template TEMPLATES` release such a lease through the template store TEMPLATES, +//! in place of the record's, which the release refuses unless the lease is of it. //! //! The commands print what they find as `KEY=VALUE` lines: //! @@ -3135,8 +3134,8 @@ enum Session { /// None yet, of a VM that booted cold, to which the command attaches one once it needs it, as /// [`Found::attach`] does. Cold, - /// The one in which a clone answered the Activate of its record again, with the Ready that it - /// answered once, as `reattach_clone` checks it. + /// The one in which a clone answered the host's `Await` with the Ready that it answered once, + /// as `reattach_clone` checks it. Clone(Box), } @@ -3167,12 +3166,12 @@ impl Found { /// booted cold: it holds the VM again, as `supervisor::reattach` does, and returns a mismatched /// VM's status, which it leaves running. A sandbox with a lease holds a clone, which it reattaches /// to with `reattach_clone`, in a new session of the adapter's clone order, in which the clone -/// answers the Activate of its record again, with its Ready, and which ends a clone that it may -/// not keep: one that runs under another runtime identity, without a word to its guest, or that -/// fails the check of its Ready. A clone that ended releases its lease, and one that failed, or -/// whose earlier failure the reattach finishes, ends the command as [`clone_failed`] does, through -/// the template store that [`clone_templates`] opens with `templates`. A clone whose state cannot -/// be determined keeps its lease and its records, and fails the command; see [`conclude`]. +/// answers the host's `Await` with its Ready, and which ends a clone that it may not keep: one that +/// runs under another runtime identity, without a word to its guest, or that fails the check of its +/// Ready. A clone that ended releases its lease, and one that failed, or whose earlier failure the +/// reattach finishes, ends the command as [`clone_failed`] does, through the template store that +/// [`clone_templates`] opens with `templates`. A clone whose state cannot be determined keeps its +/// lease and its records, and fails the command; see [`conclude`]. fn find( path: &Path, runtime: &Runtime, diff --git a/tests/runtime-adapter/src/frame.rs b/tests/runtime-adapter/src/frame.rs index be5fd4310..8f8f353ab 100644 --- a/tests/runtime-adapter/src/frame.rs +++ b/tests/runtime-adapter/src/frame.rs @@ -14,13 +14,12 @@ //! The framing carries the body of each kind as bytes, within the kind's limit. The bodies of the //! messages of a layered root, of its probes, and of executions have encodings of their own, which //! the [`root`](crate::root), [`probe`](crate::probe), and [`exec`](crate::exec) modules write and -//! read, and that of a clone's activation is the canonical encoding of an activation record, which -//! `nvx::activation` writes and reads: a body that does not decode is a [`BodyError`], which the -//! receiver answers or refuses, while the stream goes on. +//! read: a body that does not decode is a [`BodyError`], which the receiver answers or refuses, +//! while the stream goes on. A clone's activation record takes no message: OpenVMM hands it to the +//! clone in its restore packet. use std::fmt; -use nvx::activation::MAX_RECORD_BYTES; use nvx::protocol::OUTER_MAX_PAYLOAD; /// Bytes of a message's header: the body's length, then the message's kind. @@ -39,10 +38,6 @@ pub const MAX_TEXT_BODY: usize = 4096; /// fields; see [`exec`](crate::exec). pub const MAX_CONTROL_BODY: usize = 64; -/// Most bytes of the body of an `Activate`: the canonical encoding of an activation record, which -/// holds at most `nvx::activation::MAX_RECORD_BYTES`. -pub const MAX_ACTIVATE_BODY: usize = MAX_RECORD_BYTES; - /// A message of the adapter's framing. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Message { @@ -61,8 +56,8 @@ pub enum Message { /// itself, so that it answers one that it refuses with [`Message::RootFailed`]. RootConfig(Vec), /// The guest's answer to [`Message::RootConfig`] once it has assembled the root, or to - /// [`Message::Activate`] once it activated a clone, with the guest agent's build, and what it - /// reports about a clone's activation, encoded as a [`Ready`](crate::root::Ready). + /// [`Message::Await`] once a clone is ready, with the guest agent's build, and what it reports + /// about a clone's activation, encoded as a [`Ready`](crate::root::Ready). Ready(Vec), /// A stage of the guest's layered root failed, encoded as a /// [`RootFailure`](crate::root::RootFailure): the guest's answer to [`Message::RootConfig`], @@ -114,11 +109,13 @@ pub enum Message { Start, /// The guest's confirmation of [`Message::Start`]. Started, - /// The host's activation of a clone, which a restore of a template's snapshot started: the - /// clone's activation record, in its canonical encoding, which - /// `nvx::activation::ActivationRecord::encode` writes, and which the guest decodes and checks - /// itself. The guest answers it with [`Message::Ready`] once it activated the clone. - Activate(Vec), + /// The host's request for the Ready of a clone, which a restore of a template's snapshot + /// started, the first message of each of the host's sessions with the clone: the clone takes + /// its activation record from its restore packet, and the guest answers with + /// [`Message::Ready`] once the clone is ready, as often as the host asks. OpenVMM's broker + /// refuses the guest's records while no host is attached, which the guest cannot tell, so the + /// host asks. + Await, } impl Message { @@ -131,8 +128,8 @@ impl Message { /// its kind allows: [`MAX_BODY`] for an `Echo`, a `Report`, an execution's start, or its input /// or output, or a part of the mapping table, [`MAX_TEXT_BODY`] for a root configuration, the /// guest's readiness, a root's failure, a failure of the mapping table, a probe, or a probe's - /// failure, [`MAX_CONTROL_BODY`] for an execution's other messages, [`MAX_ACTIVATE_BODY`] for - /// an `Activate`, and none for the others. + /// failure, [`MAX_CONTROL_BODY`] for an execution's other messages, and none for the + /// others. pub fn encode(&self) -> Result, FrameError> { let (kind, body) = self.parts(); if body.len() > kind.max_body() { @@ -179,7 +176,7 @@ impl Message { Self::MapFailed(body) => (Kind::MapFailed, body), Self::Start => (Kind::Start, &[]), Self::Started => (Kind::Started, &[]), - Self::Activate(body) => (Kind::Activate, body), + Self::Await => (Kind::Await, &[]), } } } @@ -236,8 +233,10 @@ const KIND_MAP_FAILED: u8 = 24; const KIND_START: u8 = 25; /// Kind of a [`Message::Started`]. const KIND_STARTED: u8 = 26; -/// Kind of a [`Message::Activate`]. -const KIND_ACTIVATE: u8 = 27; +/// Kind of a [`Message::Await`]. Kind 27 was the `Activate` that carried a clone's activation +/// record, which its restore packet carries since: the framing defines it no more, so that a host +/// or a guest of either framing refuses the other's at once. +const KIND_AWAIT: u8 = 28; /// Kind of a [`Message`]. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -295,8 +294,8 @@ enum Kind { Start = KIND_START, /// [`Message::Started`]. Started = KIND_STARTED, - /// [`Message::Activate`]. - Activate = KIND_ACTIVATE, + /// [`Message::Await`]. + Await = KIND_AWAIT, } impl Kind { @@ -329,7 +328,7 @@ impl Kind { KIND_MAP_FAILED => Some(Self::MapFailed), KIND_START => Some(Self::Start), KIND_STARTED => Some(Self::Started), - KIND_ACTIVATE => Some(Self::Activate), + KIND_AWAIT => Some(Self::Await), _ => None, } } @@ -361,14 +360,14 @@ impl Kind { | Self::ExecEnd | Self::ExecInputClosed | Self::ExecTerminal => MAX_CONTROL_BODY, - Self::Activate => MAX_ACTIVATE_BODY, Self::Ping | Self::Pong | Self::Stop | Self::Stopped | Self::Mapped | Self::Start - | Self::Started => 0, + | Self::Started + | Self::Await => 0, } } @@ -402,7 +401,7 @@ impl Kind { Self::MapFailed => Message::MapFailed(body), Self::Start => Message::Start, Self::Started => Message::Started, - Self::Activate => Message::Activate(body), + Self::Await => Message::Await, } } } @@ -414,7 +413,7 @@ pub enum FrameError { /// A kind that the framing does not define. Kind(u8), /// A body longer than its kind allows: [`MAX_BODY`] bytes, [`MAX_TEXT_BODY`], - /// [`MAX_CONTROL_BODY`], [`MAX_ACTIVATE_BODY`], or none. + /// [`MAX_CONTROL_BODY`], or none. Length { /// Kind of the message. kind: u8, diff --git a/tests/runtime-adapter/src/frame/tests.rs b/tests/runtime-adapter/src/frame/tests.rs index 424d4c3db..16a1c497a 100644 --- a/tests/runtime-adapter/src/frame/tests.rs +++ b/tests/runtime-adapter/src/frame/tests.rs @@ -1,6 +1,5 @@ use super::{ - Decoder, FrameError, HEADER_LEN, Kind, MAX_ACTIVATE_BODY, MAX_BODY, MAX_CONTROL_BODY, - MAX_TEXT_BODY, Message, + Decoder, FrameError, HEADER_LEN, Kind, MAX_BODY, MAX_CONTROL_BODY, MAX_TEXT_BODY, Message, }; /// Returns one message of each kind, with bodies of no, few, and the most bytes. @@ -41,9 +40,7 @@ fn messages() -> Vec { Message::MapFailed(vec![b'm'; MAX_TEXT_BODY]), Message::Start, Message::Started, - Message::Activate(Vec::new()), - Message::Activate(b"\x01\x00record".to_vec()), - Message::Activate(vec![b'a'; MAX_ACTIVATE_BODY]), + Message::Await, Message::Stop, Message::Stopped, ] @@ -80,7 +77,7 @@ fn one_of_each() -> Vec { Message::MapFailed(body(24)), Message::Start, Message::Started, - Message::Activate(body(27)), + Message::Await, ] } @@ -167,23 +164,28 @@ fn the_header_holds_the_body_length_then_the_kind() { ); assert_eq!(Message::Start.encode().unwrap(), [0, 0, 0, 0, 25]); assert_eq!(Message::Started.encode().unwrap(), [0, 0, 0, 0, 26]); - assert_eq!( - Message::Activate(b"x".to_vec()).encode().unwrap(), - [1, 0, 0, 0, 27, b'x'] - ); - // The kinds are numbered from 1, without gaps. - for (kind, message) in (1..).zip(one_of_each()) { + assert_eq!(Message::Await.encode().unwrap(), [0, 0, 0, 0, 28]); + // The kinds are numbered from 1, without gaps but 27, the retired Activate. + for (kind, message) in DEFINED.into_iter().zip(one_of_each()) { assert_eq!(message.kind(), kind, "{message:?}"); } + assert_eq!(one_of_each().len(), DEFINED.len()); for message in messages().into_iter().chain(one_of_each()) { assert_eq!(message.encode().unwrap()[HEADER_LEN - 1], message.kind()); } } +/// The codes of the kinds that the framing defines, in order: every code from 1 to 28 but 27, +/// which the retired Activate had. +const DEFINED: [u8; 27] = [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 28, +]; + #[test] fn each_kind_decodes_from_its_code_alone() { for code in 0..=u8::MAX { - let defined = (1..=27).contains(&code).then_some(code); + let defined = DEFINED.contains(&code).then_some(code); assert_eq!(Kind::from_code(code).map(Kind::code), defined, "{code}"); } } @@ -238,12 +240,13 @@ fn malformed_headers_fail_before_their_body_is_taken() { let too_long = u32::try_from(MAX_BODY + 1).unwrap().to_le_bytes(); let too_long_text = u32::try_from(MAX_TEXT_BODY + 1).unwrap().to_le_bytes(); let too_long_control = u32::try_from(MAX_CONTROL_BODY + 1).unwrap().to_le_bytes(); - let too_long_record = u32::try_from(MAX_ACTIVATE_BODY + 1).unwrap().to_le_bytes(); let header = |length: [u8; 4], kind: u8| [length[0], length[1], length[2], length[3], kind]; let text = |kind: u8| header(too_long_text, kind); let mut cases = vec![ ([0, 0, 0, 0, 0], FrameError::Kind(0)), - ([0, 0, 0, 0, 28], FrameError::Kind(28)), + // The retired Activate, whose record the restore packet carries since. + ([4, 0, 0, 0, 27], FrameError::Kind(27)), + ([0, 0, 0, 0, 29], FrameError::Kind(29)), ([9, 0, 0, 0, 0xff], FrameError::Kind(0xff)), ( [0xff, 0xff, 0xff, 0xff, 3], @@ -252,16 +255,9 @@ fn malformed_headers_fail_before_their_body_is_taken() { length: u32::MAX as usize, }, ), - ( - header(too_long_record, 27), - FrameError::Length { - kind: 27, - length: MAX_ACTIVATE_BODY + 1, - }, - ), ]; // The kinds without a body. - for kind in [1, 2, 4, 5, 23, 25, 26] { + for kind in [1, 2, 4, 5, 23, 25, 26, 28] { let error = FrameError::Length { kind, length: 1 }; cases.push(([1, 0, 0, 0, kind], error)); } @@ -312,12 +308,6 @@ fn encoding_refuses_a_body_beyond_the_limit() { assert_eq!(maps.len(), HEADER_LEN + MAX_BODY); let ready = Message::Ready(vec![0; MAX_TEXT_BODY]).encode().unwrap(); assert_eq!(ready.len(), HEADER_LEN + MAX_TEXT_BODY); - // An Activate carries the largest record, whose limit is the activation record's own. - assert_eq!(MAX_ACTIVATE_BODY, nvx::activation::MAX_RECORD_BYTES); - let activate = Message::Activate(vec![0; MAX_ACTIVATE_BODY]) - .encode() - .unwrap(); - assert_eq!(activate.len(), HEADER_LEN + MAX_ACTIVATE_BODY); let cases = [ (Message::Echo(vec![0; MAX_BODY + 1]), 3, MAX_BODY + 1), (Message::Report(vec![0; MAX_BODY + 1]), 10, MAX_BODY + 1), @@ -365,11 +355,6 @@ fn encoding_refuses_a_body_beyond_the_limit() { 24, MAX_TEXT_BODY + 1, ), - ( - Message::Activate(vec![0; MAX_ACTIVATE_BODY + 1]), - 27, - MAX_ACTIVATE_BODY + 1, - ), ]; for (message, kind, length) in cases { assert_eq!( diff --git a/tests/runtime-adapter/src/hook.rs b/tests/runtime-adapter/src/hook.rs index e2b9073d8..2f67c261e 100644 --- a/tests/runtime-adapter/src/hook.rs +++ b/tests/runtime-adapter/src/hook.rs @@ -60,8 +60,8 @@ pub enum Step { /// which no runtime record names yet. Commit, /// `found`: `reattach` or `stop` has found the sandbox's VM running, and holds it, under the - /// sandbox's lock, with the session in which a clone answered the Activate of its record - /// again, if the VM is a clone's, but has sent the guest agent nothing more, and stopped + /// sandbox's lock, with the session in which a clone answered the host's `Await` with its + /// Ready, if the VM is a clone's, but has sent the guest agent nothing more, and stopped /// nothing. `console` reaches it once it also connected to the clone's boot console, before it /// asks the guest to write its marker. Found, diff --git a/tests/runtime-adapter/src/host.rs b/tests/runtime-adapter/src/host.rs index 9c601e917..75ea532d9 100644 --- a/tests/runtime-adapter/src/host.rs +++ b/tests/runtime-adapter/src/host.rs @@ -25,19 +25,21 @@ //! //! # A clone's order //! -//! A clone, which a restore of a template's snapshot started, takes the host's activation of its -//! record instead of a root configuration: the guest assembles the clone's root from the record, -//! with the inputs that the template's capture kept, as it activates the clone. Its first step is -//! [`activate`], which sends the clone's activation record in an `Activate`, the first request of -//! the session, and waits for the guest agent's `Ready`, whose build it checks as [`ready`] does, -//! and which reports the clone's activation, for the host to check against the record. Then the -//! restore sequence sends the clone's mapping table, which [`map_table`] sends as [`start_sandbox`] -//! does after a cold boot, and `Start`. [`CloneOrder`] runs these steps as the adapter's framing of -//! `nvx::host::template`'s restore sequence, which checks the activation, orders the steps, and -//! which replaces a session that the order lost, as [`CloneOrder`] tells, while the clone was not -//! ready or its `Ready` had not arrived: the clone answers the same record, byte for byte, with the -//! same `Ready` in a new session, and has ended once the reset of a session that it took before it -//! was ready failed its activation. +//! A clone, which a restore of a template's snapshot started, takes no root configuration: the +//! guest assembles the clone's root from the activation record that OpenVMM handed it in its +//! restore packet, with the inputs that the template's capture kept, as it activates the clone, +//! before the host attaches, or while it does. Its first step is [`await_ready`], which asks for +//! the clone's Ready in an `Await`, the first request of the session, and waits for the guest +//! agent's `Ready`, whose build it checks as [`ready`] does, and which reports the clone's +//! activation, for the host to check against the record. The host asks, rather than the guest +//! sending `Ready` unasked, since OpenVMM's broker refuses the guest's records while no host is +//! attached, which the guest cannot tell. Then the restore sequence sends the clone's mapping +//! table, which [`map_table`] sends as [`start_sandbox`] does after a cold boot, and `Start`. +//! [`CloneOrder`] runs these steps as the adapter's framing of `nvx::host::template`'s restore +//! sequence, which checks the activation, orders the steps, and which replaces a session that the +//! order lost, as [`CloneOrder`] tells, while the clone was not ready or its `Ready` had not +//! arrived: the clone answers the `Await` of a new session with the same `Ready`, since no session +//! changes the record of its restore packet. mod executions; mod map_reply; @@ -137,7 +139,7 @@ impl From for ClientError { /// message in as many of the session's records as it takes, and [`receive`](Self::receive) /// returns the guest agent's next message, whether it spans records or shares one with the next /// message. [`ping`](Self::ping), [`echo`](Self::echo), [`configure_root`](Self::configure_root), -/// [`activate`](Self::activate), [`map`](Self::map), [`start`](Self::start), +/// [`await_ready`](Self::await_ready), [`map`](Self::map), [`start`](Self::start), /// [`probe`](Self::probe), and [`stop`](Self::stop) each send a request and wait for its answer, /// and [`Executions`] runs executions over the client. /// @@ -235,7 +237,7 @@ impl Client { | Message::ExecInput(_) | Message::ExecInputEnd(_) | Message::ExecCancel(_) - | Message::Activate(_)), + | Message::Await), ) => { return Err(self.fail(ClientError::Unexpected(message.kind()))); } @@ -299,16 +301,13 @@ impl Client { } } - /// Sends `record`, the activation record of a clone, which a restore of a template's snapshot - /// started, in its canonical encoding, in an `Activate`, and waits until `deadline` for the - /// guest agent's answer: `Ready`, with the guest agent's build and the activation that it - /// reports, once it activated the clone. A clone whose activation failed answers nothing, but - /// ends, and its session with it. - /// - /// A record of more than [`MAX_ACTIVATE_BODY`](crate::frame::MAX_ACTIVATE_BODY) bytes fails - /// before anything is sent, and the client stays usable. - pub fn activate(&mut self, record: &[u8], deadline: Instant) -> Result { - let request = Message::Activate(record.to_vec()); + /// Sends `Await`, which asks for the Ready of a clone, which a restore of a template's + /// snapshot started, and waits until `deadline` for the guest agent's answer: `Ready`, with the + /// guest agent's build and the activation that it reports, once the clone activated with the + /// record of its restore packet. A clone whose activation failed answers nothing, but ends, and + /// its session with it. + pub fn await_ready(&mut self, deadline: Instant) -> Result { + let request = Message::Await; match self.request(&request, deadline)? { Message::Ready(body) => self.decoded(Ready::decode(&body)), reply => Err(self.mismatched(&request, &reply)), @@ -516,24 +515,20 @@ pub fn map_table( } /// Runs the first step of [a clone's order](self#a-clones-order) over `client`, in place of -/// [`ready`]: sends `record`, the clone's activation record in its canonical encoding, in an -/// `Activate`, which must be the first request of the client's attachment, and waits until -/// `deadline` for the guest agent's `Ready`, which it returns once it names `build`, the guest -/// agent's build that the host ships, with the activation that it reports, which the caller checks -/// against the record. +/// [`ready`]: asks for the clone's Ready in an `Await`, which must be the first request of the +/// client's attachment, and waits until `deadline` for the guest agent's `Ready`, which it returns +/// once it names `build`, the guest agent's build that the host ships, with the activation that it +/// reports, which the caller checks against the record of the clone's restore packet. /// /// Fails with [`OrderError::Build`] for a guest agent of another build, to which it sends nothing /// more, and with [`OrderError::Client`] once the client fails: as the session ends once the /// clone's activation failed, or a reset ended the attachment. -pub fn activate( +pub fn await_ready( client: &mut Client, - record: &[u8], build: &str, deadline: Instant, ) -> Result { - let ready = client - .activate(record, deadline) - .map_err(OrderError::Client)?; + let ready = client.await_ready(deadline).map_err(OrderError::Client)?; if ready.build() != build { return Err(OrderError::Build { reported: ready.build().to_owned(), @@ -544,20 +539,20 @@ pub fn activate( } /// [A clone's order](self#a-clones-order), as the adapter's framing of the clone's control -/// session, through which `nvx::host::template`'s restore sequence activates the clone and starts -/// its sandbox: it attaches a [`Client`] over each connection to the clone's control endpoint, -/// runs [`activate`] with the guest agent's build that the host ships, then [`map_table`] with the -/// mapping table that the restore sequence gives it, which the clone's record announces, and -/// `Start`. +/// session, through which `nvx::host::template`'s restore sequence awaits the clone's activation +/// and starts its sandbox: it attaches a [`Client`] over each connection to the clone's control +/// endpoint, runs [`await_ready`] with the guest agent's build that the host ships, then +/// [`map_table`] with the mapping table that the restore sequence gives it, which the clone's +/// record announces, and `Start`. /// /// It takes a session for lost once a reset of the broker, the end of the broker's stream, or a -/// failure of the transport ended it, while the client attached or awaited the clone's `Ready`: -/// the clone may answer the same record in a new session. OpenVMM's broker ends the stream both to -/// end an attachment and to refuse one, as it refuses a host whose authentication outlasts the -/// broker's deadline, so the order takes such a refusal for a lost session too, whose retries the -/// restore sequence's limit of sessions bounds. Any other failure is final: a refusal that the -/// broker sends, a deadline that passed, a guest agent of another build, and a message that breaks -/// the framing. +/// failure of the transport ended it, while the client attached or awaited the clone's `Ready`: the +/// clone answers the `Await` of a new session with the same `Ready`. OpenVMM's broker ends the +/// stream both to end an attachment and to refuse one, as it refuses a host whose authentication +/// outlasts the broker's deadline, so the order takes such a refusal for a lost session too, whose +/// retries the restore sequence's limit of sessions bounds. Any other failure is final: a refusal +/// that the broker sends, a deadline that passed, a guest agent of another build, and a message +/// that breaks the framing. /// /// It takes no failure for the guest's refusal of the request's mapping table, which /// [`CloneControl::refused`] tells: the guest's `MapFailed` does not tell whether the request's @@ -583,13 +578,12 @@ impl CloneControl for CloneOrder<'_> { Client::attach(transport, capability, deadline).map_err(OrderError::Client) } - fn activate( + fn ready( &mut self, session: &mut Client, - record: &[u8], deadline: Instant, ) -> Result, OrderError> { - activate(session, record, self.build, deadline).map(|ready| ready.activation()) + await_ready(session, self.build, deadline).map(|ready| ready.activation()) } fn map( diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 9a405e3f9..bacf2ab9a 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -50,8 +50,9 @@ //! root of its own, in [`template`] mode. The host program's `runtime-adapter-host create` //! captures such a VM through NVX's template store, which publishes the snapshot as a generation //! of a template, in the `capture` module. In each clone, the worker repairs the restore, then -//! activates the clone with the host's `Activate`, whose record the guest library applies, answers -//! `Ready` with the activation, and serves the host's order as after a cold boot. On the host, the +//! activates the clone with the activation record of its restore packet, which the guest library +//! applies, answers the host's `Await` with `Ready`, which reports the activation, and serves the +//! host's order as after a cold boot. On the host, the //! `host` module's clone order is the framing through which the restore sequence of //! `nvx::host::template` activates a clone and starts its sandbox, and the host program's //! `runtime-adapter-host start` starts a clone of a generation in a sandbox of its own, which diff --git a/tests/runtime-adapter/src/root.rs b/tests/runtime-adapter/src/root.rs index a94ce6628..6c3ee0cf4 100644 --- a/tests/runtime-adapter/src/root.rs +++ b/tests/runtime-adapter/src/root.rs @@ -51,7 +51,7 @@ //! Once the guest assembled the root, it answers the configuration with its [`Ready`]: the length //! of the guest agent's build, one byte, then the build, 1 to [`MAX_BUILD_LEN`] printable ASCII //! characters without a space, such as [`GUEST_BUILD`](crate::GUEST_BUILD). A clone, which a -//! restore of a template's snapshot started, answers the host's Activate with its `Ready` once it +//! restore of a template's snapshot started, answers the host's `Await` with its `Ready` once it //! activated, which reports its activation after the build: the canonical encoding of //! `nvx::activation::ReadyActivation`, `1`, then the transaction ID of the restore whose record it //! applied. A cold boot's `Ready` reports no activation, since a root configuration, not an @@ -741,9 +741,10 @@ pub enum RootReply { pub const MAX_BUILD_LEN: usize = u8::MAX as usize; /// The guest's readiness, with which it answers a root configuration once it assembled the root, -/// or a clone's Activate once it activated the clone: the guest agent's build, which the host can +/// or the host's `Await` once it activated a clone: the guest agent's build, which the host can /// check against the build that it ships, and, for a clone, what the guest reports about its -/// activation, which the host can check against the record that it sent. +/// activation, which the host can check against the record that it supplied at the clone's +/// launch. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Ready { /// The guest agent's build. diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index a0aa84bdd..2619418bf 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -72,12 +72,14 @@ //! //! 1. The session that the template attached before its capture, and whose first epoch it //! acknowledged, resumes on the same tty: the worker attaches none, and acknowledges the reset -//! that the restore queued, the only one that the clone reads, which ends the activation's -//! transport bootstrap and restricts the session. -//! 2. The host sends one `Activate`, whose body is the canonical encoding of a clone's -//! `nvx::activation::ActivationRecord`. The activation decodes and checks it, and checks the -//! resources that it expects against those that the repair found, before anything has a side -//! effect. +//! that the restore queued, which it writes to the stream at once, since OpenVMM's broker +//! attaches the host only once the guest acknowledged that reset, so that the host attaches +//! while the worker activates the clone. +//! 2. The activation takes the clone's `nvx::activation::ActivationRecord` from the clone's +//! restore packet, which the repair handed over, and into which OpenVMM sealed the record that +//! the host's launch supplied before the first restored vCPU ran. It decodes and checks it, and +//! checks the resources that it expects against those that the repair found, before anything +//! has a side effect. //! 3. The worker applies the record through `RootOperations::apply`, which runs //! `nvx::guest::clone::apply`: it assembles the root from the template's devices, with the //! record's image, upper, and overlay mode, once the kernel reread the partition table of a @@ -97,12 +99,16 @@ //! mount namespace, which each execution's copies. The template bootstrapped the guest for its //! root with its own configuration before its capture, so the clone has its cgroups. //! 4. Once it took what the session received while the application ran, which came before `Ready`, -//! it completes the activation, which opens the fence, and answers `Ready`, which reports that -//! the activation applied, with the record's transaction, after the guest agent's build, to the -//! `Activate` and to each retry of it, byte for byte, that came meanwhile, which joins the -//! application. Anything else that came meanwhile, a reset included, fails the activation, as -//! before the application, so the host waits for `Ready` before it sends the mapping table or -//! the start. +//! it completes the activation, and answers each `Await` of the host's current session that +//! came meanwhile, which joins the application, with `Ready`, which reports that the activation +//! applied, with the record's transaction, after the guest agent's build, as it answers each +//! `Await` that comes later at once. The clone is ready, which opens the fence, once the record +//! applied and the host asked for `Ready`: at the completion if an `Await` joined it, and at +//! the host's next `Await` otherwise. The host asks for `Ready`, rather than the worker sending +//! it unasked, since OpenVMM's broker refuses the guest's records while no host is attached, +//! which the guest cannot tell. A reset abandons the `Await`s of the earlier session, and +//! anything else that came before `Ready` fails the activation, so the host waits for `Ready` +//! before it sends the mapping table or the start. //! 5. It then serves the host's order as after a cold boot: the mapping table that the record //! announces, which it maps from the record's export, the start, the executions, the probes, and //! the stop. A clone has no workload identity of the whole sandbox: each execution names its @@ -127,14 +133,14 @@ //! fails finally with `nvx::protocol::status`'s `ACTIVATION_NETWORK`, as an application whose //! announcer could not start does. //! -//! Until the clone is ready, its session takes nothing but that `Activate`: another reset, any -//! other message, a `Stop` and a `Ping` included, an `Activate` that the activation refuses, one -//! whose header announces more bytes than a record can have, which the framing refuses before it -//! takes any byte of its body, and the passing of the activation's deadline, which ends -//! `activation::ACTIVATION_TIMEOUT` after the repair's acknowledgement of the restore, fail the -//! activation finally, as does an application that failed, which nothing retries. Once the clone -//! is ready, the worker answers a retry of the same `Activate`, byte for byte, with the same -//! `Ready`, without applying it again, and fails the activation on another. A failed activation +//! Until the clone is ready, its session takes nothing but resets and the host's `Await`: any +//! other message, a `Stop` and a `Ping` included, a restore packet without a record, or with one +//! that the activation refuses, and the passing of the activation's deadline, which ends +//! `activation::ACTIVATION_TIMEOUT` after the repair's acknowledgement of the restore, whether +//! the record applied or not, so that a clone whose host never asks ends, fail the activation +//! finally, as does an application that failed, which nothing retries. Once the clone is ready, +//! the worker answers each `Await`, in any session, with the same `Ready`, without applying the +//! record again. A failed activation //! ends the worker, and with it the VM, with the status of `nvx::protocol::status` of the //! failure's class, from 50 to 58, as `activation::failure_status` selects it, once the worker //! dropped its executions and wrote what it queued, for at most [`UNACTIVATED_DRAIN`], so that a @@ -142,8 +148,8 @@ //! whatever the stream does then. A broker that closes the stream before the clone is ready ends //! it the same way, with [`SESSION_STATUS`], as [`Worked::Abandoned`] tells, and once the clone is //! ready, as the worker of a VM that boots cold ends. The worker of a VM that boots cold, which -//! has no activation, refuses an `Activate` as it refuses any message that it does not take in -//! its state. +//! has no activation, refuses an `Await` as it refuses any message that it does not take in its +//! state. //! //! # The mapping table and the start //! @@ -480,7 +486,7 @@ pub trait RootOperations: EntryOperations { /// blocking, and returns how it ended, as a problem, or `None` while it runs, or without one. fn announcer_ended(&mut self) -> Option; - /// Applies `record`, the record of the Activate that a clone's activation accepted, to the + /// Applies `record`, the activation record that a clone's activation accepted, to the /// clone, as `nvx::guest::clone::apply` does, with `template`, the inputs of the root's /// assembly that the clone's template captured, by `deadline`, the activation's, if any, until /// which it waits for each device at the latest, and keeps the root that it assembled, with the @@ -580,12 +586,12 @@ pub fn work( /// activation fails, or the broker closes the stream. /// /// `activation` is `nvx::guest::clone::Activation::new` of the clone that the worker repaired, -/// behind the fence that its capture's barrier handed over closed, in its transport bootstrap. +/// behind the fence that its capture's barrier handed over closed, pending. /// `session` is the guest's role of the session that the template's VM attached on the same tty /// before its capture, and whose first epoch it acknowledged, which the reset that the restore /// queued starts again: the worker attaches no session. `template` holds the inputs of the root's /// assembly that the template captured, with which the worker [applies](RootOperations::apply) -/// the record of the host's `Activate`. `acknowledged` is when the repair acknowledged the +/// the record of its restore packet. `acknowledged` is when the repair acknowledged the /// restore, `nvx::guest::clone::Repaired::acknowledged_at`, if it did, from which the /// `activation::ACTIVATION_PROBE` reports the events of the clone, with `repair_delay`, how long /// the worker waited before the repair, for tests, if the template set a delay. @@ -626,13 +632,19 @@ enum Phase { } impl Phase { - /// Takes the session's reset, which the connection acknowledged already, and returns how the - /// worker ends once the reset ended the session: once it failed a clone's activation. - fn reset(&mut self) -> Option { + /// Takes the session's reset, which `connection` acknowledged already, and returns how the + /// worker ends once the reset ended the session: once a clone's activation, which the first + /// reset starts, through `operations`, failed. + #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] + fn reset( + &mut self, + operations: &mut dyn RootOperations, + connection: &mut Connection, + ) -> Option { match self { Self::Configuring | Self::Ready(_) => None, #[cfg(target_arch = "x86_64")] - Self::Clone(state) => state.reset(), + Self::Clone(state) => state.reset(operations, connection), } } @@ -648,19 +660,6 @@ impl Phase { } } - /// Returns how the worker ends once the session's receipt failed with `error`, if its phase - /// takes the failure: a clone's activation takes an `Activate` that the framing refused - /// unread, as longer than a record can be, which fails it. Otherwise the error ends the - /// session. - #[cfg_attr(not(target_arch = "x86_64"), allow(unused_variables))] - fn refused(&mut self, error: &ServeError) -> Option { - match self { - Self::Configuring | Self::Ready(_) => None, - #[cfg(target_arch = "x86_64")] - Self::Clone(state) => state.refused(error), - } - } - /// Completes what the phase defers until the session took everything that it received, and /// returns the worker's step then, if it deferred anything: a clone's activation, with the /// application that ran meanwhile, through `operations`, after which `executions` run in the @@ -736,16 +735,7 @@ fn serve( loop { connection.flush()?; while !connection.is_overloaded() { - let received = match connection.receive() { - Ok(received) => received, - Err(error) => match phase.refused(&error) { - Some(worked) => { - executions.clear(); - return end(&mut connection, worked); - } - None => return Err(error), - }, - }; + let received = connection.receive()?; let message = match received { // The session took everything that it received: a clone's activation completes // with the application that ran meanwhile, if any. @@ -767,7 +757,7 @@ fn serve( } Some(Received::Reset) => { executions.reset(Instant::now()); - if let Some(worked) = phase.reset() { + if let Some(worked) = phase.reset(operations, &mut connection) { executions.clear(); return end(&mut connection, worked); } diff --git a/tests/runtime-adapter/src/worker/activation.rs b/tests/runtime-adapter/src/worker/activation.rs index 364b8090b..55f8b44cb 100644 --- a/tests/runtime-adapter/src/worker/activation.rs +++ b/tests/runtime-adapter/src/worker/activation.rs @@ -1,7 +1,8 @@ //! The activation of a clone, which a restore of a template's snapshot started, and which the //! worker repaired, on x86_64, whose time ABI a clone needs: the worker takes the clone's own -//! configuration from the host in one `Activate`, through the clone's -//! `nvx::guest::clone::Activation`, and then serves the host's order as after a cold boot, as the +//! configuration from the activation record of its restore packet, through the clone's +//! `nvx::guest::clone::Activation`, answers the host's `Await` with `Ready` once the clone is +//! ready, and then serves the host's order as after a cold boot, as the //! [worker's documentation](super#clones) describes. //! //! The activation's failures are final, and end the worker, and with it the VM, with the status of @@ -17,9 +18,13 @@ //! nothing; //! - `repair-delay`: how long, in microseconds, the worker waited before the repair, as the //! template's `template::REPAIR_DELAY_FILE` asked, for tests, and `none` without the file; -//! - `reset`: when the clone took the reset that the restore queued; -//! - `message`: when the host's first message came, which must be the clone's `Activate`; -//! - `ready`: when the clone was ready, once the record applied; +//! - `reset`: when the clone took the reset that the restore queued, at which it started to apply +//! the record of its restore packet; +//! - `message`: when the host's first message came, which must be its `Await`; +//! - `applied`: when the application of the record completed, which may be before the host's +//! first message; +//! - `ready`: when the clone was ready: once the record applied and the host asked for its +//! Ready; //! - `started`: when the clone's sandbox started; //! - `execution`: when the start of the host's first execution came. //! @@ -32,8 +37,8 @@ use std::time::{Duration, Instant}; use nvx::activation::{self, ActivationRecord, ReadyActivation}; use nvx::guest::clone::{ - Activation, ActivationAction, ActivationCause, ActivationFailure, ActivationInput, ApplyError, - RootTemplate, + Activation, ActivationAction, ActivationCause, ActivationFailure, ActivationInput, + ActivationState, ApplyError, RootTemplate, }; use nvx::guest::entry::Publication; use nvx::protocol::status; @@ -44,16 +49,16 @@ use super::executions::{Executions, Sandbox}; use super::order::Order; use super::{RootOperations, Worked, serve_root}; use crate::GUEST_BUILD; -use crate::frame::{FrameError, Message}; +use crate::frame::Message; use crate::guest::{ServeError, Step}; use crate::probe::{Facts, Probe}; use crate::root::Ready; /// How long a clone's activation may take, from the repair's acknowledgement of the restore until -/// the clone is ready: long enough for the host to connect once the restore is ready, and for the +/// the clone is ready: long enough for the reset that the restore queued to come, and for the /// application of the record to wait for each block device as long as the template's default -/// configuration allows. The application's waits end by then at the latest, and a clone that the -/// host did not activate by then ends. +/// configuration allows. The application's waits end by then at the latest, and a clone that is +/// not ready by then ends. pub const ACTIVATION_TIMEOUT: Duration = Duration::from_secs(120); /// The probe that the worker of a clone answers with when the events of the clone's activation and @@ -67,11 +72,10 @@ pub const ACTIVATION_PROBE: &str = "activation"; /// /// - [`ACTIVATION_DEADLINE`](status::ACTIVATION_DEADLINE): the deadline passed before the clone /// was ready; -/// - [`ACTIVATION_PROTOCOL`](status::ACTIVATION_PROTOCOL): the host broke the restricted session: -/// it reset the session again, or sent a request before the clone was ready, or an Activate of -/// another transaction, or one that changes the accepted record; -/// - [`ACTIVATION_RECORD`](status::ACTIVATION_RECORD): the Activate's record does not decode, -/// breaks a limit, or is a cold boot's; +/// - [`ACTIVATION_PROTOCOL`](status::ACTIVATION_PROTOCOL): the host sent a request other than its +/// `Await` before the clone was ready; +/// - [`ACTIVATION_RECORD`](status::ACTIVATION_RECORD): the restore packet carried no activation +/// record, or its record does not decode, breaks a limit, or is a cold boot's; /// - [`ACTIVATION_RESOURCES`](status::ACTIVATION_RESOURCES): the record expects other resources /// than the repaired clone has; /// - [`ACTIVATION_ROOT`](status::ACTIVATION_ROOT): the root's assembly failed, the scratch's upper @@ -89,18 +93,18 @@ pub const ACTIVATION_PROBE: &str = "activation"; /// announcer of the workload's listeners, which the network publishes, cannot start, as /// [`ApplyError::Network`] tells. The worker also ends a clone whose announcer ended with it, /// whether the clone was ready or not, outside the activation; -/// - [`ACTIVATION_ORDER`](status::ACTIVATION_ORDER): the worker's own order completed an -/// activation that did not run, or started the sandbox before the clone was ready, or again. +/// - [`ACTIVATION_ORDER`](status::ACTIVATION_ORDER): the worker's own order accepted the record +/// again, completed an activation that did not run, or started the sandbox before the clone was +/// ready, or again. /// /// A cause that this build of the adapter does not know ends it with `ACTIVATION_PROTOCOL`. pub fn failure_status(failure: &ActivationFailure) -> u8 { match failure.cause() { ActivationCause::Expired => status::ACTIVATION_DEADLINE, - ActivationCause::Reset - | ActivationCause::Request - | ActivationCause::Transaction { .. } - | ActivationCause::Changed => status::ACTIVATION_PROTOCOL, - ActivationCause::Record(_) | ActivationCause::Cold => status::ACTIVATION_RECORD, + ActivationCause::Request => status::ACTIVATION_PROTOCOL, + ActivationCause::Missing | ActivationCause::Record(_) | ActivationCause::Cold => { + status::ACTIVATION_RECORD + } ActivationCause::Resources { .. } => status::ACTIVATION_RESOURCES, ActivationCause::Failed(error) => match error.downcast_ref::() { Some(ApplyError::Gpt(_) | ApplyError::Assembly(_)) => status::ACTIVATION_ROOT, @@ -119,7 +123,8 @@ pub fn failure_status(failure: &ActivationFailure) -> u8 { /// A clone's activation, as the worker's session drives it, with the inputs of the root's /// assembly that the clone's template captured, the application that awaits the activation's -/// completion, if any, and the worker's order once the clone is ready. +/// completion, if any, the host's `Await`s that joined it, and the worker's order once the record +/// applied. pub(super) struct CloneState { /// The clone's activation, behind the worker's fence of workload admission. activation: Activation, @@ -128,14 +133,17 @@ pub(super) struct CloneState { /// The application of the accepted record, once it ran, until the worker /// [settles](Self::settle) the activation with its result. applied: Option, - /// The worker's order, once the clone is ready. + /// The number of the host's `Await`s of the current session that joined the activation, which + /// its completion answers. + joined: usize, + /// The worker's order, once the record applied, which serves the host once the clone is ready. order: Option, /// When the events of the clone's activation and order happened. timeline: Timeline, } -/// The application of the record of the Activate that a clone's activation accepted, which ran, -/// and which awaits the activation's completion. +/// The application of the record that a clone's activation accepted, which ran, and which awaits +/// the activation's completion. struct Applied { /// Whether the application succeeded, or why it failed. result: Result<(), ApplyError>, @@ -146,16 +154,12 @@ struct Applied { /// The publication of the listeners of every execution: that of the record's network, and /// unpublished without one. publication: Publication, - /// The number of retries of the Activate that joined the application, which its completion - /// answers too. - joined: usize, } impl CloneState { - /// Returns the state of the clone whose activation is `activation`, in its transport - /// bootstrap, whose template captured `template`, whose worker waited `repair_delay` before - /// the repair, if the template set a delay, and whose repair acknowledged the restore at - /// `acknowledged`, if it did. + /// Returns the state of the clone whose activation is `activation`, pending, whose template + /// captured `template`, whose worker waited `repair_delay` before the repair, if the template + /// set a delay, and whose repair acknowledged the restore at `acknowledged`, if it did. pub(super) fn new( activation: Activation, template: RootTemplate, @@ -166,6 +170,7 @@ impl CloneState { activation, template, applied: None, + joined: 0, order: None, timeline: Timeline { acknowledged, @@ -175,15 +180,37 @@ impl CloneState { } } - /// Takes the session's reset, which the connection acknowledged already: the one that the - /// restore queued, which ends the activation's transport bootstrap, or a later one once the - /// clone is ready, as when the host attaches again. Returns how the worker ends once the reset - /// failed the activation: another reset before the clone is ready. - pub(super) fn reset(&mut self) -> Option { + /// Takes the session's reset, which `connection` acknowledged already: the one that the + /// restore queued, or a later one, as when the host attaches again, which abandons the `Await`s + /// of the earlier session. At the first, the worker activates the clone: it writes what the + /// stream takes of the acknowledgement first, since OpenVMM's broker attaches the host only + /// once the guest acknowledged the reset, so that the host's attachment and the application of + /// the record overlap, then accepts the record of the clone's restore packet, and applies it + /// through `operations`, as [`apply`](Self::apply) describes. Returns how the worker ends once + /// the activation failed: a record that the activation refuses, or a deadline that passed. + pub(super) fn reset( + &mut self, + operations: &mut dyn RootOperations, + connection: &mut Connection, + ) -> Option { let now = Instant::now(); self.timeline.reset.get_or_insert(now); - let action = self.activation.input(now, ActivationInput::Reset); - action.err().map(|failure| unactivated(&failure)) + if let Err(failure) = self.activation.input(now, ActivationInput::Reset) { + return Some(unactivated(&failure)); + } + self.joined = 0; + if self.activation.state() != ActivationState::Pending { + return None; + } + // A failure of the stream fails the worker's next flush, once the session took what it + // received, so that a clone that ends before it is ready still ends with its own status. + let _ = connection.flush(); + let record = match self.activation.accept(Instant::now()) { + Ok(record) => record, + Err(failure) => return Some(unactivated(&failure)), + }; + self.apply(operations, &record); + None } /// Returns when the activation fails unless the clone is ready, if it is not. @@ -191,10 +218,10 @@ impl CloneState { self.activation.deadline() } - /// Returns whether the clone is ready: whether the activation completed, which gave the worker - /// its order. + /// Returns whether the clone is ready: whether the record applied, which gave the worker its + /// order, and the host asked for Ready. pub(super) fn is_ready(&self) -> bool { - self.order.is_some() + self.activation.established().is_some() } /// Returns how the worker ends once the activation's deadline passed before the clone was @@ -204,39 +231,17 @@ impl CloneState { ticked.err().map(|failure| unactivated(&failure)) } - /// Returns how the worker ends once the session's receipt failed with `error`, if the - /// activation takes the failure: an `Activate` whose header announced more than - /// [`MAX_ACTIVATE_BODY`](crate::frame::MAX_ACTIVATE_BODY) bytes, the most that a record has, - /// which the framing refused before it took any byte of its body. The activation takes it as - /// an `Activate` whose record has that many bytes, which fails it, with - /// [`ACTIVATION_RECORD`](status::ACTIVATION_RECORD) once the session is restricted, whether - /// the clone is ready or not. Any other failure ends the session as it ends that of a VM that - /// boots cold. - pub(super) fn refused(&mut self, error: &ServeError) -> Option { - let ServeError::Frame(FrameError::Length { kind, length }) = *error else { - return None; - }; - if kind != Message::Activate(Vec::new()).kind() { - return None; - } - let input = ActivationInput::TooLong(length); - let failure = self.activation.input(Instant::now(), input).err()?; - Some(unactivated(&failure)) - } - /// Handles `message` at `now` through the activation, and, once the clone is ready, in the /// worker's order, through `operations`, with `executions`, which queue what they answer on /// `connection`, as [`serve_root`] does once the root of a VM that boots cold is ready. /// - /// Before the clone is ready, the activation takes an `Activate`, whose record it hands back - /// once it accepted it, and which the worker applies then, through `operations`, and keeps the - /// result of until it [settles](Self::settle) the activation. Until then, the activation takes - /// what the session received meanwhile, as while it activates: a retry of the same - /// `Activate`, byte for byte, joins the application, and any other message fails the - /// activation. The worker answers an `Activate` that the activation completed, and each retry - /// that joined its application, with `Ready`, which reports the activation, as it answers a - /// retry once the clone is ready, and the [`ACTIVATION_PROBE`] itself. Once the activation - /// failed, the worker drops its executions, and ends. + /// While the record applies, the activation takes the host's `Await`, which joins it, and + /// which its completion answers with `Ready`, once the worker [settles](Self::settle) it. Once + /// the record applied, the worker answers an `Await` with `Ready` at once, the first of which + /// makes the clone ready, and it answers the [`ACTIVATION_PROBE`] itself once the clone is + /// ready, and serves every other message in its order. Before the clone is ready, any other + /// message fails the activation. Once the activation failed, the worker drops its executions, + /// and ends. pub(super) fn handle( &mut self, operations: &mut dyn RootOperations, @@ -247,7 +252,7 @@ impl CloneState { ) -> Result, ServeError> { self.timeline.message.get_or_insert(now); let input = match &message { - Message::Activate(record) => ActivationInput::Activate(record), + Message::Await => ActivationInput::Await, _ => ActivationInput::Request, }; let action = match self.activation.input(now, input) { @@ -255,17 +260,14 @@ impl CloneState { Err(failure) => return Ok(self.end(executions, &failure)), }; Ok(match action { - ActivationAction::Apply(record) => { - self.apply(operations, &record); - Step::Continue(Vec::new()) - } ActivationAction::Join => { - if let Some(applied) = &mut self.applied { - applied.joined += 1; - } + self.joined += 1; Step::Continue(Vec::new()) } - ActivationAction::Ready(activation) => Step::Continue(vec![ready(activation)]), + ActivationAction::Ready(activation) => { + self.timeline.ready.get_or_insert(now); + Step::Continue(vec![ready(activation)]) + } ActivationAction::Serve => { if let Some(answer) = self.timeline.answer(&message) { return Ok(Step::Continue(vec![answer])); @@ -293,7 +295,7 @@ impl CloneState { }) } - /// Applies `record`, the record of the Activate that the activation accepted, through + /// Applies `record`, the record of the restore packet that the activation accepted, through /// `operations`, with the template's inputs, by the activation's deadline, and keeps the /// application's result, with the record's hostname, its publication of listeners, and the /// number of its mappings, until the worker [settles](Self::settle) the activation with it. @@ -320,17 +322,17 @@ impl CloneState { mappings, hostname: configuration.workload.hostname.clone(), publication, - joined: 0, }); } /// Completes the activation with the result of the application that ran, if any, once the /// worker took what the session received while it ran, which comes before `Ready`, and - /// returns the worker's step: `Ready`, which reports the activation, for the `Activate` and - /// for each retry that joined its application, once the clone is ready, after which - /// `executions` run in the cgroups that the template's bootstrap prepared, each as the identity - /// that it names, with the record's hostname and publication of listeners, and the worker's - /// order awaits the record's mappings; or the end, once the activation failed. + /// returns the worker's step: `Ready`, which reports the activation, for each `Await` of the + /// current session that joined it, which made the clone ready, and nothing without one, until + /// the host's next `Await`, once the record applied, after which `executions` run in the + /// cgroups that the template's bootstrap prepared, each as the identity that it names, with + /// the record's hostname and publication of listeners, and the worker's order awaits the + /// record's mappings; or the end, once the activation failed. pub(super) fn settle( &mut self, operations: &mut dyn RootOperations, @@ -340,14 +342,18 @@ impl CloneState { let result = applied.result.map_err(Into::into); Some(match self.activation.complete(Instant::now(), result) { Ok(activation) => { - self.timeline.ready = Some(Instant::now()); + let now = Instant::now(); + self.timeline.applied = Some(now); + if self.activation.established().is_some() { + self.timeline.ready = Some(now); + } // A clone has no workload identity of the whole sandbox: each execution names // its own. let sandbox = Sandbox::new(None, applied.publication).with_hostname(applied.hostname); executions.run_in(operations.cgroups(), sandbox); self.order = Some(Order::new(applied.mappings)); - Step::Continue(vec![ready(activation); 1 + applied.joined]) + Step::Continue(vec![ready(activation); std::mem::take(&mut self.joined)]) } Err(failure) => self.end(executions, &failure), }) @@ -380,6 +386,8 @@ struct Timeline { reset: Option, /// When the host's first message came. message: Option, + /// When the application of the record completed. + applied: Option, /// When the clone was ready. ready: Option, /// When the clone's sandbox started. @@ -423,6 +431,7 @@ impl Timeline { let events = [ ("reset", self.reset), ("message", self.message), + ("applied", self.applied), ("ready", self.ready), ("started", self.started), ("execution", self.execution), diff --git a/tests/runtime-adapter/src/worker/activation/tests.rs b/tests/runtime-adapter/src/worker/activation/tests.rs index e2ac0c156..cfca4af2e 100644 --- a/tests/runtime-adapter/src/worker/activation/tests.rs +++ b/tests/runtime-adapter/src/worker/activation/tests.rs @@ -52,32 +52,32 @@ fn record(transaction: u8, uid: u32) -> ActivationRecord { ActivationRecord::new(Some(restore), configuration).unwrap() } -/// Returns a clone's activation, in its transport bootstrap, behind a closed fence, with -/// `resources`, which the deadline ends `timeout` after now. -fn activation(resources: ExpectedResources, timeout: Duration) -> Activation { +/// Returns a clone's activation, pending, behind a closed fence, with `resources`, whose restore +/// packet carried `record`, if it carried one, which the deadline ends `timeout` after now. +fn activation( + resources: ExpectedResources, + record: Option<&[u8]>, + timeout: Duration, +) -> Activation { let mut fence = Admission::default(); fence.close().unwrap(); let now = Instant::now(); - Activation::with_resources(resources, Some(now), fence, timeout, now) + Activation::with_resources(resources, record, Some(now), fence, timeout, now) } -/// Returns an activation that took the session's reset, and is restricted. -fn restricted() -> Activation { - let mut activation = activation(RESOURCES, ACTIVATION_TIMEOUT); - let action = activation.input(Instant::now(), ActivationInput::Reset); - assert_eq!(action.unwrap(), ActivationAction::Acknowledge); - activation +/// Returns a pending activation whose restore packet carried the record of the transaction 1. +fn pending() -> Activation { + let bytes = record(1, 1000).encode(); + activation(RESOURCES, Some(&bytes), ACTIVATION_TIMEOUT) } -/// Returns an activation that accepted the record of the transaction 1. +/// Returns an activation that took the session's reset, and accepted the record of the +/// transaction 1 from its restore packet. fn activating() -> Activation { - let mut activation = restricted(); - let bytes = record(1, 1000).encode(); - let action = activation.input(Instant::now(), ActivationInput::Activate(&bytes)); - assert!( - matches!(action, Ok(ActivationAction::Apply(_))), - "{action:?}" - ); + let mut activation = pending(); + let action = activation.input(Instant::now(), ActivationInput::Reset); + assert_eq!(action.unwrap(), ActivationAction::Acknowledge); + activation.accept(Instant::now()).unwrap(); activation } @@ -93,46 +93,36 @@ fn each_class_of_failure_ends_the_clone_with_its_own_status() { let mut cases = Vec::new(); // The deadline. - let mut expired = activation(RESOURCES, Duration::ZERO); + let mut expired = activation(RESOURCES, Some(&bytes), Duration::ZERO); cases.push((failed(expired.tick(now())), status::ACTIVATION_DEADLINE)); - // The host's session: another reset, a request, another transaction, a changed record. - let mut reset = restricted(); - let failure = failed(reset.input(now(), ActivationInput::Reset)); - cases.push((failure, status::ACTIVATION_PROTOCOL)); - let mut request = restricted(); + // The host's session: a request before the clone is ready, before the activation accepted + // the record, or while it applies it. + let mut request = pending(); let failure = failed(request.input(now(), ActivationInput::Request)); + assert!(matches!(failure.cause(), ActivationCause::Request)); cases.push((failure, status::ACTIVATION_PROTOCOL)); - let early = activation(RESOURCES, ACTIVATION_TIMEOUT).input(now(), ActivationInput::Request); - cases.push((failed(early), status::ACTIVATION_PROTOCOL)); - let other = record(2, 1000).encode(); - let failure = failed(activating().input(now(), ActivationInput::Activate(&other))); - assert!(matches!( - failure.cause(), - ActivationCause::Transaction { .. } - )); - cases.push((failure, status::ACTIVATION_PROTOCOL)); - let changed = record(1, 1001).encode(); - let failure = failed(activating().input(now(), ActivationInput::Activate(&changed))); - assert!(matches!(failure.cause(), ActivationCause::Changed)); + let failure = failed(activating().input(now(), ActivationInput::Request)); cases.push((failure, status::ACTIVATION_PROTOCOL)); - // The record: one that does not decode, and a cold boot's. - let failure = failed(restricted().input(now(), ActivationInput::Activate(b"\x01"))); + // The record: none in the restore packet, one that does not decode, and a cold boot's. + let failure = failed(activation(RESOURCES, None, ACTIVATION_TIMEOUT).accept(now())); + assert!(matches!(failure.cause(), ActivationCause::Missing)); + cases.push((failure, status::ACTIVATION_RECORD)); + let mut undecodable = activation(RESOURCES, Some(b"\x01"), ACTIVATION_TIMEOUT); + let failure = failed(undecodable.accept(now())); + assert!(matches!(failure.cause(), ActivationCause::Record(_))); cases.push((failure, status::ACTIVATION_RECORD)); let cold = ActivationRecord::new(None, record(1, 1000).configuration().clone()).unwrap(); let cold = cold.encode(); - let failure = failed(restricted().input(now(), ActivationInput::Activate(&cold))); + let failure = failed(activation(RESOURCES, Some(&cold), ACTIVATION_TIMEOUT).accept(now())); assert!(matches!(failure.cause(), ActivationCause::Cold)); cases.push((failure, status::ACTIVATION_RECORD)); // The resources. - let mut more = activation( - ExpectedResources { - cpus: 4, - ..RESOURCES - }, - ACTIVATION_TIMEOUT, - ); - more.input(now(), ActivationInput::Reset).unwrap(); - let failure = failed(more.input(now(), ActivationInput::Activate(&bytes))); + let more = ExpectedResources { + cpus: 4, + ..RESOURCES + }; + let failure = failed(activation(more, Some(&bytes), ACTIVATION_TIMEOUT).accept(now())); + assert!(matches!(failure.cause(), ActivationCause::Resources { .. })); cases.push((failure, status::ACTIVATION_RESOURCES)); // The application: of the root, of the workload, of the export, of the network, and any // other. @@ -205,11 +195,15 @@ fn each_class_of_failure_ends_the_clone_with_its_own_status() { } let other = activating().complete(now(), Err("another failure of the application".into())); cases.push((failed(other), status::ACTIVATION_ROOT)); - // The worker's own order. - let failure = failed(restricted().complete(now(), Ok(()))); + // The worker's own order: a second acceptance, a completion without an application, and a + // start before the clone is ready. + let failure = failed(activating().accept(now())); + assert!(matches!(failure.cause(), ActivationCause::Completion)); + cases.push((failure, status::ACTIVATION_ORDER)); + let failure = failed(pending().complete(now(), Ok(()))); assert!(matches!(failure.cause(), ActivationCause::Completion)); cases.push((failure, status::ACTIVATION_ORDER)); - let failure = failed(restricted().start()); + let failure = failed(activating().start()); assert!(matches!(failure.cause(), ActivationCause::Start)); cases.push((failure, status::ACTIVATION_ORDER)); @@ -261,6 +255,7 @@ fn the_activation_probe_reports_the_clone_s_events_after_the_acknowledgement() { repair_delay: None, reset: Some(after(10)), message: Some(after(250)), + applied: Some(after(3_900)), ready: Some(after(4_000)), started: Some(after(4_100)), execution: None, @@ -271,6 +266,7 @@ fn the_activation_probe_reports_the_clone_s_events_after_the_acknowledgement() { ("repair-delay", "none"), ("reset", "10"), ("message", "250"), + ("applied", "3900"), ("ready", "4000"), ("started", "4100"), ("execution", "none"), diff --git a/tests/runtime-adapter/src/worker/capture.rs b/tests/runtime-adapter/src/worker/capture.rs index 11f476594..480cf52a3 100644 --- a/tests/runtime-adapter/src/worker/capture.rs +++ b/tests/runtime-adapter/src/worker/capture.rs @@ -45,9 +45,9 @@ //! [`work_clone`]: through the clone's activation, `nvx::guest::clone::Activation` of the repaired //! clone and of the fence, whose deadline ends [`ACTIVATION_TIMEOUT`] after the repair's //! acknowledgement, it answers the reset that the restore queued for the session that the snapshot -//! holds, takes the host's `Activate`, and applies its record with [`root_template`] of the -//! template's configuration and its NIC's gateway, then serves the host's order as after a cold -//! boot, as the +//! holds, applies the activation record of the clone's restore packet with [`root_template`] of +//! the template's configuration and its NIC's gateway, answers the host's `Await` with `Ready`, +//! then serves the host's order as after a cold boot, as the //! [worker's documentation](super#clones) describes. The clone ends as that session does. Once its //! activation failed, or the broker closed the stream before the clone was ready, the worker waits //! for the tty to send what it wrote for at most [`UNACTIVATED_DRAIN`], so that a tty that sends @@ -98,8 +98,8 @@ pub fn template_config() -> RootConfig { /// root, with the sizes that the configuration names, if any; how long the assembly waits for /// each; the host filesystem device, by [`FILESYSTEM_TAG`], through which the host attaches the /// export of a clone's record; and the gateway, to which the announcer of the listeners of a -/// clone whose record publishes them announces them. The record of the clone's `Activate` holds -/// the rest. +/// clone whose record publishes them announces them. The activation record of the clone's restore +/// packet holds the rest. pub fn root_template(config: &RootConfig, gateway: Option) -> RootTemplate { RootTemplate { runtime: PathBuf::from(&config.runtime), diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index 4980e9e67..f97c84260 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -24,12 +24,14 @@ //! On x86_64, the worker of a clone, which a restore of a template's snapshot started, resumes the //! session that the template's VM attached before its capture, active in the epoch of its broker's //! first reset, with the same fake root operations, whose application of an activation record -//! mirrors the guest's: the tests cover the reset that the restore queued, the one `Activate` that -//! the clone takes before it is ready, the application of its record, the `Ready` that reports the -//! activation, retries of the `Activate`, the order that the worker serves once the clone is -//! ready, the status of each class of the activation's failures, which the worker answers with -//! nothing, before any side effect where the class allows, its deadline, also once the stream -//! takes nothing more, the record's workload identity, as which every execution runs, the +//! mirrors the guest's: the tests cover the reset that the restore queued, at which the clone +//! applies the record of its restore packet, the host's `Await`s, which the clone takes before it +//! is ready, which join the application, or make the clone ready once the record applied, the +//! `Ready` that reports the activation, to each `Await` of any session, the order that the worker +//! serves once the clone is ready, the status of each class of the activation's failures, which +//! the worker answers with nothing, before any side effect where the class allows, its deadline, +//! which ends a clone whose host never asks, also once the stream takes nothing more, the record's +//! workload identity, as which every execution runs, the //! mapping table of the record's export, which the worker applies from the export that the //! application mounted, while it leaves the endpoint of the network's proxy to the host, and the //! announcer of the listeners of a record whose network publishes them, whose end ends the clone, @@ -770,7 +772,8 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { }; let cases = [ (header([0; 4], 0), FrameError::Kind(0)), - (header([0; 4], 28), FrameError::Kind(28)), + (header([0; 4], 27), FrameError::Kind(27)), + (header([0; 4], 29), FrameError::Kind(29)), ( header(too_long, 3), FrameError::Length { @@ -791,8 +794,8 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { // The guest agent answers requests, and refuses the messages that only it sends, and those // about a layered root, which its VM does not have: its configuration, a part of its mapping - // table, and the start of its workloads, a clone's activation, and executions, which only the - // worker of a layered root runs. + // table, and the start of its workloads, the request for a clone's Ready, and executions, + // which only the worker of a layered root runs. let refused_messages = [ Message::Pong, Message::Stopped, @@ -806,7 +809,7 @@ fn the_guest_agent_refuses_messages_that_its_framing_does_not_allow() { Message::RootConfig(RootConfig::default().encode()), Message::Maps(maps_part()), Message::Start, - Message::Activate(Vec::new()), + Message::Await, Message::ExecStart(Vec::new()), Message::ExecCredit(Vec::new()), Message::ExecTerminal(Vec::new()), @@ -1006,13 +1009,13 @@ fn the_guest_binary_serves_the_session_on_its_standard_streams() { // The host sends a malformed message. let (mut broker, child) = start_binary(); broker.attach(FIRST); - broker.send_data(&[0, 0, 0, 0, 28], OUTER_MAX_PAYLOAD); + broker.send_data(&[0, 0, 0, 0, 29], OUTER_MAX_PAYLOAD); let output = child.finished(); assert_eq!(output.status.code(), Some(1), "{output:?}"); let diagnostic = String::from_utf8_lossy(&output.stderr); assert_eq!( diagnostic, - "runtime-adapter-guest: unknown message kind 28\n" + "runtime-adapter-guest: unknown message kind 29\n" ); } @@ -2134,14 +2137,14 @@ fn the_worker_accepts_only_its_configuration_or_a_stop_before_its_root() { assert_eq!(root.calls, []); // Any other message ends it with an error, as do the messages that only the guest sends, and - // a clone's activation, which a VM that boots cold does not take. + // the request for a clone's Ready, which a VM that boots cold does not take. let refused = [ Message::Ping, Message::Echo(b"early".to_vec()), Message::Probe(b"mounts".to_vec()), Message::Maps(maps_part()), Message::Start, - Message::Activate(Vec::new()), + Message::Await, Message::Pong, ready(), Message::Report(Vec::new()), @@ -2160,9 +2163,10 @@ fn the_worker_accepts_only_its_configuration_or_a_stop_before_its_root() { assert_eq!(root.calls, [], "{message:?}"); } - // Once the root is ready, a second configuration ends it with an error too, as does a clone's - // activation, and the worker leaves the root that it assembled to its operations. - for refused in [configuration(), Message::Activate(Vec::new())] { + // Once the root is ready, a second configuration ends it with an error too, as does the + // request for a clone's Ready, and the worker leaves the root that it assembled to its + // operations. + for refused in [configuration(), Message::Await] { let (mut broker, working) = start_worker(FakeRoot::booting(&layout())); broker.attach(FIRST); let replies = broker.exchange(&[configuration()], OUTER_MAX_PAYLOAD, 1); @@ -4114,8 +4118,9 @@ fn the_worker_runs_executions_on_a_tty() { /// The worker of a clone, which a restore of a template's snapshot started, on x86_64, whose time /// ABI a clone needs. It resumes the session that the template's VM attached before its capture, -/// answers the reset that the restore queued, takes nothing but the host's `Activate` until the -/// clone is ready, applies its record, and then serves the host's order, as after a cold boot. +/// answers the reset that the restore queued, applies the record of its restore packet then, takes +/// nothing but resets and the host's `Await` until the clone is ready, which it is once the record +/// applied and the host asked for `Ready`, and then serves the host's order, as after a cold boot. #[cfg(target_arch = "x86_64")] mod clones { use std::net::{IpAddr, Ipv4Addr}; @@ -4129,7 +4134,6 @@ mod clones { use nvx::guest::clone::{Activation, AdmissionFence, RootTemplate}; use nvx::protocol::nvxs::{GuestEvent, GuestSession}; use nvx::protocol::status; - use runtime_adapter::frame::MAX_ACTIVATE_BODY; use runtime_adapter::worker::UNACTIVATED_DRAIN; use runtime_adapter::worker::activation::{ACTIVATION_PROBE, ACTIVATION_TIMEOUT}; use runtime_adapter::worker::capture::{self, Admission}; @@ -4159,7 +4163,7 @@ mod clones { /// The IPv4 gateway of the NIC of the clones' template. const GATEWAY: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1); - /// Returns the record of a clone's Activate of the transaction `transaction`, whose workload + /// Returns the activation record of a clone of the transaction `transaction`, whose workload /// runs as `identity`, if any: an EROFS image over a scratch, without an export, a mapping, or /// a network, which expects [`RESOURCES`]. fn record(transaction: u8, identity: Option<(u32, u32)>) -> ActivationRecord { @@ -4237,13 +4241,25 @@ mod clones { }) } - /// Returns the activation of a clone that the worker repaired, with [`RESOURCES`], behind the + /// Returns the activation of a clone that the worker repaired, with [`RESOURCES`], whose + /// restore packet carried the activation record `packet`, if it carried one, behind the /// worker's fence, which the capture's barrier handed over closed, and whose deadline ends /// `timeout` after `acknowledged`, as from the repair's acknowledgement of the restore. - fn repaired(acknowledged: Instant, timeout: Duration) -> Activation { + fn repaired( + packet: Option<&[u8]>, + acknowledged: Instant, + timeout: Duration, + ) -> Activation { let mut fence = Admission::default(); fence.close().unwrap(); - Activation::with_resources(RESOURCES, Some(acknowledged), fence, timeout, acknowledged) + Activation::with_resources( + RESOURCES, + packet, + Some(acknowledged), + fence, + timeout, + acknowledged, + ) } /// Returns the inputs of the activation of the clones that their template captured: those of @@ -4270,34 +4286,46 @@ mod clones { session } - /// Starts a clone's worker in a thread, with the root operations `root`, on one end of a - /// socket pair, and returns the broker on the other end with the thread. The worker resumes - /// the session that the template's VM attached, so it sends no attachment, and its - /// activation's deadline ends `timeout` from now. - fn start_clone(root: FakeRoot, timeout: Duration) -> (Broker, Working) { - start_clone_after(root, timeout, |_| {}) + /// Starts a clone's worker in a thread, with the root operations `root`, whose restore packet + /// carried `record`, on one end of a socket pair, and returns the broker on the other end with + /// the thread. The worker resumes the session that the template's VM attached, so it sends no + /// attachment, and the activation's deadline ends `timeout` from now. + fn start_clone( + root: FakeRoot, + record: &ActivationRecord, + timeout: Duration, + ) -> (Broker, Working) { + start_clone_after(root, Some(record.encode()), timeout, |_| {}) } - /// Starts a clone's worker as [`start_clone`] does, once `ahead` sent what the broker sends - /// before the worker runs, which the worker then takes at once, however late its thread runs. + /// Starts a clone's worker as [`start_clone`] does, whose restore packet carried the activation + /// record `packet`, if it carried one, once `ahead` sent what the broker sends before the + /// worker runs, which the worker then takes at once, however late its thread runs. fn start_clone_after( root: FakeRoot, + packet: Option>, timeout: Duration, ahead: impl FnOnce(&mut Broker), ) -> (Broker, Working) { let (broker, guest) = UnixStream::pair().unwrap(); let mut broker = Broker::new(broker); ahead(&mut broker); - (broker, run_clone(guest, root, timeout)) + (broker, run_clone(guest, root, packet, timeout)) } - /// Runs a clone's worker in a thread, with the root operations `root`, on `guest`, the guest's - /// end of the stream, and returns the thread, as [`start_clone`] does. The clone's repair - /// acknowledged the restore now. - fn run_clone(guest: UnixStream, root: FakeRoot, timeout: Duration) -> Working { + /// Runs a clone's worker in a thread, with the root operations `root`, whose restore packet + /// carried the activation record `packet`, if it carried one, on `guest`, the guest's end of + /// the stream, and returns the thread, as [`start_clone`] does. The clone's repair acknowledged + /// the restore now. + fn run_clone( + guest: UnixStream, + root: FakeRoot, + packet: Option>, + timeout: Duration, + ) -> Working { let session = captured_session(); let acknowledged = Instant::now(); - let activation = repaired(acknowledged, timeout); + let activation = repaired(packet.as_deref(), acknowledged, timeout); thread::spawn(move || { let mut root = root; let worked = worker::work_clone( @@ -4351,14 +4379,9 @@ mod clones { } } - /// Returns the host's Activate of `record`. - fn activate(record: &ActivationRecord) -> Message { - Message::Activate(record.encode()) - } - - /// Returns the worker's answer to the Activate of `record` once it activated the clone: its - /// readiness, with the guest agent's build, and the activation, which reports that it applied, - /// with the record's transaction. + /// Returns the worker's answer to the host's `Await` once it activated the clone with + /// `record`: its readiness, with the guest agent's build, and the activation, which reports + /// that it applied, with the record's transaction. fn activated(record: &ActivationRecord) -> Message { let ready = Ready::new(GUEST_BUILD).unwrap(); Message::Ready(ready.with_activation(ReadyActivation::of(record)).encode()) @@ -4386,19 +4409,21 @@ mod clones { impl Host { /// Starts a clone's worker with the root operations `root`, and returns the host, once the - /// worker activated the clone with `record`, which announces no mapping, and started the - /// sandbox. The host waits for Ready before it starts the sandbox. + /// worker activated the clone with `record`, the record of its restore packet, which + /// announces no mapping, and started the sandbox. The host waits for Ready, which it asks + /// for, before it starts the sandbox. fn activated(root: FakeRoot, record: &ActivationRecord) -> Self { Self::mapped(root, record, &[]) } /// Starts a clone's worker with the root operations `root`, and returns the host, once the - /// worker activated the clone with `record`, applied `parts`, the parts of the mapping - /// table that the record announces, in order, and started the sandbox. + /// worker activated the clone with `record`, the record of its restore packet, applied + /// `parts`, the parts of the mapping table that the record announces, in order, and started + /// the sandbox. fn mapped(root: FakeRoot, record: &ActivationRecord, parts: &[Vec]) -> Self { - let (mut broker, working) = start_clone(root, ACTIVATION_TIMEOUT); + let (mut broker, working) = start_clone(root, record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); - let replies = broker.exchange(&[activate(record)], OUTER_MAX_PAYLOAD, 1); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(record)]); let mut messages = parts.iter().cloned().map(Message::Maps).collect::>(); messages.push(Message::Start); @@ -4417,13 +4442,15 @@ mod clones { } #[test] - fn a_clone_answers_the_restore_s_reset_and_activates_with_the_record_of_its_activate() { - let (mut broker, working) = start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + fn a_clone_activates_with_the_record_of_its_restore_packet_and_answers_the_host_s_await() { + let record = record(1, Some(WORKLOAD)); + let root = FakeRoot::booting(&layout()); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); // The worker sends no attachment: it acknowledges the reset that the restore queued, with - // its receive window, first. + // its receive window, first, and then applies the record of its restore packet, whatever + // the host sends. The host asks for the clone's Ready with its `Await`. broker.reset(RESTORED); - let record = record(1, Some(WORKLOAD)); - let replies = broker.exchange(&[activate(&record)], OUTER_MAX_PAYLOAD, 1); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(&record)]); let Message::Ready(body) = &replies[0] else { panic!("{replies:?}"); @@ -4432,26 +4459,25 @@ mod clones { assert_eq!(ready.build(), GUEST_BUILD); assert_eq!(ready.activation(), Some(ReadyActivation::of(&record))); - // A retry of the Activate, byte for byte, gets the same Ready, without a second - // application, in records of any size, and after a later reset, as once the host attached - // again. - let replies = broker.exchange(&[activate(&record)], 7, 1); + // Another `Await` gets the same Ready, without a second application, in records of any + // size, and after a later reset, as once the host attached again. + let replies = broker.exchange(&[Message::Await], 2, 1); assert_eq!(replies, [activated(&record)]); broker.settle(); broker.reset(Identity { epoch: 2, ..RESTORED }); - let replies = broker.exchange(&[activate(&record)], OUTER_MAX_PAYLOAD, 1); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(&record)]); // The worker then serves the host's order, as after a cold boot: the record announces no - // mapping, so the sandbox starts at once, and a retry still gets the same Ready. + // mapping, so the sandbox starts at once, and an `Await` still gets the same Ready. let messages = [ Message::Ping, Message::Start, Message::Probe(b"mounts".to_vec()), - activate(&record), + Message::Await, Message::Stop, ]; let expected = [ @@ -4479,10 +4505,12 @@ mod clones { } #[test] - fn a_clone_takes_nothing_but_its_activate_before_it_is_ready() { - // Any other message fails the activation, finally, with the status of the host's session, - // a stop and a ping included, as do the messages that only the guest sends. The worker - // answers nothing, and applies, mounts, and spawns nothing. + fn a_clone_takes_nothing_but_resets_and_awaits_before_it_is_ready() { + // Before the host's first `Await`, while the record applies or once it applied, any other + // message fails the activation, finally, with the status of the host's session, a stop and + // a ping included, as do the messages that only the guest sends. The worker answers + // nothing, and mounts and spawns nothing more. + let record = record(1, Some(WORKLOAD)); let refused = [ Message::Ping, Message::Stop, @@ -4497,55 +4525,63 @@ mod clones { Message::Started, ]; for message in refused { - let (mut broker, working) = - start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + let root = FakeRoot::booting(&layout()); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); broker.send_data(&message.encode().unwrap(), OUTER_MAX_PAYLOAD); let root = unactivated(&mut broker, working, status::ACTIVATION_PROTOCOL); - assert_eq!(root.calls, [], "{message:?}"); + assert_eq!(root.calls, [applied(&record)], "{message:?}"); } - // So does another reset, which the worker acknowledges first. - let (mut broker, working) = start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + // A later reset, as once the host attached again, which the worker acknowledges, abandons + // the `Await`s of the earlier session, which no Ready answers, and the clone, whose record + // applied meanwhile, answers the `Await` of the new session, which makes it ready. + let (root, gate) = FakeRoot::booting(&layout()).gated(); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); - broker.reset(Identity { + broker.send_data(&Message::Await.encode().unwrap(), OUTER_MAX_PAYLOAD); + broker.reset_ahead(Identity { epoch: 2, ..RESTORED }); - let root = unactivated(&mut broker, working, status::ACTIVATION_PROTOCOL); - assert_eq!(root.calls, []); + gate.send(()).unwrap(); + broker.acknowledged(); + let replies = broker.exchange(&[Message::Await, Message::Stop], OUTER_MAX_PAYLOAD, 2); + assert_eq!(replies, [activated(&record), Message::Stopped]); + let (root, worked) = joined(working); + assert_eq!(worked.unwrap(), Worked::Stopped); + assert_eq!(root.calls, [applied(&record), Call::Teardown]); } #[test] fn a_clone_s_activation_fails_finally_with_the_status_of_the_failure_s_class() { - // The activation refuses a record that does not decode, a cold boot's, and one that - // expects other resources than the repair found, before anything has a side effect. + // The activation refuses a restore packet without a record, and a record that does not + // decode, a cold boot's, and one that expects other resources than the repair found, at + // the restore's reset, before anything has a side effect. let clone = record(1, None); let cold = ActivationRecord::new(None, clone.configuration().clone()).unwrap(); let more = changed(&clone, |configuration| configuration.resources.memory *= 2); let cases = [ - (Message::Activate(Vec::new()), status::ACTIVATION_RECORD), - ( - Message::Activate(b"\x01".to_vec()), - status::ACTIVATION_RECORD, - ), - (activate(&cold), status::ACTIVATION_RECORD), - (activate(&more), status::ACTIVATION_RESOURCES), + (None, status::ACTIVATION_RECORD), + (Some(Vec::new()), status::ACTIVATION_RECORD), + (Some(b"\x01".to_vec()), status::ACTIVATION_RECORD), + (Some(cold.encode()), status::ACTIVATION_RECORD), + (Some(more.encode()), status::ACTIVATION_RESOURCES), ]; - for (message, expected) in cases { + for (packet, expected) in cases { + let root = FakeRoot::booting(&layout()); let (mut broker, working) = - start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + start_clone_after(root, packet.clone(), ACTIVATION_TIMEOUT, |_| {}); broker.reset(RESTORED); - broker.send_data(&message.encode().unwrap(), OUTER_MAX_PAYLOAD); let root = unactivated(&mut broker, working, expected); - assert_eq!(root.calls, [], "{message:?}"); + assert_eq!(root.calls, [], "{packet:?}"); } // The application fails once the kernel refused to reread a GPT disk's partition table, // and refuses a machine ID that it cannot generate, and a hostname or machine ID that it // cannot set for every execution, an export that it cannot mount, a resolver file that it // cannot write, and an announcer of the workload's listeners that it cannot start, before - // Ready. + // Ready, without the host's `Await`. let gpt = changed(&clone, |configuration| { configuration.image = activation::Image::Gpt { binding: activation::Identity::from_bytes([0x11; 32]), @@ -4589,116 +4625,89 @@ mod clones { ), ]; for (record, root, expected) in cases { - let (mut broker, working) = start_clone(root, ACTIVATION_TIMEOUT); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); - broker.send_data(&activate(&record).encode().unwrap(), OUTER_MAX_PAYLOAD); let root = unactivated(&mut broker, working, expected); assert_eq!(root.calls, [applied(&record)]); } } #[test] - fn a_clone_fails_on_an_activate_longer_than_a_record_before_it_takes_its_body() { - // The framing refuses the header of an Activate that announces more bytes than a record - // can have before it takes any byte of its body, and the activation takes it as a record - // that long: it fails with the record's status, whether the clone is ready or not, and the - // worker answers nothing more. - let accepted = record(1, Some(WORKLOAD)); - let kind = activate(&accepted).kind(); - for length in [MAX_ACTIVATE_BODY + 1, usize::try_from(u32::MAX).unwrap()] { - let mut header = u32::try_from(length).unwrap().to_le_bytes().to_vec(); - header.push(kind); - for ready in [false, true] { - let (mut broker, working) = - start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); - broker.reset(RESTORED); - let mut calls = Vec::new(); - if ready { - let replies = broker.exchange(&[activate(&accepted)], OUTER_MAX_PAYLOAD, 1); - assert_eq!(replies, [activated(&accepted)]); - calls.push(applied(&accepted)); - } - broker.send_data(&header, OUTER_MAX_PAYLOAD); - let (root, worked) = joined(working); - let worked = worked.unwrap(); - let Worked::Unactivated { - status: code, - problem, - } = &worked - else { - panic!("{length}, ready {ready}: {worked:?}"); - }; - assert_eq!(*code, status::ACTIVATION_RECORD, "{problem}"); - assert!(problem.contains(&format!("{length} bytes")), "{problem}"); - while let Some(record) = broker.read() { - broker.check(&record); - assert_eq!(broker.accept(&record), [], "{problem}"); - } - assert_eq!(root.calls, calls, "{problem}"); - } - } - } - - #[test] - fn a_clone_that_the_host_does_not_activate_in_time_ends_with_the_deadline_s_status() { - // Whether the restore's reset came or not, or part of an Activate, the deadline ends the - // activation, before anything has a side effect. The broker sends what it sends before the - // worker runs, which then takes it at once: a worker that runs only once the deadline - // passed still acknowledges the reset, and fails with the deadline's status. + fn a_clone_that_is_not_ready_in_time_ends_with_the_deadline_s_status() { + // Without the restore's reset, the clone does not activate, and the deadline ends the + // activation, before anything has a side effect, as it does once it passed when the reset + // came, which the worker still acknowledges. The broker sends what it sends before the + // worker runs, which then takes it at once. let record = record(1, Some(WORKLOAD)); - let bytes = activate(&record).encode().unwrap(); - let cases: [Option<&[u8]>; 3] = [None, Some(&[]), Some(&bytes[..bytes.len() / 2])]; - for sent in cases { - let deadline = Duration::from_millis(200); + for (reset, deadline) in [(false, Duration::from_millis(200)), (true, Duration::ZERO)] { let root = FakeRoot::booting(&layout()); - let (mut broker, working) = start_clone_after(root, deadline, |broker| { - if let Some(sent) = sent { + let packet = Some(record.encode()); + let (mut broker, working) = start_clone_after(root, packet, deadline, |broker| { + if reset { broker.reset_ahead(RESTORED); - broker.send_data(sent, OUTER_MAX_PAYLOAD); } }); - if sent.is_some() { + if reset { broker.acknowledged(); } let root = unactivated(&mut broker, working, status::ACTIVATION_DEADLINE); - assert_eq!(root.calls, [], "{sent:?}"); + assert_eq!(root.calls, [], "reset {reset}"); } - // Once the clone is ready, the deadline ends nothing. + // A clone whose record applied, at the restore's reset, is not ready until the host asks + // for its `Ready`: one whose host never asks ends at the deadline too. + let deadline = Duration::from_secs(1); + let started = Instant::now(); + let root = FakeRoot::booting(&layout()); + let (mut broker, working) = start_clone(root, &record, deadline); + broker.reset(RESTORED); + let root = unactivated(&mut broker, working, status::ACTIVATION_DEADLINE); + assert!(started.elapsed() >= deadline); + assert_eq!(root.calls, [applied(&record)]); + + // The clone applies its record before the host's first message, its `Await`, which then + // makes it ready at once, and once it is ready, the deadline ends nothing. let deadline = Duration::from_secs(2); let root = FakeRoot::booting(&layout()); - let (mut broker, working) = start_clone_after(root, deadline, |broker| { - broker.reset_ahead(RESTORED); - broker.send_data(&bytes, OUTER_MAX_PAYLOAD); - }); - broker.acknowledged(); - let mut replies = Vec::new(); - while replies.is_empty() { - replies.extend(broker.take()); - } + let (mut broker, working) = start_clone(root, &record, deadline); + broker.reset(RESTORED); + thread::sleep(Duration::from_millis(500)); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(&record)]); - thread::sleep(deadline + Duration::from_millis(500)); - let replies = broker.exchange(&[Message::Ping, Message::Stop], OUTER_MAX_PAYLOAD, 2); - assert_eq!(replies, [Message::Pong, Message::Stopped]); - let (_, worked) = joined(working); + thread::sleep(deadline); + let probe = Message::Probe(ACTIVATION_PROBE.as_bytes().to_vec()); + let messages = [probe, Message::Ping, Message::Stop]; + let replies = broker.exchange(&messages, OUTER_MAX_PAYLOAD, 3); + let facts = reported(&replies[0]); + assert!( + after(&facts, "applied") < after(&facts, "message"), + "{facts:?}" + ); + assert!( + after(&facts, "message") <= after(&facts, "ready"), + "{facts:?}" + ); + assert_eq!(replies[1..], [Message::Pong, Message::Stopped]); + let (root, worked) = joined(working); assert_eq!(worked.unwrap(), Worked::Stopped); + assert_eq!(root.calls, [applied(&record), Call::Teardown]); } #[test] fn a_clone_s_application_ends_by_the_activation_s_deadline() { // The worker hands the application the activation's deadline, until which it waits for - // the devices at the latest: an Activate whose application would outlast the deadline, - // such as one whose device never appears, ends the clone with the deadline's status once - // the deadline passed, rather than once the application's waits would have ended, with - // no Ready. + // the devices at the latest: a record whose application would outlast the deadline, such + // as one whose device never appears, ends the clone with the deadline's status once the + // deadline passed, rather than once the application's waits would have ended, with no + // Ready. let timeout = Duration::from_secs(2); let slow = Duration::from_secs(60); let root = FakeRoot::booting(&layout()).assembling_for(slow); let started = Instant::now(); - let (mut broker, working) = start_clone(root, timeout); - broker.reset(RESTORED); let record = record(1, Some(WORKLOAD)); - broker.send_data(&activate(&record).encode().unwrap(), OUTER_MAX_PAYLOAD); + let (mut broker, working) = start_clone(root, &record, timeout); + broker.reset(RESTORED); + broker.send_data(&Message::Await.encode().unwrap(), OUTER_MAX_PAYLOAD); let root = unactivated(&mut broker, working, status::ACTIVATION_DEADLINE); let ended = Instant::now(); assert!(ended - started < slow, "{:?}", ended - started); @@ -4714,39 +4723,26 @@ mod clones { #[test] fn a_clone_takes_what_came_during_its_application_before_it_is_ready() { - // The application runs until the test opens the gate, by when the host sent more after - // its Activate, before Ready, which the worker takes before it completes the activation: - // another reset, which the worker acknowledges, or a request fails the activation, with - // the status of the host's session, after one application, and with no Ready. + // The application runs until the test opens the gate, by when the host sent its `Await`, + // and a request, before Ready, which the worker takes before it completes the activation: + // the request fails the activation, with the status of the host's session, after one + // application, and with no Ready. let record = record(1, Some(WORKLOAD)); - let bytes = activate(&record).encode().unwrap(); - for reset in [true, false] { - let (root, gate) = FakeRoot::booting(&layout()).gated(); - let (mut broker, working) = start_clone(root, ACTIVATION_TIMEOUT); - broker.reset(RESTORED); - broker.send_data(&bytes, OUTER_MAX_PAYLOAD); - if reset { - broker.reset_ahead(Identity { - epoch: 2, - ..RESTORED - }); - } else { - broker.send_data(&Message::Ping.encode().unwrap(), OUTER_MAX_PAYLOAD); - } - gate.send(()).unwrap(); - if reset { - broker.acknowledged(); - } - let root = unactivated(&mut broker, working, status::ACTIVATION_PROTOCOL); - assert_eq!(root.calls, [applied(&record)], "reset {reset}"); - } - - // A retry of the Activate, byte for byte, joins the application: the worker answers both - // with Ready, after one application, then serves the host's order. let (root, gate) = FakeRoot::booting(&layout()).gated(); - let (mut broker, working) = start_clone(root, ACTIVATION_TIMEOUT); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); + let bytes = encode(&[Message::Await, Message::Ping]); broker.send_data(&bytes, OUTER_MAX_PAYLOAD); + gate.send(()).unwrap(); + let root = unactivated(&mut broker, working, status::ACTIVATION_PROTOCOL); + assert_eq!(root.calls, [applied(&record)]); + + // Each `Await` that came meanwhile joins the application: the worker answers each with + // Ready, after one application, then serves the host's order. + let (root, gate) = FakeRoot::booting(&layout()).gated(); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); + broker.reset(RESTORED); + let bytes = encode(&[Message::Await, Message::Await]); broker.send_data(&bytes, OUTER_MAX_PAYLOAD); gate.send(()).unwrap(); let mut replies = Vec::new(); @@ -4764,45 +4760,49 @@ mod clones { #[test] fn a_clone_whose_stream_takes_nothing_still_ends_once_its_activation_failed() { // The guest's direction of the stream is full, as once the broker failed: the worker - // cannot send its acknowledgement of the restore's reset, and still ends once the - // deadline passed, with the deadline's status, once it waited for the stream for - // `UNACTIVATED_DRAIN`. + // cannot send its acknowledgement of the restore's reset, still applies the record of its + // restore packet, for which no host asks, and ends once the deadline passed, with the + // deadline's status, once it waited for the stream for `UNACTIVATED_DRAIN`. + let record = record(1, Some(WORKLOAD)); let (broker, guest) = stalled(); let reset = super::record(RecordType::Reset, RESTORED, 0, &[]); (&broker).write_all(&reset).unwrap(); - let deadline = Duration::from_millis(200); + let deadline = Duration::from_secs(1); let started = Instant::now(); - let working = run_clone(guest, FakeRoot::booting(&layout()), deadline); + let root = FakeRoot::booting(&layout()); + let working = run_clone(guest, root, Some(record.encode()), deadline); let (root, worked) = joined(working); let worked = worked.unwrap(); assert_eq!(worked.status(), status::ACTIVATION_DEADLINE, "{worked:?}"); assert!(started.elapsed() >= deadline + UNACTIVATED_DRAIN); - assert_eq!(root.calls, []); + assert_eq!(root.calls, [applied(&record)]); // So does a worker whose broker closes its end of the stream before the clone is ready, - // as an abandoned clone, long before the deadline passes. + // as an abandoned clone, long before the deadline passes, once it applied its record. let (broker, guest) = stalled(); (&broker).write_all(&reset).unwrap(); broker.shutdown(Shutdown::Write).unwrap(); let started = Instant::now(); - let working = run_clone(guest, FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + let root = FakeRoot::booting(&layout()); + let working = run_clone(guest, root, Some(record.encode()), ACTIVATION_TIMEOUT); let (root, worked) = joined(working); assert_eq!(worked.unwrap(), Worked::Abandoned); assert!(started.elapsed() >= UNACTIVATED_DRAIN); - assert_eq!(root.calls, []); + assert_eq!(root.calls, [applied(&record)]); } #[test] fn a_clone_whose_broker_closes_the_stream_before_it_is_ready_is_abandoned() { - // Whether the restore's reset came or not, or part of an Activate, a broker that closes - // the stream before the clone is ready ends the clone with the status of the host's - // session, with nothing applied, and its fence closed. + // Whether the restore's reset came or not, or part of an `Await`, a broker that closes + // the stream before the clone is ready, which it is not before the host's `Await`, ends + // the clone with the status of the host's session, with its fence closed, whether its + // record applied or not. let record = record(1, Some(WORKLOAD)); - let bytes = activate(&record).encode().unwrap(); - let cases: [Option<&[u8]>; 3] = [None, Some(&[]), Some(&bytes[..bytes.len() / 2])]; + let bytes = Message::Await.encode().unwrap(); + let cases: [Option<&[u8]>; 3] = [None, Some(&[]), Some(&bytes[..2])]; for sent in cases { - let (mut broker, working) = - start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + let root = FakeRoot::booting(&layout()); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); if let Some(sent) = sent { broker.reset(RESTORED); broker.send_data(sent, OUTER_MAX_PAYLOAD); @@ -4815,7 +4815,12 @@ mod clones { (&Worked::Abandoned, 1), "{sent:?}" ); - assert_eq!(root.calls, [], "{sent:?}"); + let calls = if sent.is_some() { + vec![applied(&record)] + } else { + Vec::new() + }; + assert_eq!(root.calls, calls, "{sent:?}"); while let Some(record) = broker.read() { broker.check(&record); assert_eq!(broker.accept(&record), [], "{sent:?}"); @@ -4824,9 +4829,10 @@ mod clones { // Once the clone is ready, it ends as the worker of a VM that boots cold does, and leaves // the root that it assembled to its operations. - let (mut broker, working) = start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + let root = FakeRoot::booting(&layout()); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); - let replies = broker.exchange(&[activate(&record)], OUTER_MAX_PAYLOAD, 1); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(&record)]); broker.close(); let (root, worked) = joined(working); @@ -4839,8 +4845,10 @@ mod clones { fn a_clone_whose_broker_is_gone_still_ends_as_its_activation_tells() { // The broker sends the restore's reset, and a ping, which fails the activation, or // nothing more, and closes its socket before the worker runs: the worker cannot write - // its acknowledgement, and still ends with the activation's status, or as an abandoned - // clone, rather than with the failure of its stream. + // its acknowledgement, applies the record of its restore packet, and still ends with the + // activation's status, or as an abandoned clone, rather than with the failure of its + // stream. + let record = record(1, Some(WORKLOAD)); let reset = super::record(RecordType::Reset, RESTORED, 0, &[]); let ping = Message::Ping.encode().unwrap(); let ping = super::record(RecordType::Data, RESTORED, 1, &ping); @@ -4855,7 +4863,8 @@ mod clones { let (broker, guest) = UnixStream::pair().unwrap(); (&broker).write_all(&sent).unwrap(); drop(broker); - let working = run_clone(guest, FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); + let root = FakeRoot::booting(&layout()); + let working = run_clone(guest, root, Some(record.encode()), ACTIVATION_TIMEOUT); let (root, worked) = joined(working); let worked = worked.unwrap(); match expected { @@ -4865,34 +4874,7 @@ mod clones { } None => assert_eq!(worked, Worked::Abandoned), } - assert_eq!(root.calls, [], "{worked:?}"); - } - } - - #[test] - fn a_ready_clone_fails_on_an_activate_that_is_not_a_retry() { - // An Activate of another transaction, or of the same transaction with another record, once - // the clone is ready or its sandbox started, fails the activation with the status of the - // host's session. The worker leaves the root that it assembled to its operations. - let accepted = record(1, Some(WORKLOAD)); - let cases = [ - (record(2, Some(WORKLOAD)), false), - (record(1, None), false), - (record(1, None), true), - ]; - for (other, started) in cases { - let (mut broker, working) = - start_clone(FakeRoot::booting(&layout()), ACTIVATION_TIMEOUT); - broker.reset(RESTORED); - let replies = broker.exchange(&[activate(&accepted)], OUTER_MAX_PAYLOAD, 1); - assert_eq!(replies, [activated(&accepted)]); - if started { - let replies = broker.exchange(&[Message::Start], OUTER_MAX_PAYLOAD, 1); - assert_eq!(replies, [Message::Started]); - } - broker.send_data(&activate(&other).encode().unwrap(), OUTER_MAX_PAYLOAD); - let root = unactivated(&mut broker, working, status::ACTIVATION_PROTOCOL); - assert_eq!(root.calls, [applied(&accepted)], "{other:?}"); + assert_eq!(root.calls, [applied(&record)], "{worked:?}"); } } @@ -4998,12 +4980,12 @@ mod clones { let record = published(&record(1, None)); for started in [false, true] { let (root, ending) = FakeRoot::booting(&layout()).ending(); - let (mut broker, working) = start_clone(root, ACTIVATION_TIMEOUT); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); broker.reset(RESTORED); - let replies = broker.exchange(&[activate(&record)], OUTER_MAX_PAYLOAD, 1); + let replies = broker.exchange(&[Message::Await], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [activated(&record)]); // The host waits for Ready before it starts the sandbox: until the activation - // settled, any message but a retry of the Activate fails it. + // settled, any message but an `Await` fails it. if started { let replies = broker.exchange(&[Message::Start], OUTER_MAX_PAYLOAD, 1); assert_eq!(replies, [Message::Started]); @@ -5038,7 +5020,8 @@ mod clones { let mut broker = Broker::new(broker); let session = captured_session(); let acknowledged = Instant::now(); - let activation = repaired(acknowledged, ACTIVATION_TIMEOUT); + let packet = record.encode(); + let activation = repaired(Some(&packet), acknowledged, ACTIVATION_TIMEOUT); let gatewayless = RootTemplate { gateway: None, ..template() @@ -5057,7 +5040,6 @@ mod clones { (root, worked) }); broker.reset(RESTORED); - broker.send_data(&activate(&record).encode().unwrap(), OUTER_MAX_PAYLOAD); let root = unactivated(&mut broker, working, status::ACTIVATION_NETWORK); assert!(root.announcer.is_none()); } @@ -5083,8 +5065,8 @@ mod clones { fn a_clone_reports_when_its_events_happened_after_the_acknowledgement() { // The worker answers the probe of the clone's activation itself, with no root operation, // executions running or not: the restore's reset came after the repair's acknowledgement, - // then the host's first message, the clone's Activate, then its Ready and its start, and - // no execution yet. + // then the application of the record of its restore packet, and the host's first message, + // its `Await`, in either order, then the clone's Ready, its start, and no execution yet. let record = record(1, Some(WORKLOAD)); let mut host = Host::activated(FakeRoot::booting(&layout()), &record); let probe = Message::Probe(ACTIVATION_PROBE.as_bytes().to_vec()); @@ -5095,8 +5077,11 @@ mod clones { // Its template delayed no repair. assert_eq!(facts.get("repair-delay"), Some("none")); assert_eq!(facts.get("execution"), Some("none")); - let events = ["reset", "message", "ready", "started"].map(|name| after(&facts, name)); - assert!(events.is_sorted(), "{facts:?}"); + let [reset, message, applied, ready, started] = + ["reset", "message", "applied", "ready", "started"].map(|name| after(&facts, name)); + assert!(reset <= message.min(applied), "{facts:?}"); + assert!(message.max(applied) <= ready, "{facts:?}"); + assert!(ready <= started, "{facts:?}"); // The start of the first execution came once the sandbox started. let workload = exec::Identity::new(WORKLOAD.0, WORKLOAD.1); @@ -5107,7 +5092,7 @@ mod clones { host.send(&[probe]); host.run_until(|host| !host.replies.is_empty()); let facts = reported(&host.replies.remove(0)); - assert!(after(&facts, "execution") >= events[3], "{facts:?}"); + assert!(after(&facts, "execution") >= started, "{facts:?}"); host.run_until(|host| host.outcome(1).is_some()); check_lossless(&host, 1, Status::Exited(0)); let (root, worked) = host.stop(); diff --git a/tests/runtime-adapter/tests/host_session.rs b/tests/runtime-adapter/tests/host_session.rs index e52e2a68f..8339c5e36 100644 --- a/tests/runtime-adapter/tests/host_session.rs +++ b/tests/runtime-adapter/tests/host_session.rs @@ -11,10 +11,10 @@ //! teardown failed, and the host's order of a layered root: the guest agent's readiness and build, //! the mapping table in parts, and `Start`, and each failure of the order, after which the host //! sends nothing more, or stops the guest. They also cover a clone's order, the framing of the -//! restore sequence of `nvx::host::template`: the bootstrap of its session, its record as the +//! restore sequence of `nvx::host::template`: the bootstrap of its session, its `Await` as the //! first request, the `Ready` that reports its activation, the sessions that it takes for lost, -//! after a later reset, the end of the stream, or a failure of the transport, whose successor sends -//! the same record, and the failures that are final. The broker's guest also runs the runners of +//! after a later reset, the end of the stream, or a failure of the transport, whose successor gets +//! the same `Ready`, and the failures that are final. The broker's guest also runs the runners of //! NVX's execution engine for the executions that the host program's driver runs, so that the tests //! cover the driver's output, credit, outcomes, and timers, the refusals that end its run, a guest //! that stops reading, and a stop that skips the guest's late messages about the executions that @@ -44,9 +44,7 @@ use nvx::protocol::{ }; use runtime_adapter::GUEST_BUILD; use runtime_adapter::exec::{GuestMessage, HostMessage, MAX_EXECUTIONS, Start}; -use runtime_adapter::frame::{ - Decoder, FrameError, HEADER_LEN, MAX_ACTIVATE_BODY, MAX_BODY, MAX_TEXT_BODY, Message, -}; +use runtime_adapter::frame::{Decoder, FrameError, HEADER_LEN, MAX_BODY, MAX_TEXT_BODY, Message}; use runtime_adapter::host::{ self, Client, ClientError, CloneOrder, Executions, ExecutionsError, MAX_OUTPUT_FRAMES, MapReply, OrderError, @@ -1153,7 +1151,8 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { let too_long_text = u32::try_from(MAX_TEXT_BODY + 1).unwrap().to_le_bytes(); let cases = [ (header([0; 4], 0), FrameError::Kind(0)), - (header([0; 4], 28), FrameError::Kind(28)), + (header([0; 4], 27), FrameError::Kind(27)), + (header([0; 4], 29), FrameError::Kind(29)), ( header(too_long, 3), FrameError::Length { @@ -1224,7 +1223,7 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { HostMessage::Input(key, b"input").encode(), HostMessage::InputEnd(key).encode(), HostMessage::Cancel(key, Cause::Cancelled).encode(), - Message::Activate(activation_record()), + Message::Await, ]; for message in host_messages { let (mut broker, mut client) = attached(FIRST, usize::MAX); @@ -1280,7 +1279,7 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { let stop: Request = |client| client.stop(deadline()).map(drop); let map: Request = |client| client.map(&maps_part(), deadline()).map(drop); let start: Request = |client| client.start(deadline()); - let activate: Request = |client| client.activate(&activation_record(), deadline()).map(drop); + let await_ready: Request = |client| client.await_ready(deadline()).map(drop); let teardown = RootFailure::new(RootStage::Teardown, "busy").encode(); // Each request, and answers that don't answer it. let cases: [(&str, Request, Vec, Kinds); 18] = [ @@ -1384,18 +1383,18 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { Some((25, 24)), ), ( - "an Activate answered with Started", - activate, + "an Await answered with Started", + await_ready, vec![Message::Started], - Some((27, 26)), + Some((28, 26)), ), ( - "an Activate answered with a root's failure", - activate, + "an Await answered with a root's failure", + await_ready, vec![Message::RootFailed( RootFailure::new(RootStage::Layout, "late").encode(), )], - Some((27, 8)), + Some((28, 8)), ), ]; for (name, request, replies, kinds) in cases { @@ -1522,7 +1521,7 @@ fn answers_whose_bodies_do_not_decode_end_the_client() { }; let stop: Request = |client| client.stop(deadline()).map(drop); let map: Request = |client| client.map(&maps_part(), deadline()).map(drop); - let activate: Request = |client| client.activate(&activation_record(), deadline()).map(drop); + let await_ready: Request = |client| client.await_ready(deadline()).map(drop); let mut partial_activation = ready().encode(); partial_activation.extend([0; 4]); let cases: [(&str, Request, Message, &str); 12] = [ @@ -1594,7 +1593,7 @@ fn answers_whose_bodies_do_not_decode_end_the_client() { ), ( "a clone's readiness with bytes after its build that are not an activation", - activate, + await_ready, Message::Ready(partial_activation), "has 4 bytes after its build, which are not an activation", ), @@ -1855,12 +1854,6 @@ fn a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest() ); } -/// Returns the bytes that stand for a clone's activation record in its canonical encoding, which -/// the host's client carries in an `Activate` without decoding them. -fn activation_record() -> Vec { - b"\x01the clone's activation record".to_vec() -} - /// Returns the activation that the guest of a clone reports in its `Ready`. fn clone_activation() -> ReadyActivation { ReadyActivation { @@ -1879,17 +1872,14 @@ fn clone_order() -> CloneOrder<'static> { CloneOrder { build: GUEST_BUILD } } -/// Runs the first step of a clone's order with `client` in a thread: sends [`activation_record`] in an -/// `Activate`. The thread returns the client with the activation that `Ready` reports, or how the -/// step failed. -fn activating(client: Client) -> JoinHandle<(Client, Result, OrderError>)> { - spawn(client, |client| { - clone_order().activate(client, &activation_record(), deadline()) - }) +/// Runs the first step of a clone's order with `client` in a thread: sends `Await`. The thread +/// returns the client with the activation that `Ready` reports, or how the step failed. +fn awaiting(client: Client) -> JoinHandle<(Client, Result, OrderError>)> { + spawn(client, |client| clone_order().ready(client, deadline())) } #[test] -fn a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot() { +fn a_clone_s_order_awaits_its_ready_first_then_starts_its_sandbox_as_after_a_cold_boot() { // The broker waits for the guest's acknowledgement of the restore's reset, the session's // bootstrap, before it attaches the host, which waits for it. let (mut broker, transport) = connect(FIRST, usize::MAX); @@ -1904,10 +1894,11 @@ fn a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_col let client = joined(attaching).unwrap(); assert_eq!(client.identity(), FIRST); - // The record is the session's first request, after which the host waits for `Ready`, which - // arrives in records of 7 bytes. - let activation = activating(client); - assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + // `Await`, which carries nothing, since the clone took its record from its restore packet, is + // the session's first request, after which the host waits for `Ready`, which arrives in + // records of 7 bytes. + let activation = awaiting(client); + assert_eq!(broker.messages(1), [Message::Await]); broker.assert_quiet(); broker.send_data(&clone_ready().encode().unwrap(), 7); let (client, activated) = joined(activation); @@ -1934,16 +1925,16 @@ fn a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_col assert!(broker.read().is_none(), "the host sent more"); // A `Ready` that reports no activation, as a cold boot's does, is returned as none, for the - // restore sequence to refuse; `host::activate` returns the readiness with its build. + // restore sequence to refuse; `host::await_ready` returns the readiness with its build. let (mut broker, client) = attached(FIRST, usize::MAX); - let activation = activating(client); - broker.expect(Message::Activate(activation_record()), &ready_message()); + let activation = awaiting(client); + broker.expect(Message::Await, &ready_message()); let (client, activated) = joined(activation); assert_eq!(activated.unwrap(), None); let activation = spawn(client, |client| { - host::activate(client, &activation_record(), GUEST_BUILD, deadline()) + host::await_ready(client, GUEST_BUILD, deadline()) }); - broker.expect(Message::Activate(activation_record()), &clone_ready()); + broker.expect(Message::Await, &clone_ready()); let readiness = joined(activation).1.unwrap(); assert_eq!(readiness.build(), GUEST_BUILD); assert_eq!(readiness.activation(), Some(clone_activation())); @@ -1958,9 +1949,9 @@ fn a_clone_s_order_ends_its_table_at_a_mapping_that_the_guest_fails_and_stops_th let parts = encode_maps(&table).unwrap(); assert!(parts.len() > 2); let (mut broker, client) = attached(FIRST, usize::MAX); - // The table follows the clone's `Ready`, which answers its record. - let activation = activating(client); - broker.expect(Message::Activate(activation_record()), &clone_ready()); + // The table follows the clone's `Ready`, which answers its `Await`. + let activation = awaiting(client); + broker.expect(Message::Await, &clone_ready()); let (client, activated) = joined(activation); assert_eq!(activated.unwrap(), Some(clone_activation())); let mapping = spawn(client, move |client| { @@ -2021,9 +2012,9 @@ fn a_clone_s_order_maps_its_export_s_children_by_their_indices() { ] ); let (mut broker, client) = attached(FIRST, usize::MAX); - // The table follows the clone's `Ready`, which answers its record. - let activation = activating(client); - broker.expect(Message::Activate(activation_record()), &clone_ready()); + // The table follows the clone's `Ready`, which answers its `Await`. + let activation = awaiting(client); + broker.expect(Message::Await, &clone_ready()); let (client, activated) = joined(activation); assert_eq!(activated.unwrap(), Some(clone_activation())); let mapping = spawn(client, move |client| { @@ -2037,25 +2028,25 @@ fn a_clone_s_order_maps_its_export_s_children_by_their_indices() { } #[test] -fn a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record() { +fn a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_gets_the_same_ready() { let next = Identity { epoch: 2, ..FIRST }; // The broker resets the session, which a broker announces, or ends its stream, as OpenVMM's // broker ends an attachment, while the host awaits `Ready`; or the transport fails as the - // host sends the record. + // host sends its `Await`. for ending in ["reset", "end of stream", "transport"] { let (mut broker, client) = attached(FIRST, usize::MAX); if ending == "transport" { broker.from_host.close(); } - let activation = activating(client); + let activation = awaiting(client); match ending { "reset" => { - assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + assert_eq!(broker.messages(1), [Message::Await]); broker.identity = next; broker.send(RecordType::Reset, &[]); } "end of stream" => { - assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + assert_eq!(broker.messages(1), [Message::Await]); broker.close(); } _ => {} @@ -2065,11 +2056,11 @@ fn a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_th assert!(clone_order().lost(&error), "{ending}: {error:?}"); assert_ended(&mut client); - // The order's next session sends the same record, byte for byte, as its first request, - // and the clone, which was ready, answers it with the same `Ready`. + // The order's next session sends its `Await` as its first request, and the clone, which + // was ready, answers it with the same `Ready`. let (mut broker, client) = attached(next, usize::MAX); - let activation = activating(client); - broker.expect(Message::Activate(activation_record()), &clone_ready()); + let activation = awaiting(client); + broker.expect(Message::Await, &clone_ready()); let (_, activated) = joined(activation); assert_eq!(activated.unwrap(), Some(clone_activation()), "{ending}"); } @@ -2111,14 +2102,11 @@ fn only_a_session_that_ended_is_lost_to_a_clone_s_order() { // The guest agent that activated the clone is of another build: the host sends it nothing // more. let (mut broker, client) = attached(FIRST, usize::MAX); - let activation = activating(client); + let activation = awaiting(client); let another = Ready::new("another-build") .unwrap() .with_activation(clone_activation()); - broker.expect( - Message::Activate(activation_record()), - &Message::Ready(another.encode()), - ); + broker.expect(Message::Await, &Message::Ready(another.encode())); let (client, activated) = joined(activation); drop(client); assert!(broker.read().is_none(), "the host sent more"); @@ -2133,9 +2121,9 @@ fn only_a_session_that_ended_is_lost_to_a_clone_s_order() { // `Ready` does not arrive in time. let (mut broker, client) = attached(FIRST, usize::MAX); let activation = spawn(client, |client| { - clone_order().activate(client, &activation_record(), Instant::now() + SHORT) + clone_order().ready(client, Instant::now() + SHORT) }); - assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + assert_eq!(broker.messages(1), [Message::Await]); let (mut client, activated) = joined(activation); let error = activated.unwrap_err(); let late = matches!( @@ -2145,45 +2133,26 @@ fn only_a_session_that_ended_is_lost_to_a_clone_s_order() { assert!(late && not_lost(&error), "{error:?}"); assert_ended(&mut client); - // The guest answers the record with another kind, or with a `Ready` whose activation does not - // decode. + // The guest answers the `Await` with another kind, or with a `Ready` whose activation does + // not decode. let mut partial = ready().encode(); partial.push(1); for reply in [Message::Pong, Message::Ready(partial)] { let (mut broker, client) = attached(FIRST, usize::MAX); - let activation = activating(client); - broker.expect(Message::Activate(activation_record()), &reply); + let activation = awaiting(client); + broker.expect(Message::Await, &reply); let (mut client, activated) = joined(activation); let error = activated.unwrap_err(); let failed = matches!( &error, OrderError::Client(ClientError::Reply { - request: 27, + request: 28, reply: 2 }) | OrderError::Client(ClientError::Body(_)) ); assert!(failed && not_lost(&error), "{reply:?}: {error:?}"); assert_ended(&mut client); } - - // A record that the framing refuses fails before it is sent, and the client stays usable. - let (mut broker, client) = attached(FIRST, usize::MAX); - let activation = spawn(client, |client| { - let too_long = vec![0; MAX_ACTIVATE_BODY + 1]; - clone_order().activate(client, &too_long, deadline()) - }); - let (client, activated) = joined(activation); - let error = activated.unwrap_err(); - let expected = FrameError::Length { - kind: 27, - length: MAX_ACTIVATE_BODY + 1, - }; - let refused = - matches!(&error, OrderError::Client(ClientError::Frame(found)) if *found == expected); - assert!(refused && not_lost(&error), "{error:?}"); - let pinging = spawn(client, |client| client.ping(deadline())); - assert_eq!(broker.answer(OUTER_MAX_PAYLOAD), Message::Ping); - joined(pinging).1.unwrap(); } /// The chunk of the executions of the tests of the host's driver. diff --git a/tests/runtime-adapter/tests/vm_clone.rs b/tests/runtime-adapter/tests/vm_clone.rs index 0e586bbc5..6e46d0563 100644 --- a/tests/runtime-adapter/tests/vm_clone.rs +++ b/tests/runtime-adapter/tests/vm_clone.rs @@ -51,8 +51,8 @@ //! - A clone whose activation fails ends before it is ready, with the status of the failure's //! class, which attributes the failure, and the VMM's exit: a scratch image that holds no ext4 //! file system, a journal that the clone's record forbids, and a host that sends anything but the -//! clone's Activate first, a ping, a start, or an execution. The sandbox records the failure -//! until the release of its lease. +//! clone's Await first, a ping, a start, or an execution, which fails the clone whether its +//! record applied or not. The sandbox records the failure until the release of its lease. //! - A clone refuses a scratch image whose upper is not fresh, as an earlier clone left it, and //! ends before it is ready. //! - Clones at once each keep a root of their own, and every execution of a clone sees its @@ -103,8 +103,8 @@ //! holds, gets its answer within a fraction of the delay, as the positive control, and the guest //! reports that it waited the delay before its repair. //! - Each command after the start reattaches to the clone from a new process: the clone answers -//! the Activate of its record again, in a new session of the same broker, with its Ready, without -//! activating again, and serves the command. +//! the Await of the new session, of the same broker, with its Ready, without activating again, +//! and serves the command. //! - A clone's boot console, which its start asks to listen, delivers to each reader, from a new //! process, what the guest writes to its kernel's log once the reader connected, at the endpoint //! that the clone's lease fixes, and discards what the guest writes while no reader is connected, @@ -2246,17 +2246,22 @@ fn assert_generated(view: &View) { /// Checks the events of a clone's activation that its `activation` probe found: the repair /// acknowledged the restore, and the clone took its reset, the host's first message, its Ready, -/// and its start, in that order, after the acknowledgement. Returns when each came, in +/// and its start, in that order, after the acknowledgement, and the application of the record of +/// its restore packet completed between its reset and its Ready, before the host's first message +/// or after it. Returns when the reset, the first message, Ready, and the start came, in /// microseconds after the acknowledgement. fn assert_timeline(facts: &Printed) -> [u64; 4] { assert_eq!(facts["activation.acknowledged"], "yes", "{facts:?}"); - let events = ["reset", "message", "ready", "started"].map(|event| { - let value = &facts[&format!("activation.{event}")]; - value - .parse::() - .unwrap_or_else(|_| panic!("the clone's {event} is {value}: {facts:?}")) - }); + let [reset, message, applied, ready, started] = + ["reset", "message", "applied", "ready", "started"].map(|event| { + let value = &facts[&format!("activation.{event}")]; + value + .parse::() + .unwrap_or_else(|_| panic!("the clone's {event} is {value}: {facts:?}")) + }); + let events = [reset, message, ready, started]; assert!(events.is_sorted(), "{facts:?}"); + assert!((reset..=ready).contains(&applied), "{facts:?}"); events } @@ -2326,7 +2331,7 @@ fn other_boot(boot: BootId) -> BootId { } /// The adapter's framing of a clone's control session that sends `early`, which the host must not -/// send before the clone is ready, in place of the clone's Activate, and, once the clone ended its +/// send before the clone is ready, in place of the clone's Await, and, once the clone ended its /// session, waits for the clone's VMM to exit, so that the restore sequence, which then ends the /// launch, ends it once the VM ended on its own. struct Early { @@ -2351,10 +2356,9 @@ impl CloneControl for Early { Client::attach(transport, capability, deadline).map_err(io::Error::other) } - fn activate( + fn ready( &mut self, session: &mut Client, - _record: &[u8], deadline: Instant, ) -> io::Result> { session @@ -2387,7 +2391,7 @@ impl CloneControl for Early { } /// Starts a clone of `template` in RAM, in a sandbox of its own, through NVX's public API, whose -/// host sends `early` before the clone's Activate, and checks that the clone ended before it was +/// host sends `early` before the clone's Await, and checks that the clone ended before it was /// ready, with the status of the host's session, which attributes the failure to no one, as the /// sandbox records it, and that the VMM exited, then releases the clone's lease, which removes the /// record. @@ -2650,13 +2654,12 @@ impl CloneControl for Watched { attached } - fn activate( + fn ready( &mut self, session: &mut Client, - record: &[u8], deadline: Instant, ) -> Result, OrderError> { - self.order.activate(session, record, deadline) + self.order.ready(session, deadline) } fn map( @@ -3212,8 +3215,9 @@ fn a_clone_whose_activation_fails_ends_before_it_is_ready() { ended.assert_unactivated(&template, evidence, expected); } - // Until the clone is ready, it takes nothing but its Activate: a ping, a start, and the start - // of an execution, before it, each end the clone with the status of the host's session. + // Until the clone is ready, which it is not before the host's Await, whether its record + // applied or not, it takes nothing but its Await: a ping, a start, and the start of an + // execution, before it, each end the clone with the status of the host's session. let timers = Timers::new(Duration::from_secs(1), Duration::from_secs(5)); let execution = Start::new(Key::new(1, 1), limits(), timers, [b"/bin/true".to_vec()]); for early in [ @@ -3924,8 +3928,8 @@ fn each_command_reattaches_to_the_clone_from_a_new_process() { assert!(records.journal.is_none() && records.failure.is_none()); // Each command, from a new process, reattaches to the clone in a new session of the same - // broker, in which the clone answers the Activate of its record again with its Ready, without - // activating again, so that the events of its activation stay as they were. + // broker, in which the clone answers the host's Await with its Ready, without activating + // again, so that the events of its activation stay as they were. let first = cloned.sandbox.run("reattach", &[], 0); assert_eq!(first["reattached"], "live", "{first:?}"); assert_eq!(first["pid"], cloned.started["pid"]); From 1f2d44bd83b9a5657a6ae25802d41dba636fb3b3 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 13:04:49 -0800 Subject: [PATCH 2/4] doc: say that clones activate from the restore packet's record The guide's restore packet (5.5.5) says that NVX supplies an activation record at every launch of a platform clone, whose guest activates from it as soon as it runs, without waiting for the host, which only asks for the clone's Ready, and that a template key's activation contract refuses a template whose guest takes its record otherwise (microsoft/nvx#544). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/5.5-restore-protocol.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/design/5.5-restore-protocol.md b/doc/design/5.5-restore-protocol.md index b1edb272b..846224795 100644 --- a/doc/design/5.5-restore-protocol.md +++ b/doc/design/5.5-restore-protocol.md @@ -285,7 +285,11 @@ port access, as every byte of the packet does. The flag was a reserved bit, which a guest that predates it rejects, so such a guest fails a restore that hands it a record rather than ignore the record, and restores without a record are unchanged. Such a restore always sets `ACK_REQUIRED`, so the guest has read -the record before host input reaches it. +the record before host input reaches it. NVX supplies a record at every launch +of a platform clone, whose guest activates from it as soon as it runs, without +waiting for the host, which only asks for the clone's Ready on the control +channel; a template key's activation contract refuses a template whose guest +takes its record otherwise. ## 5.5.6 Generations From e7baa636cafa534cdc849f768dbf0fd8fb678350 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 19:06:20 -0800 Subject: [PATCH 3/4] runtime-adapter: name the renamed clone order scenarios in the suite The clone order tests that ask for Ready took new names when Await replaced Activate, but the suite's scenarios and the README's stage scenarios still named the old ones, so test-runtime-adapter refused to run on every backend: host_session had no tests by those names, and two tests that no scenario named. Name the new tests, in the order of their names, and say that the order sends Await first and that the next session's Await gets the same Ready. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/runtime-adapter/README.md | 4 ++-- tests/runtime-adapter/scenarios.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 6da1f6376..4cd71bcf9 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -3176,10 +3176,10 @@ layered roots, captures of templates, and the starts and runs of their clones. | `clone_start` | `the_host_program_refuses_options_that_do_not_make_a_boot_console_or_its_read` | The values of `--boot-console` that a start or a run refuses, the commands that do not take it, and the command lines of `console` that the host program refuses, each with status 2; a start that asks for a boot console that listens, or for none, whose selection the template store refuses without a lease | | `clone_start` | `the_host_program_refuses_options_that_do_not_make_a_run` | The command lines of `run` that the host program refuses, its execution's included, each with status 2, and a clock fixture that is not a host's time | | `clone_start` | `the_host_program_refuses_options_that_do_not_make_a_start` | The command lines of `start` that the host program refuses, each with status 2, and a clock fixture that is not a host's time | +| `host_session` | `a_clone_s_order_awaits_its_ready_first_then_starts_its_sandbox_as_after_a_cold_boot` | A clone's order sends `Await` first, since the clone took its record from its restore packet, returns the activation that `Ready` reports, then sends the table in parts and `Start`, and fails on a refusal, a deadline, another build, or an answer that does not decode | | `host_session` | `a_clone_s_order_ends_its_table_at_a_mapping_that_the_guest_fails_and_stops_the_guest` | A clone's order sends no more of its table, nor `Start`, once the guest fails a mapping, and asks the guest to stop | | `host_session` | `a_clone_s_order_maps_its_export_s_children_by_their_indices` | A clone's order sends its table as a cold launch's, whose sources name each path by the index of the export's child that holds it | -| `host_session` | `a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot` | A clone's order sends its activation record first, returns the activation that `Ready` reports, then sends the table in parts and `Start`, and fails on a refusal, a deadline, another build, or an answer that does not decode | -| `host_session` | `a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record` | A clone's session that a reset, the end of its stream, or its transport ended is lost, and the next session sends the same record first | +| `host_session` | `a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_gets_the_same_ready` | A clone's session that a reset, the end of its stream, or its transport ended is lost, and the next session's `Await` gets the same `Ready` | | `host_session` | `a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest` | A mapping that the guest fails, at the first, a middle, or the last part, ends the table without `Start`, and the host stops the guest and learns how the stop ended | | `host_session` | `a_receive_that_times_out_keeps_the_partial_message` | A receive that times out inside a message keeps what arrived, and a later receive returns the whole message | | `host_session` | `a_reset_or_a_disconnect_in_the_middle_of_a_message_drops_it` | A reset or a disconnect in the middle of a message drops the message, in the next epoch or with a new instance | diff --git a/tests/runtime-adapter/scenarios.txt b/tests/runtime-adapter/scenarios.txt index 4b0743e55..37ee8313c 100644 --- a/tests/runtime-adapter/scenarios.txt +++ b/tests/runtime-adapter/scenarios.txt @@ -29,10 +29,10 @@ clone_start the_host_program_refuses_options_that_do_not_make_a_boot_console_or_ clone_start the_host_program_refuses_options_that_do_not_make_a_run clone_start the_host_program_refuses_options_that_do_not_make_a_start +host_session a_clone_s_order_awaits_its_ready_first_then_starts_its_sandbox_as_after_a_cold_boot host_session a_clone_s_order_ends_its_table_at_a_mapping_that_the_guest_fails_and_stops_the_guest host_session a_clone_s_order_maps_its_export_s_children_by_their_indices -host_session a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot -host_session a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record +host_session a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_gets_the_same_ready host_session a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest host_session a_receive_that_times_out_keeps_the_partial_message host_session a_reset_or_a_disconnect_in_the_middle_of_a_message_drops_it From a4e7f20d1f24c398f9d1a95db4ca837223340cbf Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Mon, 9 Nov 2026 20:01:04 -0800 Subject: [PATCH 4/4] runtime-adapter: report a clone's application when it completes The activation probe's `applied` took the instant at which the worker completed the activation, once it took what the session received while the record applied, not when the application completed. An `Await` that came during the application therefore came before `applied` in the probe's report, and the application's time held the worker's input. Take the instant when the application returns, before the worker takes more input, and keep `ready` at the activation's completion. A new test holds the application until an `Await` came, and checks that the clone reports its reset, its application, the host's first message, and its Ready in that order; it fails without the change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../runtime-adapter/src/worker/activation.rs | 7 +++-- tests/runtime-adapter/tests/guest_session.rs | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/runtime-adapter/src/worker/activation.rs b/tests/runtime-adapter/src/worker/activation.rs index 55f8b44cb..a8d2e7988 100644 --- a/tests/runtime-adapter/src/worker/activation.rs +++ b/tests/runtime-adapter/src/worker/activation.rs @@ -304,6 +304,9 @@ impl CloneState { // the completion fails the activation, whatever the application's result. let deadline = self.activation.deadline(); let result = operations.apply(record, &self.template, deadline); + // The application completed: before the worker takes what the session received while it + // ran, such as the host's `Await`, which therefore comes after it. + self.timeline.applied = Some(Instant::now()); let configuration = record.configuration(); // A record announces at most `nvx::activation::MAX_MAPPINGS` mappings, which fit a `u32`. let mappings = configuration.mappings as u32; @@ -342,10 +345,8 @@ impl CloneState { let result = applied.result.map_err(Into::into); Some(match self.activation.complete(Instant::now(), result) { Ok(activation) => { - let now = Instant::now(); - self.timeline.applied = Some(now); if self.activation.established().is_some() { - self.timeline.ready = Some(now); + self.timeline.ready = Some(Instant::now()); } // A clone has no workload identity of the whole sandbox: each execution names // its own. diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index f97c84260..fbf3b0b3b 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -5104,6 +5104,34 @@ mod clones { ); } + #[test] + fn a_clone_reports_its_application_before_an_await_that_came_during_it() { + // The host's `Await` came while the record applied, which runs until the test opens the + // gate: the application completed before the worker took the `Await`, which made the + // clone ready once the worker took what came meanwhile, so the clone reports its reset, + // its application, the host's first message, and its Ready in that order. + let record = record(1, Some(WORKLOAD)); + let (root, gate) = FakeRoot::booting(&layout()).gated(); + let (mut broker, working) = start_clone(root, &record, ACTIVATION_TIMEOUT); + broker.reset(RESTORED); + broker.send_data(&encode(&[Message::Await]), OUTER_MAX_PAYLOAD); + gate.send(()).unwrap(); + let mut replies = Vec::new(); + while replies.is_empty() { + replies.extend(broker.take()); + } + assert_eq!(replies, [activated(&record)]); + let probe = Message::Probe(ACTIVATION_PROBE.as_bytes().to_vec()); + let replies = broker.exchange(&[probe, Message::Stop], OUTER_MAX_PAYLOAD, 2); + let facts = reported(&replies[0]); + let events = ["reset", "applied", "message", "ready"].map(|name| after(&facts, name)); + assert!(events.is_sorted(), "{facts:?}"); + assert_eq!(replies[1], Message::Stopped); + let (root, worked) = joined(working); + assert_eq!(worked.unwrap(), Worked::Stopped); + assert_eq!(root.calls, [applied(&record), Call::Teardown]); + } + #[test] fn a_clone_maps_its_record_s_table_from_its_export_and_leaves_the_proxy_to_the_host() { // The application mounted the record's export, and wrote the resolver file of its