nvme_driver: don't flr nvme devices (#1714)

The default `vfio` device behavior is to issue a function level reset
when attaching or detaching devices. It does so because the device is in
an unknown or untrusted state. However, within the context of a trusted
virtualization stack, OpenHCL can reasonably trust the state and
behavior of the device. So, optimize performance by removing these
function level resets for nvme devices. This follows the same model as
already exists for MANA devices.

The `nvme_driver` already shuts down the device (see
`NvmeDriver::reset()`) and waits for the device to become disabled. A
well behaved nvme device will not issue DMA after this point. That same
device should tolerate a graceful start without an FLR.

Pending work before this PR is ready to commit:

- [x] Initial poc
- [x] Parameterize via command line (provide a way to disable, and also
easily run A/B tests)
- [x] Fix
#1714 (comment)
- [x] Check no regressions on CI
- [x] Test servicing locally with real NVMe devices

Author: mattkur

openhcl/underhill_core/src/lib.rs

@@ -323,6 +323,7 @@ async fn launch_workers(
         gdbstub: opt.gdbstub,
         hide_isolation: opt.hide_isolation,
         nvme_keep_alive: opt.nvme_keep_alive,
+        nvme_always_flr: opt.nvme_always_flr,
         test_configuration: opt.test_configuration,
         disable_uefi_frontpage: opt.disable_uefi_frontpage,
     };

openhcl/underhill_core/src/nvme_manager.rs

@@ -26,9 +26,12 @@ use pal_async::task::Spawn;
 use pal_async::task::Task;
 use std::collections::HashMap;
 use std::collections::hash_map;
+use std::sync::Arc;
 use thiserror::Error;
 use tracing::Instrument;
+use user_driver::vfio::PciDeviceResetMethod;
 use user_driver::vfio::VfioDevice;
+use user_driver::vfio::vfio_set_device_reset_method;
 use vm_resource::AsyncResolveResource;
 use vm_resource::ResourceId;
 use vm_resource::ResourceResolver;
@@ -92,6 +95,7 @@ impl NvmeManager {
         driver_source: &VmTaskDriverSource,
         vp_count: u32,
         save_restore_supported: bool,
+        nvme_always_flr: bool,
         is_isolated: bool,
         saved_state: Option<NvmeSavedState>,
         dma_client_spawner: DmaClientSpawner,
@@ -103,6 +107,7 @@ impl NvmeManager {
             devices: HashMap::new(),
             vp_count,
             save_restore_supported,
+            nvme_always_flr,
             is_isolated,
             dma_client_spawner,
         };
@@ -220,12 +225,99 @@ struct NvmeManagerWorker {
     vp_count: u32,
     /// Running environment (memory layout) allows save/restore.
     save_restore_supported: bool,
+    nvme_always_flr: bool,
     /// If this VM is isolated or not. This influences DMA client allocations.
     is_isolated: bool,
     #[inspect(skip)]
     dma_client_spawner: DmaClientSpawner,
 }
 
+async fn create_nvme_device(
+    driver_source: &VmTaskDriverSource,
+    pci_id: &str,
+    vp_count: u32,
+    nvme_always_flr: bool,
+    is_isolated: bool,
+    dma_client: Arc<dyn user_driver::DmaClient>,
+) -> Result<nvme_driver::NvmeDriver<VfioDevice>, InnerError> {
+    // Disable FLR on vfio attach/detach; this allows faster system
+    // startup/shutdown with the caveat that the device needs to be properly
+    // sent through the shutdown path during servicing operations, as that is
+    // the only cleanup performed. If the device fails to initialize, turn FLR
+    // on and try again, so that the reset is invoked on the next attach.
+    let update_reset = |method: PciDeviceResetMethod| {
+        if let Err(err) = vfio_set_device_reset_method(pci_id, method) {
+            tracing::warn!(
+                ?method,
+                err = &err as &dyn std::error::Error,
+                "Failed to update reset_method"
+            );
+        }
+    };
+    let mut last_err = None;
+    let reset_methods = if nvme_always_flr {
+        &[PciDeviceResetMethod::Flr][..]
+    } else {
+        // If this code can't create a device without resetting it, then still try to issue an FLR
+        // in case that unwedges something weird in the device state.
+        // (This is implicit when the code in [`try_create_nvme_device`] opens a handle to the
+        // Vfio device).
+        &[PciDeviceResetMethod::NoReset, PciDeviceResetMethod::Flr][..]
+    };
+    for reset_method in reset_methods {
+        update_reset(*reset_method);
+        match try_create_nvme_device(
+            driver_source,
+            pci_id,
+            vp_count,
+            is_isolated,
+            dma_client.clone(),
+        )
+        .await
+        {
+            Ok(device) => {
+                if !nvme_always_flr && !matches!(reset_method, PciDeviceResetMethod::NoReset) {
+                    update_reset(PciDeviceResetMethod::NoReset);
+                }
+                return Ok(device);
+            }
+            Err(err) => {
+                tracing::error!(
+                    pci_id,
+                    ?reset_method,
+                    err = &err as &dyn std::error::Error,
+                    "failed to create nvme device"
+                );
+                last_err = Some(err);
+            }
+        }
+    }
+    // Return the most reliable error (this code assumes that the reset methods are in increasing order
+    // of reliability).
+    Err(last_err.unwrap())
+}
+
+async fn try_create_nvme_device(
+    driver_source: &VmTaskDriverSource,
+    pci_id: &str,
+    vp_count: u32,
+    is_isolated: bool,
+    dma_client: Arc<dyn user_driver::DmaClient>,
+) -> Result<nvme_driver::NvmeDriver<VfioDevice>, InnerError> {
+    let device = VfioDevice::new(driver_source, pci_id, dma_client)
+        .instrument(tracing::info_span!("vfio_device_open", pci_id))
+        .await
+        .map_err(InnerError::Vfio)?;
+
+    // TODO: For now, any isolation means use bounce buffering. This
+    // needs to change when we have nvme devices that support DMA to
+    // confidential memory.
+    nvme_driver::NvmeDriver::new(driver_source, vp_count, device, is_isolated)
+        .instrument(tracing::info_span!("nvme_driver_init", pci_id))
+        .await
+        .map_err(InnerError::DeviceInitFailed)
+}
+
 impl NvmeManagerWorker {
     async fn run(&mut self, mut recv: mesh::Receiver<Request>) {
         let (join_span, nvme_keepalive) = loop {
@@ -315,26 +407,18 @@ impl NvmeManagerWorker {
                     })
                     .map_err(InnerError::DmaClient)?;
 
-                let device = VfioDevice::new(&self.driver_source, entry.key(), dma_client)
-                    .instrument(tracing::info_span!("vfio_device_open", pci_id))
-                    .await
-                    .map_err(InnerError::Vfio)?;
-
-                // TODO: For now, any isolation means use bounce buffering. This
-                // needs to change when we have nvme devices that support DMA to
-                // confidential memory.
-                let driver = nvme_driver::NvmeDriver::new(
+                let driver = create_nvme_device(
                     &self.driver_source,
+                    &pci_id,
                     self.vp_count,
-                    device,
+                    self.nvme_always_flr,
                     self.is_isolated,
+                    dma_client,
                 )
-                .instrument(tracing::info_span!(
-                    "nvme_driver_init",
-                    pci_id = entry.key()
-                ))
-                .await
-                .map_err(InnerError::DeviceInitFailed)?;
+                .instrument(
+                    tracing::info_span!("create_nvme_device", %pci_id, self.nvme_always_flr),
+                )
+                .await?;
 
                 entry.insert(driver)
             }

openhcl/underhill_core/src/options.rs

@@ -138,6 +138,11 @@ pub struct Options {
     /// (OPENHCL_NVME_KEEP_ALIVE=1) Enable nvme keep alive when servicing.
     pub nvme_keep_alive: bool,
 
+    /// (OPENHCL_NVME_ALWAYS_FLR=1)
+    /// Always use the FLR (Function Level Reset) path for NVMe devices,
+    /// even if we would otherwise attempt to use VFIO's NoReset support.
+    pub nvme_always_flr: bool,
+
     /// (OPENHCL_TEST_CONFIG=\<TestScenarioConfig\>)
     /// Test configurations are designed to replicate specific behaviors and
     /// conditions in order to simulate various test scenarios.
@@ -233,6 +238,7 @@ impl Options {
         let gdbstub = parse_legacy_env_bool("OPENHCL_GDBSTUB");
         let gdbstub_port = parse_legacy_env_number("OPENHCL_GDBSTUB_PORT")?.map(|x| x as u32);
         let nvme_keep_alive = parse_env_bool("OPENHCL_NVME_KEEP_ALIVE");
+        let nvme_always_flr = parse_env_bool("OPENHCL_NVME_ALWAYS_FLR");
         let test_configuration = parse_env_string("OPENHCL_TEST_CONFIG").and_then(|x| {
             x.to_string_lossy()
                 .parse::<TestScenarioConfig>()
@@ -299,6 +305,7 @@ impl Options {
             halt_on_guest_halt,
             no_sidecar_hotplug,
             nvme_keep_alive,
+            nvme_always_flr,
             test_configuration,
             disable_uefi_frontpage,
         })

openhcl/underhill_core/src/worker.rs

@@ -282,6 +282,8 @@ pub struct UnderhillEnvCfg {
     pub hide_isolation: bool,
     /// Enable nvme keep alive.
     pub nvme_keep_alive: bool,
+    /// Don't skip FLR for NVMe devices.
+    pub nvme_always_flr: bool,
     /// test configuration
     pub test_configuration: Option<TestScenarioConfig>,
     /// Disable the UEFI front page.
@@ -1881,6 +1883,7 @@ async fn new_underhill_vm(
             &driver_source,
             processor_topology.vp_count(),
             save_restore_supported,
+            env_cfg.nvme_always_flr,
             isolation.is_isolated(),
             servicing_state.nvme_state.unwrap_or(None),
             dma_manager.client_spawner(),