From 03e5061467acaebb4b314a6e99bd7bc16f48a4aa Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 15 Nov 2025 00:30:51 -0800 Subject: [PATCH 01/18] Expose memory mapping; Add optional memfile dump --- .tool-versions | 1 + src/firecracker/src/api_server/mod.rs | 4 +-- .../src/api_server/request/snapshot.rs | 4 +-- src/firecracker/swagger/firecracker.yaml | 26 +++++++++++++++ src/vmm/src/persist.rs | 7 ++-- src/vmm/src/rpc_interface.rs | 16 ++++++--- src/vmm/src/vmm_config/instance_info.rs | 3 ++ src/vmm/src/vmm_config/snapshot.rs | 4 ++- src/vmm/src/vstate/vm.rs | 33 ++++++++++++++++++- src/vmm/tests/integration_tests.rs | 2 +- 10 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000000..0aed332b367 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +rust 1.85.0 diff --git a/src/firecracker/src/api_server/mod.rs b/src/firecracker/src/api_server/mod.rs index 71d1856b0d5..d7351f1f1bf 100644 --- a/src/firecracker/src/api_server/mod.rs +++ b/src/firecracker/src/api_server/mod.rs @@ -275,7 +275,7 @@ mod tests { Box::new(VmmAction::CreateSnapshot(CreateSnapshotParams { snapshot_type: SnapshotType::Diff, snapshot_path: PathBuf::new(), - mem_file_path: PathBuf::new(), + mem_file_path: Some(PathBuf::new()), })), start_time_us, ); @@ -288,7 +288,7 @@ mod tests { Box::new(VmmAction::CreateSnapshot(CreateSnapshotParams { snapshot_type: SnapshotType::Diff, snapshot_path: PathBuf::new(), - mem_file_path: PathBuf::new(), + mem_file_path: Some(PathBuf::new()), })), start_time_us, ); diff --git a/src/firecracker/src/api_server/request/snapshot.rs b/src/firecracker/src/api_server/request/snapshot.rs index 4a96292d11d..07ddc40c23c 100644 --- a/src/firecracker/src/api_server/request/snapshot.rs +++ b/src/firecracker/src/api_server/request/snapshot.rs @@ -140,7 +140,7 @@ mod tests { let expected_config = CreateSnapshotParams { snapshot_type: SnapshotType::Diff, snapshot_path: PathBuf::from("foo"), - mem_file_path: PathBuf::from("bar"), + mem_file_path: Some(PathBuf::from("bar")), }; assert_eq!( vmm_action_from_request(parse_put_snapshot(&Body::new(body), Some("create")).unwrap()), @@ -154,7 +154,7 @@ mod tests { let expected_config = CreateSnapshotParams { snapshot_type: SnapshotType::Full, snapshot_path: PathBuf::from("foo"), - mem_file_path: PathBuf::from("bar"), + mem_file_path: Some(PathBuf::from("bar")), }; assert_eq!( vmm_action_from_request(parse_put_snapshot(&Body::new(body), Some("create")).unwrap()), diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index 9c335290157..5135da22c6c 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -996,6 +996,32 @@ definitions: vmm_version: description: MicroVM hypervisor build version. type: string + memory_regions: + type: array + description: The regions of the guest memory. + items: + $ref: "#/definitions/GuestMemoryRegion" + + GuestMemoryRegionMapping: + type: object + description: Describes the region of guest memory that can be used for creating the memfile. + required: + - base_host_virt_addr + - size + - offset + - page_size + properties: + base_host_virt_addr: + type: integer + size: + description: The size of the region in bytes. + type: integer + offset: + description: The offset of the region in bytes. + type: integer + page_size: + description: The page size of the region in pages. + type: integer Logger: type: object diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index aeacadeb66e..b84f518919c 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -162,8 +162,11 @@ pub fn create_snapshot( snapshot_state_to_file(µvm_state, ¶ms.snapshot_path)?; - vmm.vm - .snapshot_memory_to_file(¶ms.mem_file_path, params.snapshot_type)?; + // Dump memory to file only if mem_file_path is specified + if let Some(ref mem_file_path) = params.mem_file_path { + vmm.vm + .snapshot_memory_to_file(mem_file_path, params.snapshot_type)?; + } // We need to mark queues as dirty again for all activated devices. The reason we // do it here is because we don't mark pages as dirty during runtime diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index 127b75e594e..38e22a0fd0b 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -647,9 +647,17 @@ impl RuntimeApiController { GetVmMachineConfig => Ok(VmmData::MachineConfiguration( self.vm_resources.machine_config.clone(), )), - GetVmInstanceInfo => Ok(VmmData::InstanceInformation( - self.vmm.lock().expect("Poisoned lock").instance_info(), - )), + GetVmInstanceInfo => { + let locked_vmm = self.vmm.lock().expect("Poisoned lock"); + + let mut instance_info = locked_vmm.instance_info(); + + instance_info.memory_regions = locked_vmm + .vm + .guest_memory_mappings(&VmInfo::from(&self.vm_resources)); + + Ok(VmmData::InstanceInformation(instance_info)) + } GetVmmVersion => Ok(VmmData::VmmVersion( self.vmm.lock().expect("Poisoned lock").version(), )), @@ -1147,7 +1155,7 @@ mod tests { CreateSnapshotParams { snapshot_type: SnapshotType::Full, snapshot_path: PathBuf::new(), - mem_file_path: PathBuf::new(), + mem_file_path: Some(PathBuf::new()), }, ))); #[cfg(target_arch = "x86_64")] diff --git a/src/vmm/src/vmm_config/instance_info.rs b/src/vmm/src/vmm_config/instance_info.rs index cd5b44f30ba..308f8c38040 100644 --- a/src/vmm/src/vmm_config/instance_info.rs +++ b/src/vmm/src/vmm_config/instance_info.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::fmt::{self, Display, Formatter}; +use crate::vstate::vm::GuestMemoryRegionMapping; use serde::{Serialize, ser}; /// Enumerates microVM runtime states. @@ -46,4 +47,6 @@ pub struct InstanceInfo { pub vmm_version: String, /// The name of the application that runs the microVM. pub app_name: String, + /// The regions of the guest memory. + pub memory_regions: Vec, } diff --git a/src/vmm/src/vmm_config/snapshot.rs b/src/vmm/src/vmm_config/snapshot.rs index 27a7841d5a4..657dc653f6f 100644 --- a/src/vmm/src/vmm_config/snapshot.rs +++ b/src/vmm/src/vmm_config/snapshot.rs @@ -44,7 +44,9 @@ pub struct CreateSnapshotParams { /// Path to the file that will contain the microVM state. pub snapshot_path: PathBuf, /// Path to the file that will contain the guest memory. - pub mem_file_path: PathBuf, + /// If not specified, the memory is not dumped to a file. + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_file_path: Option, } /// Allows for changing the mapping between tap devices and host devices diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs index 7a8965a4b9a..954d3efbe32 100644 --- a/src/vmm/src/vstate/vm.rs +++ b/src/vmm/src/vstate/vm.rs @@ -13,11 +13,12 @@ use std::sync::Arc; use kvm_bindings::{KVM_MEM_LOG_DIRTY_PAGES, kvm_userspace_memory_region}; use kvm_ioctls::VmFd; +use serde::{Deserialize, Serialize}; use vmm_sys_util::eventfd::EventFd; pub use crate::arch::{ArchVm as Vm, ArchVmError, VmState}; use crate::logger::info; -use crate::persist::CreateSnapshotError; +use crate::persist::{CreateSnapshotError, VmInfo}; use crate::utils::u64_to_usize; use crate::vmm_config::snapshot::SnapshotType; use crate::vstate::memory::{ @@ -36,6 +37,20 @@ pub struct VmCommon { pub guest_memory: GuestMemoryMmap, } +/// Describes the region of guest memory that can be used for creating the memfile. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct GuestMemoryRegionMapping { + /// Base host virtual address where the guest memory contents for this region + /// should be copied/populated. + pub base_host_virt_addr: u64, + /// Region size. + pub size: usize, + /// Offset in the backend file/buffer where the region contents are. + pub offset: u64, + /// The configured page size for this memory region. + pub page_size: usize, +} + /// Errors associated with the wrappers over KVM ioctls. /// Needs `rustfmt::skip` to make multiline comments work #[rustfmt::skip] @@ -185,6 +200,22 @@ impl Vm { &self.common.guest_memory } + /// Gets the mappings for the guest memory. + pub fn guest_memory_mappings(&self, vm_info: &VmInfo) -> Vec { + let mut offset = 0; + let mut mappings = Vec::new(); + for mem_region in self.guest_memory().iter() { + mappings.push(GuestMemoryRegionMapping { + base_host_virt_addr: mem_region.as_ptr() as u64, + size: mem_region.size(), + offset, + page_size: vm_info.huge_pages.page_size(), + }); + offset += mem_region.size() as u64; + } + mappings + } + /// Resets the KVM dirty bitmap for each of the guest's memory regions. pub fn reset_dirty_bitmap(&self) { self.guest_memory() diff --git a/src/vmm/tests/integration_tests.rs b/src/vmm/tests/integration_tests.rs index 55fb07c1aae..b8656ec967e 100644 --- a/src/vmm/tests/integration_tests.rs +++ b/src/vmm/tests/integration_tests.rs @@ -215,7 +215,7 @@ fn verify_create_snapshot(is_diff: bool) -> (TempFile, TempFile) { let snapshot_params = CreateSnapshotParams { snapshot_type, snapshot_path: snapshot_file.as_path().to_path_buf(), - mem_file_path: memory_file.as_path().to_path_buf(), + mem_file_path: Some(memory_file.as_path().to_path_buf()), }; controller From eff49ed1e76b1af3e3eda0b3b749d400986315ab Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 15 Nov 2025 01:11:58 -0800 Subject: [PATCH 02/18] Fix compile errors --- src/cpu-template-helper/src/utils/mod.rs | 1 + src/firecracker/src/main.rs | 1 + src/vmm/src/rpc_interface.rs | 8 +++++--- src/vmm/src/vmm_config/instance_info.rs | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/cpu-template-helper/src/utils/mod.rs b/src/cpu-template-helper/src/utils/mod.rs index f23871df1a9..41d355594e7 100644 --- a/src/cpu-template-helper/src/utils/mod.rs +++ b/src/cpu-template-helper/src/utils/mod.rs @@ -125,6 +125,7 @@ pub fn build_microvm_from_config( state: VmState::NotStarted, vmm_version: CPU_TEMPLATE_HELPER_VERSION.to_string(), app_name: "cpu-template-helper".to_string(), + memory_regions: None, }; let mut vm_resources = VmResources::from_json(&config, &instance_info, HTTP_MAX_PAYLOAD_SIZE, None) diff --git a/src/firecracker/src/main.rs b/src/firecracker/src/main.rs index 6b01f776729..1f3790f0f94 100644 --- a/src/firecracker/src/main.rs +++ b/src/firecracker/src/main.rs @@ -342,6 +342,7 @@ fn main_exec() -> Result<(), MainError> { state: VmState::NotStarted, vmm_version: FIRECRACKER_VERSION.to_string(), app_name: "Firecracker".to_string(), + memory_regions: None, }; if let Some(metrics_path) = arguments.single_value("metrics-path") { diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index 38e22a0fd0b..ce1e13307d2 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -652,9 +652,11 @@ impl RuntimeApiController { let mut instance_info = locked_vmm.instance_info(); - instance_info.memory_regions = locked_vmm - .vm - .guest_memory_mappings(&VmInfo::from(&self.vm_resources)); + instance_info.memory_regions = Some( + locked_vmm + .vm + .guest_memory_mappings(&VmInfo::from(&self.vm_resources)), + ); Ok(VmmData::InstanceInformation(instance_info)) } diff --git a/src/vmm/src/vmm_config/instance_info.rs b/src/vmm/src/vmm_config/instance_info.rs index 308f8c38040..3ee954b6bc9 100644 --- a/src/vmm/src/vmm_config/instance_info.rs +++ b/src/vmm/src/vmm_config/instance_info.rs @@ -3,7 +3,7 @@ use std::fmt::{self, Display, Formatter}; use crate::vstate::vm::GuestMemoryRegionMapping; -use serde::{Serialize, ser}; +use serde::{ser, Serialize}; /// Enumerates microVM runtime states. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -48,5 +48,5 @@ pub struct InstanceInfo { /// The name of the application that runs the microVM. pub app_name: String, /// The regions of the guest memory. - pub memory_regions: Vec, + pub memory_regions: Option>, } From 5cfd4e39f15c7a45c3ae0f747c4ccffa05d1464f Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 3 Dec 2025 14:26:13 -0800 Subject: [PATCH 03/18] Remove required field from spec --- src/firecracker/swagger/firecracker.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index 5135da22c6c..e8461f6f81b 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -1225,7 +1225,6 @@ definitions: type: object required: - mem_file_path - - snapshot_path properties: mem_file_path: type: string From 523dc12c9b9a43b7f3a66af74f3a6576e2d60721 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Fri, 5 Dec 2025 14:36:38 -0800 Subject: [PATCH 04/18] Fix parameter --- src/firecracker/swagger/firecracker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index e8461f6f81b..596487f68c6 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -1224,7 +1224,7 @@ definitions: SnapshotCreateParams: type: object required: - - mem_file_path + - snapshot_path properties: mem_file_path: type: string From 8769de7cfabc802e3673f527ef6161277ccef3a7 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Fri, 5 Dec 2025 14:40:58 -0800 Subject: [PATCH 05/18] Fix type --- src/firecracker/swagger/firecracker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index 596487f68c6..a715bd742c4 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -1000,7 +1000,7 @@ definitions: type: array description: The regions of the guest memory. items: - $ref: "#/definitions/GuestMemoryRegion" + $ref: "#/definitions/GuestMemoryRegionMapping" GuestMemoryRegionMapping: type: object @@ -1020,7 +1020,7 @@ definitions: description: The offset of the region in bytes. type: integer page_size: - description: The page size of the region in pages. + description: The page size in bytes. type: integer Logger: From 55b6479f2e980daa4662aa0dc88ef3aefba4cd9e Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Fri, 5 Dec 2025 17:00:16 -0800 Subject: [PATCH 06/18] Add upload script --- .gitignore | 1 + .tool-versions | 1 + Makefile | 12 ++++++++++++ scripts/build.sh | 17 +++++++++++++++++ scripts/upload.sh | 13 +++++++++++++ 5 files changed, 44 insertions(+) create mode 100644 Makefile create mode 100755 scripts/build.sh create mode 100755 scripts/upload.sh diff --git a/.gitignore b/.gitignore index 155e4cbd8a8..f56db437d09 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ test_results/* /resources/linux /resources/x86_64 /resources/aarch64 +.env \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index 0aed332b367..3e828d8e972 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ +gcloud 534.0.0 rust 1.85.0 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..1fda2f26881 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +-include .env + +.PHONY: build +build: + ./scripts/build.sh + +.PHONY: upload +upload: + ./scripts/upload.sh $(GCP_PROJECT_ID) + +.PHONY: build-and-upload +make build-and-upload: build upload diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 00000000000..b3b97e31d9d --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +# The format will be: v.._g — e.g. v1.7.2_g8bb88311 +# Extract full version from src/firecracker/swagger/firecracker.yaml +FC_VERSION=$(python3 -c "import yaml; print(yaml.safe_load(open('src/firecracker/swagger/firecracker.yaml'))['info']['version'])") +commit_hash=$(git rev-parse --short HEAD) +version_name="v${FC_VERSION}_g${commit_hash}" +echo "Version name: $version_name" + +echo "Starting to build Firecracker version: $version_name" +tools/devtool -y build --release + +mkdir -p "./build/fc/${version_name}" +cp ./build/cargo_target/x86_64-unknown-linux-musl/release/firecracker "./build/fc/${version_name}/firecracker" +echo "Finished building Firecracker version: $version_name and copied to ./build/fc/${version_name}/firecracker" diff --git a/scripts/upload.sh b/scripts/upload.sh new file mode 100755 index 00000000000..4227c642593 --- /dev/null +++ b/scripts/upload.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail + +GCP_PROJECT_ID=$1 + +gsutil -h "Cache-Control:no-cache, max-age=0" cp -r "build/fc/*" "gs://${GCP_PROJECT_ID}-fc-versions" +if [ "$GCP_PROJECT_ID" == "e2b-prod" ]; then + # Upload kernel to GCP public builds bucket + gsutil -h "Cache-Control:no-cache, max-age=0" cp -r "build/fc/*" "gs://${GCP_PROJECT_ID}-public-builds/firecrackers/" +fi + +rm -rf build/fc/* From 1133bd6cda187bd5cebbe64d164376ae9e603a42 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Mon, 8 Dec 2025 16:54:52 -0800 Subject: [PATCH 07/18] Parse without python --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index b3b97e31d9d..6f459f63e07 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,7 +4,7 @@ set -euo pipefail # The format will be: v.._g — e.g. v1.7.2_g8bb88311 # Extract full version from src/firecracker/swagger/firecracker.yaml -FC_VERSION=$(python3 -c "import yaml; print(yaml.safe_load(open('src/firecracker/swagger/firecracker.yaml'))['info']['version'])") +FC_VERSION=$(awk '/^info:/{flag=1} flag && /^ version:/{print $2; exit}' src/firecracker/swagger/firecracker.yaml) commit_hash=$(git rev-parse --short HEAD) version_name="v${FC_VERSION}_g${commit_hash}" echo "Version name: $version_name" From 0bb99c51d2f37ced18c27cfd7b0e516bf05d0013 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 14:56:32 -0800 Subject: [PATCH 08/18] Refactor --- .../seccomp/aarch64-unknown-linux-musl.json | 4 + .../seccomp/x86_64-unknown-linux-musl.json | 10 +- .../src/api_server/parsed_request.rs | 50 ++++ .../src/api_server/request/memory.rs | 39 +++ src/firecracker/src/api_server/request/mod.rs | 1 + src/firecracker/swagger/firecracker.yaml | 68 ++++- src/vmm/src/rpc_interface.rs | 40 ++- src/vmm/src/vmm_config/instance_info.rs | 17 ++ src/vmm/src/vstate/vm.rs | 110 ++++++++ tests/framework/http_api.py | 2 + .../integration_tests/functional/test_api.py | 237 ++++++++++++++++++ 11 files changed, 559 insertions(+), 19 deletions(-) create mode 100644 src/firecracker/src/api_server/request/memory.rs diff --git a/resources/seccomp/aarch64-unknown-linux-musl.json b/resources/seccomp/aarch64-unknown-linux-musl.json index db3abe1eced..6a95eca603d 100644 --- a/resources/seccomp/aarch64-unknown-linux-musl.json +++ b/resources/seccomp/aarch64-unknown-linux-musl.json @@ -212,6 +212,10 @@ "syscall": "madvise", "comment": "Used by the VirtIO balloon device and by musl for some customer workloads. It is also used by aws-lc during random number generation. They setup a memory page that mark with MADV_WIPEONFORK to be able to detect forks. They also call it with -1 to see if madvise is supported in certain platforms." }, + { + "syscall": "mincore", + "comment": "Used by get_memory_dirty_bitmap to check if memory pages are resident" + }, { "syscall": "mmap", "comment": "Used by the VirtIO balloon device", diff --git a/resources/seccomp/x86_64-unknown-linux-musl.json b/resources/seccomp/x86_64-unknown-linux-musl.json index 95ceca1b7ef..9c22667c20e 100644 --- a/resources/seccomp/x86_64-unknown-linux-musl.json +++ b/resources/seccomp/x86_64-unknown-linux-musl.json @@ -212,6 +212,10 @@ "syscall": "madvise", "comment": "Used by the VirtIO balloon device and by musl for some customer workloads. It is also used by aws-lc during random number generation. They setup a memory page that mark with MADV_WIPEONFORK to be able to detect forks. They also call it with -1 to see if madvise is supported in certain platforms." }, + { + "syscall": "mincore", + "comment": "Used by get_memory_dirty_bitmap to check if memory pages are resident" + }, { "syscall": "mmap", "comment": "Used by the VirtIO balloon device", @@ -524,8 +528,8 @@ "comment": "sigaltstack is used by Rust stdlib to remove alternative signal stack during thread teardown." }, { - "syscall": "getrandom", - "comment": "getrandom is used by `HttpServer` to reinialize `HashMap` after moving to the API thread" + "syscall": "getrandom", + "comment": "getrandom is used by `HttpServer` to reinialize `HashMap` after moving to the API thread" }, { "syscall": "accept4", @@ -1152,4 +1156,4 @@ } ] } -} +} \ No newline at end of file diff --git a/src/firecracker/src/api_server/parsed_request.rs b/src/firecracker/src/api_server/parsed_request.rs index 10d5c3d97ea..ccfbfc34695 100644 --- a/src/firecracker/src/api_server/parsed_request.rs +++ b/src/firecracker/src/api_server/parsed_request.rs @@ -21,6 +21,7 @@ use super::request::logger::parse_put_logger; use super::request::machine_configuration::{ parse_get_machine_config, parse_patch_machine_config, parse_put_machine_config, }; +use super::request::memory::{parse_get_memory, parse_get_memory_mappings}; use super::request::metrics::parse_put_metrics; use super::request::mmds::{parse_get_mmds, parse_patch_mmds, parse_put_mmds}; use super::request::net::{parse_patch_net, parse_put_net}; @@ -82,6 +83,14 @@ impl TryFrom<&Request> for ParsedRequest { Ok(ParsedRequest::new_sync(VmmAction::GetFullVmConfig)) } (Method::Get, "machine-config", None) => parse_get_machine_config(), + (Method::Get, "memory", None) => match path_tokens.next() { + Some("mappings") => parse_get_memory_mappings(), + None => parse_get_memory(), + _ => Err(RequestError::InvalidPathMethod( + request_uri.to_string(), + Method::Get, + )), + }, (Method::Get, "mmds", None) => parse_get_mmds(), (Method::Get, _, Some(_)) => method_to_error(Method::Get), (Method::Put, "actions", Some(body)) => parse_put_actions(body), @@ -172,6 +181,8 @@ impl ParsedRequest { } VmmData::BalloonStats(stats) => Self::success_response_with_data(stats), VmmData::InstanceInformation(info) => Self::success_response_with_data(info), + VmmData::MemoryMappings(mappings) => Self::success_response_with_data(mappings), + VmmData::Memory(memory) => Self::success_response_with_data(memory), VmmData::VmmVersion(version) => Self::success_response_with_data( &serde_json::json!({ "firecracker_version": version.as_str() }), ), @@ -568,6 +579,12 @@ pub mod tests { VmmData::InstanceInformation(info) => { http_response(&serde_json::to_string(info).unwrap(), 200) } + VmmData::MemoryMappings(mappings) => { + http_response(&serde_json::to_string(mappings).unwrap(), 200) + } + VmmData::Memory(memory) => { + http_response(&serde_json::to_string(memory).unwrap(), 200) + } VmmData::VmmVersion(version) => http_response( &serde_json::json!({ "firecracker_version": version.as_str() }).to_string(), 200, @@ -589,6 +606,15 @@ pub mod tests { verify_ok_response_with(VmmData::MachineConfiguration(MachineConfig::default())); verify_ok_response_with(VmmData::MmdsValue(serde_json::from_str("{}").unwrap())); verify_ok_response_with(VmmData::InstanceInformation(InstanceInfo::default())); + verify_ok_response_with(VmmData::MemoryMappings( + vmm::vmm_config::instance_info::MemoryMappingsResponse { mappings: vec![] }, + )); + verify_ok_response_with(VmmData::Memory( + vmm::vmm_config::instance_info::MemoryResponse { + resident: vec![], + empty: vec![], + }, + )); verify_ok_response_with(VmmData::VmmVersion(String::default())); // Error. @@ -662,6 +688,30 @@ pub mod tests { ParsedRequest::try_from(&req).unwrap(); } + #[test] + fn test_try_from_get_memory_mappings() { + let (mut sender, receiver) = UnixStream::pair().unwrap(); + let mut connection = HttpConnection::new(receiver); + sender + .write_all(http_request("GET", "/memory/mappings", None).as_bytes()) + .unwrap(); + connection.try_read().unwrap(); + let req = connection.pop_parsed_request().unwrap(); + ParsedRequest::try_from(&req).unwrap(); + } + + #[test] + fn test_try_from_get_memory() { + let (mut sender, receiver) = UnixStream::pair().unwrap(); + let mut connection = HttpConnection::new(receiver); + sender + .write_all(http_request("GET", "/memory", None).as_bytes()) + .unwrap(); + connection.try_read().unwrap(); + let req = connection.pop_parsed_request().unwrap(); + ParsedRequest::try_from(&req).unwrap(); + } + #[test] fn test_try_from_get_version() { let (mut sender, receiver) = UnixStream::pair().unwrap(); diff --git a/src/firecracker/src/api_server/request/memory.rs b/src/firecracker/src/api_server/request/memory.rs new file mode 100644 index 00000000000..e879d6b3b02 --- /dev/null +++ b/src/firecracker/src/api_server/request/memory.rs @@ -0,0 +1,39 @@ +// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use vmm::logger::{IncMetric, METRICS}; +use vmm::rpc_interface::VmmAction; + +use super::super::parsed_request::{ParsedRequest, RequestError}; + +pub(crate) fn parse_get_memory_mappings() -> Result { + METRICS.get_api_requests.instance_info_count.inc(); + Ok(ParsedRequest::new_sync(VmmAction::GetMemoryMappings)) +} + +pub(crate) fn parse_get_memory() -> Result { + METRICS.get_api_requests.instance_info_count.inc(); + Ok(ParsedRequest::new_sync(VmmAction::GetMemory)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api_server::parsed_request::RequestAction; + + #[test] + fn test_parse_get_memory_mappings_request() { + match parse_get_memory_mappings().unwrap().into_parts() { + (RequestAction::Sync(action), _) if *action == VmmAction::GetMemoryMappings => {} + _ => panic!("Test failed."), + } + } + + #[test] + fn test_parse_get_memory_request() { + match parse_get_memory().unwrap().into_parts() { + (RequestAction::Sync(action), _) if *action == VmmAction::GetMemory => {} + _ => panic!("Test failed."), + } + } +} diff --git a/src/firecracker/src/api_server/request/mod.rs b/src/firecracker/src/api_server/request/mod.rs index 0c1622798f4..4442436986c 100644 --- a/src/firecracker/src/api_server/request/mod.rs +++ b/src/firecracker/src/api_server/request/mod.rs @@ -10,6 +10,7 @@ pub mod entropy; pub mod instance_info; pub mod logger; pub mod machine_configuration; +pub mod memory; pub mod metrics; pub mod mmds; pub mod net; diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index a715bd742c4..2cd2d92f208 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 1.12.1 + version: 1.12.2 termsOfService: "" contact: email: "firecracker-maintainers@amazon.com" @@ -618,6 +618,35 @@ paths: schema: $ref: "#/definitions/Error" + /memory/mappings: + get: + summary: Gets the memory mappings with skippable pages bitmap. + operationId: getMemoryMappings + responses: + 200: + description: OK + schema: + $ref: "#/definitions/MemoryMappingsResponse" + default: + description: Internal server error + schema: + $ref: "#/definitions/Error" + + /memory: + get: + summary: Gets the memory info (resident and empty pages). + description: Returns an object with resident and empty bitmaps. The resident bitmap marks all pages that are resident. The empty bitmap marks zero pages (subset of resident pages). This is checked at the pageSize of each region. All regions must have the same page size. + operationId: getMemory + responses: + 200: + description: OK + schema: + $ref: "#/definitions/MemoryResponse" + default: + description: Internal server error + schema: + $ref: "#/definitions/Error" + /version: get: summary: Gets the Firecracker version. @@ -996,11 +1025,6 @@ definitions: vmm_version: description: MicroVM hypervisor build version. type: string - memory_regions: - type: array - description: The regions of the guest memory. - items: - $ref: "#/definitions/GuestMemoryRegionMapping" GuestMemoryRegionMapping: type: object @@ -1023,6 +1047,38 @@ definitions: description: The page size in bytes. type: integer + MemoryMappingsResponse: + type: object + description: Response containing memory region mappings. + required: + - mappings + properties: + mappings: + type: array + description: The memory region mappings. + items: + $ref: "#/definitions/GuestMemoryRegionMapping" + + MemoryResponse: + type: object + description: Response containing the memory info (resident and empty pages). + required: + - resident + - empty + properties: + resident: + type: array + description: The resident bitmap as a vector of u64 values. Each bit represents if the page is resident. + items: + type: integer + format: uint64 + empty: + type: array + description: The empty bitmap as a vector of u64 values. Each bit represents if the page is zero (empty). This is a subset of the resident pages. + items: + type: integer + format: uint64 + Logger: type: object description: diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index ce1e13307d2..be0d74b0d3b 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -26,7 +26,7 @@ use crate::vmm_config::balloon::{ use crate::vmm_config::boot_source::{BootSourceConfig, BootSourceConfigError}; use crate::vmm_config::drive::{BlockDeviceConfig, BlockDeviceUpdateConfig, DriveError}; use crate::vmm_config::entropy::{EntropyDeviceConfig, EntropyDeviceError}; -use crate::vmm_config::instance_info::InstanceInfo; +use crate::vmm_config::instance_info::{InstanceInfo, MemoryMappingsResponse, MemoryResponse}; use crate::vmm_config::machine_config::{MachineConfig, MachineConfigError, MachineConfigUpdate}; use crate::vmm_config::metrics::{MetricsConfig, MetricsConfigError}; use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError}; @@ -65,6 +65,10 @@ pub enum VmmAction { GetVmMachineConfig, /// Get microVM instance information. GetVmInstanceInfo, + /// Get memory mappings with skippable pages bitmap. + GetMemoryMappings, + /// Get memory info (resident and empty pages). + GetMemory, /// Get microVM version. GetVmmVersion, /// Flush the metrics. This action can only be called after the logger has been configured. @@ -189,6 +193,10 @@ pub enum VmmData { MmdsValue(serde_json::Value), /// The microVM instance information. InstanceInformation(InstanceInfo), + /// Memory mappings with skippable pages bitmap. + MemoryMappings(MemoryMappingsResponse), + /// Memory info (resident and empty pages). + Memory(MemoryResponse), /// The microVM version. VmmVersion(String), } @@ -419,6 +427,7 @@ impl<'a> PrebootApiController<'a> { self.vm_resources.machine_config.clone(), )), GetVmInstanceInfo => Ok(VmmData::InstanceInformation(self.instance_info.clone())), + GetMemoryMappings | GetMemory => Err(VmmActionError::OperationNotSupportedPreBoot), GetVmmVersion => Ok(VmmData::VmmVersion(self.instance_info.vmm_version.clone())), InsertBlockDevice(config) => self.insert_block_device(config), InsertNetworkDevice(config) => self.insert_net_device(config), @@ -649,17 +658,28 @@ impl RuntimeApiController { )), GetVmInstanceInfo => { let locked_vmm = self.vmm.lock().expect("Poisoned lock"); - - let mut instance_info = locked_vmm.instance_info(); - - instance_info.memory_regions = Some( - locked_vmm - .vm - .guest_memory_mappings(&VmInfo::from(&self.vm_resources)), - ); - + let instance_info = locked_vmm.instance_info(); Ok(VmmData::InstanceInformation(instance_info)) } + GetMemoryMappings => { + let locked_vmm = self.vmm.lock().expect("Poisoned lock"); + let mappings = locked_vmm + .vm + .guest_memory_mappings(&VmInfo::from(&self.vm_resources)); + + Ok(VmmData::MemoryMappings(MemoryMappingsResponse { mappings })) + } + GetMemory => { + let locked_vmm = self.vmm.lock().expect("Poisoned lock"); + let (resident_bitmap, empty_bitmap) = locked_vmm + .vm + .get_memory_info(&VmInfo::from(&self.vm_resources)) + .map_err(|e| VmmActionError::InternalVmm(VmmError::Vm(e)))?; + Ok(VmmData::Memory(MemoryResponse { + resident: resident_bitmap, + empty: empty_bitmap, + })) + } GetVmmVersion => Ok(VmmData::VmmVersion( self.vmm.lock().expect("Poisoned lock").version(), )), diff --git a/src/vmm/src/vmm_config/instance_info.rs b/src/vmm/src/vmm_config/instance_info.rs index 3ee954b6bc9..1d8f4fb57b1 100644 --- a/src/vmm/src/vmm_config/instance_info.rs +++ b/src/vmm/src/vmm_config/instance_info.rs @@ -50,3 +50,20 @@ pub struct InstanceInfo { /// The regions of the guest memory. pub memory_regions: Option>, } + +/// Response structure for the memory mappings endpoint. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MemoryMappingsResponse { + /// The memory region mappings. + pub mappings: Vec, +} + +/// Response structure for the memory endpoint. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MemoryResponse { + /// The resident bitmap as a vector of u64 values. Each bit represents if the page is resident. + pub resident: Vec, + /// The empty bitmap as a vector of u64 values. Each bit represents if the page is zero (empty). + /// This is a subset of the resident pages. + pub empty: Vec, +} diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs index 954d3efbe32..f6f99ef95c0 100644 --- a/src/vmm/src/vstate/vm.rs +++ b/src/vmm/src/vstate/vm.rs @@ -16,6 +16,7 @@ use kvm_ioctls::VmFd; use serde::{Deserialize, Serialize}; use vmm_sys_util::eventfd::EventFd; +use crate::arch::host_page_size; pub use crate::arch::{ArchVm as Vm, ArchVmError, VmState}; use crate::logger::info; use crate::persist::{CreateSnapshotError, VmInfo}; @@ -70,6 +71,8 @@ pub enum VmError { NotEnoughMemorySlots, /// Memory Error: {0} VmMemory(#[from] vm_memory::Error), + /// Invalid memory configuration: {0} + InvalidMemoryConfiguration(String), } /// Contains Vm functions that are usable across CPU architectures @@ -216,6 +219,113 @@ impl Vm { mappings } + /// Gets the memory info (resident and empty pages) for all memory regions. + /// Returns two bitmaps: resident (all resident pages) and empty (zero pages, subset of resident). + /// This checks at the pageSize of each region and requires all regions to have the same page size. + pub fn get_memory_info(&self, vm_info: &VmInfo) -> Result<(Vec, Vec), VmError> { + let mappings = self.guest_memory_mappings(vm_info); + + if mappings.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + + // Check that all regions have the same page size + let page_size = mappings[0].page_size; + if mappings.iter().any(|m| m.page_size != page_size) { + return Err(VmError::InvalidMemoryConfiguration( + "All memory regions must have the same page size".to_string(), + )); + } + + // Calculate total number of pages across all regions + let total_pages: usize = mappings.iter().map(|m| m.size / page_size).sum(); + let bitmap_size = total_pages.div_ceil(64); + let mut resident_bitmap = vec![0u64; bitmap_size]; + let mut empty_bitmap = vec![0u64; bitmap_size]; + + let mut global_page_idx = 0; + + // SAFETY: We're reading from valid memory regions that we own + unsafe { + // Pre-allocate zero buffer once per page size (reused for all pages) + // This is the most important optimization - avoids repeated allocations + let zero_buf = vec![0u8; page_size]; + + for mapping in &mappings { + // Find the memory region that matches this mapping + let mem_region = self + .guest_memory() + .iter() + .find(|region| region.as_ptr() as u64 == mapping.base_host_virt_addr) + .expect("Memory region not found for mapping"); + + let region_ptr = mem_region.as_ptr(); + let region_size = mem_region.size(); + let num_pages = region_size / page_size; + + // Use mincore on the entire region to check residency + let sys_page_size = host_page_size(); + let mincore_pages = region_size.div_ceil(sys_page_size); + let mut mincore_vec = vec![0u8; mincore_pages]; + + let mincore_result = libc::mincore( + region_ptr.cast::(), + region_size, + mincore_vec.as_mut_ptr(), + ); + + // Check each page + for page_idx in 0..num_pages { + let page_offset = page_idx * page_size; + let page_ptr = region_ptr.add(page_offset); + + // Check if page is resident using mincore + let is_resident = if mincore_result == 0 { + let page_mincore_start = page_offset / sys_page_size; + let page_mincore_count = page_size.div_ceil(sys_page_size); + if page_mincore_start + page_mincore_count <= mincore_vec.len() { + // Page is resident if any 4KB sub-page is resident (check LSB only) + mincore_vec[page_mincore_start..page_mincore_start + page_mincore_count] + .iter() + .any(|&v| (v & 0x1) != 0) + } else { + false + } + } else { + // If mincore failed, assume resident (conservative approach) + true + }; + + let bitmap_idx = global_page_idx / 64; + let bit_idx = global_page_idx % 64; + + if is_resident { + // Set bit in resident bitmap + if bitmap_idx < resident_bitmap.len() { + resident_bitmap[bitmap_idx] |= 1u64 << bit_idx; + } + + // Check if page is zero (empty) + let is_zero = libc::memcmp( + page_ptr.cast::(), + zero_buf.as_ptr().cast::(), + page_size, + ) == 0; + + // Set bit in empty bitmap if page is zero + if is_zero && bitmap_idx < empty_bitmap.len() { + empty_bitmap[bitmap_idx] |= 1u64 << bit_idx; + } + } + + global_page_idx += 1; + } + } + } + + Ok((resident_bitmap, empty_bitmap)) + } + /// Resets the KVM dirty bitmap for each of the guest's memory regions. pub fn reset_dirty_bitmap(&self) { self.guest_memory() diff --git a/tests/framework/http_api.py b/tests/framework/http_api.py index ea8efd3df4f..97a70a44a03 100644 --- a/tests/framework/http_api.py +++ b/tests/framework/http_api.py @@ -121,3 +121,5 @@ def __init__(self, api_usocket_full_name): self.snapshot_load = Resource(self, "/snapshot/load") self.cpu_config = Resource(self, "/cpu-config") self.entropy = Resource(self, "/entropy") + self.memory_mappings = Resource(self, "/memory/mappings") + self.memory = Resource(self, "/memory") diff --git a/tests/integration_tests/functional/test_api.py b/tests/integration_tests/functional/test_api.py index 864c6d5eda9..ed377bf12bf 100644 --- a/tests/integration_tests/functional/test_api.py +++ b/tests/integration_tests/functional/test_api.py @@ -17,6 +17,7 @@ import host_tools.drive as drive_tools import host_tools.network as net_tools from framework import utils, utils_cpuid +from framework.microvm import HugePagesConfig from framework.utils import get_firecracker_version_from_toml from framework.utils_cpu_templates import SUPPORTED_CPU_TEMPLATES @@ -1355,3 +1356,239 @@ def test_negative_snapshot_load_api(microvm_factory): # The snapshot/memory files above don't exist, but the request is otherwise syntactically valid. # In this case, Firecracker exits. vm.mark_killed() + + +def test_memory_mappings_pre_boot(uvm_plain): + """Test that memory mappings endpoint is not available before boot.""" + test_microvm = uvm_plain + test_microvm.spawn() + test_microvm.basic_config() + + # Use session directly since get() asserts on 200 + url = test_microvm.api.endpoint + "/memory/mappings" + res = test_microvm.api.session.get(url) + assert res.status_code == 400 + assert NOT_SUPPORTED_BEFORE_START in res.json()["fault_message"] + + +def test_memory_pre_boot(uvm_plain): + """Test that memory endpoint is not available before boot.""" + test_microvm = uvm_plain + test_microvm.spawn() + test_microvm.basic_config() + + # Use session directly since get() asserts on 200 + url = test_microvm.api.endpoint + "/memory" + res = test_microvm.api.session.get(url) + assert res.status_code == 400 + assert NOT_SUPPORTED_BEFORE_START in res.json()["fault_message"] + + +def test_memory_mappings_post_boot(uvm_plain): + """Test that memory mappings endpoint works after boot with hugepages.""" + test_microvm = uvm_plain + test_microvm.spawn() + test_microvm.basic_config(huge_pages=HugePagesConfig.HUGETLBFS_2MB) + test_microvm.start() + + response = test_microvm.api.memory_mappings.get() + assert response.status_code == 200 + + data = response.json() + assert isinstance(data, dict) + assert "mappings" in data + mappings = data["mappings"] + assert isinstance(mappings, list) + assert len(mappings) > 0 + + # Verify structure of each mapping + for mapping in mappings: + assert "base_host_virt_addr" in mapping + assert "size" in mapping + assert "offset" in mapping + assert "page_size" in mapping + assert isinstance(mapping["base_host_virt_addr"], int) + assert isinstance(mapping["size"], int) + assert isinstance(mapping["offset"], int) + assert isinstance(mapping["page_size"], int) + assert mapping["size"] > 0 + # Verify page size is 2MB (2097152 bytes) for hugepages + assert mapping["page_size"] == 2 * 1024 * 1024 + + +def test_memory_post_boot(uvm_plain): + """Test that memory endpoint works after boot with hugepages.""" + test_microvm = uvm_plain + test_microvm.spawn() + test_microvm.basic_config(huge_pages=HugePagesConfig.HUGETLBFS_2MB) + test_microvm.start() + + # Get memory mappings to determine page size and total memory + mappings_response = test_microvm.api.memory_mappings.get() + assert mappings_response.status_code == 200 + mappings_data = mappings_response.json() + assert isinstance(mappings_data, dict) + assert "mappings" in mappings_data + mappings = mappings_data["mappings"] + assert len(mappings) > 0 + + # All regions should have the same page size (2MB for hugepages) + page_size = mappings[0]["page_size"] + assert page_size == 2 * 1024 * 1024, "Expected 2MB page size for hugepages" + + # Verify all regions have the same page size + for mapping in mappings: + assert ( + mapping["page_size"] == page_size + ), "All regions must have the same page size" + + total_memory_size = sum(mapping["size"] for mapping in mappings) + total_pages = total_memory_size // page_size + expected_bitmap_size = (total_pages + 63) // 64 # ceil(total_pages / 64) + + # Get memory info + response = test_microvm.api.memory.get() + assert response.status_code == 200 + + data = response.json() + assert isinstance(data, dict) + assert "resident" in data + assert "empty" in data + resident_bitmap = data["resident"] + empty_bitmap = data["empty"] + assert isinstance(resident_bitmap, list) + assert isinstance(empty_bitmap, list) + assert len(resident_bitmap) == expected_bitmap_size + assert len(empty_bitmap) == expected_bitmap_size + + # Verify all values are valid u64 integers + for value in resident_bitmap: + assert isinstance(value, int) + assert value >= 0 + assert value <= 0xFFFFFFFFFFFFFFFF # Max u64 value + + for value in empty_bitmap: + assert isinstance(value, int) + assert value >= 0 + assert value <= 0xFFFFFFFFFFFFFFFF # Max u64 value + + # After boot, there should be at least one resident page + has_resident_page = any(value != 0 for value in resident_bitmap) + assert has_resident_page, "Expected at least one resident page after VM boot" + + # Empty pages should be a subset of resident pages + # (empty_bitmap & resident_bitmap) == empty_bitmap + for i in range(len(empty_bitmap)): + assert (empty_bitmap[i] & resident_bitmap[i]) == empty_bitmap[i], \ + "Empty pages must be a subset of resident pages" + + +@pytest.mark.nonci +def test_memory_benchmark(microvm_factory, guest_kernel_linux_6_1, rootfs): + """Benchmark the memory endpoint performance (resident + zero page checking).""" + test_microvm = microvm_factory.build(guest_kernel_linux_6_1, rootfs) + test_microvm.spawn() + + # Use larger memory size for benchmarking + # Check available hugepages and use a size that fits (need at least some headroom) + # Default to 256MB if we can't determine, or use available - 64MB headroom + try: + with open('/sys/kernel/mm/hugepages/hugepages-2048kB/free_hugepages', 'r') as f: + free_hugepages = int(f.read().strip()) + # Each hugepage is 2MB, reserve 32 pages (64MB) for system + available_mib = max(128, (free_hugepages - 32) * 2) + mem_size_mib = min(1024, available_mib) # Cap at 1GB for proper benchmark + except (FileNotFoundError, ValueError, OSError): + # Fallback to 256MB if we can't read hugepage info + mem_size_mib = 256 + test_microvm.basic_config( + mem_size_mib=mem_size_mib, + huge_pages=HugePagesConfig.HUGETLBFS_2MB + ) + # Add network interface for SSH access + test_microvm.add_net_iface() + test_microvm.start() + + # Get memory mappings to determine actual memory size + mappings_response = test_microvm.api.memory_mappings.get() + assert mappings_response.status_code == 200 + mappings_data = mappings_response.json() + mappings = mappings_data["mappings"] + + # Calculate total memory size + total_memory_bytes = sum(mapping["size"] for mapping in mappings) + total_memory_mib = total_memory_bytes / (1024 * 1024) + page_size = mappings[0]["page_size"] + + # Ensure memory is resident by writing zeros to it via guest + # This will fault in the pages and make them resident + # Using tmpfs (/dev/shm) ensures the memory is actually resident + # Allocate a reasonable portion (e.g., 256MB) to avoid freezing the sandbox + fault_memory_mib = min(256, int(total_memory_mib * 0.25)) # 25% or max 256MB + test_microvm.ssh.run("dd if=/dev/zero of=/dev/shm/zero_mem bs=1M count={} 2>/dev/null || true".format(fault_memory_mib)) + + # Give the system a moment to fault in pages + time.sleep(0.1) + + # Benchmark the /memory endpoint call + start_time = time.perf_counter() + response = test_microvm.api.memory.get() + end_time = time.perf_counter() + + assert response.status_code == 200 + data = response.json() + assert "resident" in data + assert "empty" in data + + # Verify the response is valid + resident_bitmap = data["resident"] + empty_bitmap = data["empty"] + + # Calculate expected bitmap size + page_size = mappings[0]["page_size"] + total_pages = total_memory_bytes // page_size + expected_bitmap_size = (total_pages + 63) // 64 + + assert len(resident_bitmap) == expected_bitmap_size + assert len(empty_bitmap) == expected_bitmap_size + + # Count actual resident pages (faulted-in memory) + resident_page_count = 0 + for bitmap_value in resident_bitmap: + # Count set bits in each u64 value + resident_page_count += bin(bitmap_value).count('1') + + # Calculate resident memory size (actual memory that was checked) + resident_memory_bytes = resident_page_count * page_size + resident_memory_mib = resident_memory_bytes / (1024 * 1024) + + # Calculate elapsed time and throughput based on actual resident memory + elapsed_seconds = end_time - start_time + + if resident_memory_bytes > 0: + throughput_mib_per_sec = resident_memory_mib / elapsed_seconds + time_per_mb_ms = (elapsed_seconds * 1000) / resident_memory_mib + else: + throughput_mib_per_sec = 0 + time_per_mb_ms = 0 + + # Count empty pages + empty_page_count = 0 + for bitmap_value in empty_bitmap: + empty_page_count += bin(bitmap_value).count('1') + + # Print benchmark results + print(f"\n{'='*60}") + print(f"Memory Benchmark Results") + print(f"{'='*60}") + print(f"Total Memory: {total_memory_mib:.2f} MiB ({total_memory_bytes / (1024**3):.3f} GB)") + print(f"Resident Pages: {resident_page_count} / {total_pages} ({resident_page_count * 100 / total_pages:.1f}%)") + print(f"Resident Memory: {resident_memory_mib:.2f} MiB ({resident_memory_bytes / (1024**3):.3f} GB)") + print(f"Empty Pages: {empty_page_count} / {resident_page_count} ({empty_page_count * 100 / resident_page_count if resident_page_count > 0 else 0:.1f}% of resident)") + print(f"Elapsed Time: {elapsed_seconds*1000:.2f} ms") + print(f"Throughput (resident): {throughput_mib_per_sec:.2f} MiB/s") + print(f"Time per MB (resident): {time_per_mb_ms:.3f} ms/MB") + print(f"{'='*60}\n") + + # Verify at least some pages are resident + assert resident_page_count > 0, "Expected at least one resident page" From ecaa883d74b7f3f0a7973b2cab428ea12a12b8d7 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 17:55:47 -0800 Subject: [PATCH 09/18] Add PR tests --- .github/workflows/pr_tests.yml | 88 ++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/pr_tests.yml diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml new file mode 100644 index 00000000000..e409fbd047f --- /dev/null +++ b/.github/workflows/pr_tests.yml @@ -0,0 +1,88 @@ +name: PR Tests + +on: + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + rust-unit-tests: + name: Rust Unit Tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Run unit tests + run: cargo test --workspace + + - name: Run doc tests + run: cargo test --workspace --doc + + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Build release + run: cargo build --release --workspace + + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + functional-tests: + name: Functional Tests + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Python dependencies + run: | + pip install pytest pytest-timeout pytest-json-report requests psutil tenacity filelock "urllib3<2.0" requests-unixsocket aws-embedded-metrics + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build Firecracker release + run: cargo build --release + + - name: Check KVM availability + id: check_kvm + run: | + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + echo "kvm_available=true" >> $GITHUB_OUTPUT + else + echo "kvm_available=false" >> $GITHUB_OUTPUT + echo "⚠️ KVM not available - functional tests will skip" + fi + + - name: Run functional tests + if: steps.check_kvm.outputs.kvm_available == 'true' + run: | + cd tests + sudo env PATH="$PATH" PYTHONPATH=. pytest -m 'not nonci and not no_block_pr' integration_tests/functional/ + continue-on-error: true + + - name: Collect functional tests (dry run) + if: steps.check_kvm.outputs.kvm_available == 'false' + run: | + cd tests + pytest --collect-only -m 'not nonci and not no_block_pr' integration_tests/functional/ From 29d8160745200e274370e6b91de6b366e439e3bd Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 17:59:44 -0800 Subject: [PATCH 10/18] Improve test gha --- .github/workflows/pr_tests.yml | 48 +++++----------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index e409fbd047f..6363e85c5f9 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -15,13 +15,10 @@ jobs: uses: actions/checkout@v4 - name: Set up Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@1.85.0 - name: Run unit tests - run: cargo test --workspace - - - name: Run doc tests - run: cargo test --workspace --doc + run: cargo test --workspace --lib --bins build: name: Build @@ -31,7 +28,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@1.85.0 with: components: clippy @@ -44,45 +41,14 @@ jobs: functional-tests: name: Functional Tests runs-on: ubuntu-latest - needs: build steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Install Python dependencies + - name: Run functional tests with devtool run: | - pip install pytest pytest-timeout pytest-json-report requests psutil tenacity filelock "urllib3<2.0" requests-unixsocket aws-embedded-metrics - - - name: Set up Rust - uses: dtolnay/rust-toolchain@stable - - - name: Build Firecracker release - run: cargo build --release - - - name: Check KVM availability - id: check_kvm - run: | - if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then - echo "kvm_available=true" >> $GITHUB_OUTPUT - else - echo "kvm_available=false" >> $GITHUB_OUTPUT - echo "⚠️ KVM not available - functional tests will skip" - fi - - - name: Run functional tests - if: steps.check_kvm.outputs.kvm_available == 'true' - run: | - cd tests - sudo env PATH="$PATH" PYTHONPATH=. pytest -m 'not nonci and not no_block_pr' integration_tests/functional/ + ./tools/devtool -y test -- integration_tests/functional/ continue-on-error: true - - - name: Collect functional tests (dry run) - if: steps.check_kvm.outputs.kvm_available == 'false' - run: | - cd tests - pytest --collect-only -m 'not nonci and not no_block_pr' integration_tests/functional/ From e22005f146a7f61877498e7f339fa1a4255dff28 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:04:36 -0800 Subject: [PATCH 11/18] Improve the tests --- .github/workflows/pr_tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 6363e85c5f9..bc3b0eff24e 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -14,6 +14,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libseccomp-dev pkg-config build-essential + - name: Set up Rust uses: dtolnay/rust-toolchain@1.85.0 @@ -27,6 +32,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libseccomp-dev pkg-config build-essential + - name: Set up Rust uses: dtolnay/rust-toolchain@1.85.0 with: From 2eaa08e464604ec7d21bc363acbfbd4992a758e1 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:15:47 -0800 Subject: [PATCH 12/18] Tweak test --- .github/workflows/pr_tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index bc3b0eff24e..80e079a4a83 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -58,7 +58,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Install AWS CLI + run: | + sudo apt-get update + sudo apt-get install -y awscli + - name: Run functional tests with devtool run: | ./tools/devtool -y test -- integration_tests/functional/ - continue-on-error: true From fa95bcf2ae6e0ce37699fd639dc705d6fde0436e Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:18:21 -0800 Subject: [PATCH 13/18] Fix awscli install --- .github/workflows/pr_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 80e079a4a83..ecadb5ae307 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -60,8 +60,10 @@ jobs: - name: Install AWS CLI run: | - sudo apt-get update - sudo apt-get install -y awscli + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + rm -rf aws awscliv2.zip - name: Run functional tests with devtool run: | From 103cf7c90f270125594ed5e5b545d7429a56467f Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:22:54 -0800 Subject: [PATCH 14/18] Cleanup --- .github/workflows/pr_tests.yml | 43 +--------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index ecadb5ae307..230d1151629 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -7,47 +7,6 @@ env: CARGO_TERM_COLOR: always jobs: - rust-unit-tests: - name: Rust Unit Tests - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libseccomp-dev pkg-config build-essential - - - name: Set up Rust - uses: dtolnay/rust-toolchain@1.85.0 - - - name: Run unit tests - run: cargo test --workspace --lib --bins - - build: - name: Build - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libseccomp-dev pkg-config build-essential - - - name: Set up Rust - uses: dtolnay/rust-toolchain@1.85.0 - with: - components: clippy - - - name: Build release - run: cargo build --release --workspace - - - name: Clippy - run: cargo clippy --workspace --all-targets -- -D warnings - functional-tests: name: Functional Tests runs-on: ubuntu-latest @@ -67,4 +26,4 @@ jobs: - name: Run functional tests with devtool run: | - ./tools/devtool -y test -- integration_tests/functional/ + ./tools/devtool -y test From e4006e78c9752326009d7c4ff7f84acd767328e6 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:23:05 -0800 Subject: [PATCH 15/18] Cleanup --- .github/workflows/pr_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 230d1151629..4d84ba7d45d 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -7,8 +7,8 @@ env: CARGO_TERM_COLOR: always jobs: - functional-tests: - name: Functional Tests + tests: + name: Tests runs-on: ubuntu-latest steps: - name: Checkout repository @@ -24,6 +24,6 @@ jobs: sudo ./aws/install rm -rf aws awscliv2.zip - - name: Run functional tests with devtool + - name: Run tests with devtool run: | ./tools/devtool -y test From ffc30eb2d711851a9b9cb7e9331967e371acf45c Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:25:31 -0800 Subject: [PATCH 16/18] Remove aws install --- .github/workflows/pr_tests.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 4d84ba7d45d..67488029d22 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -17,13 +17,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Install AWS CLI - run: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install - rm -rf aws awscliv2.zip - - name: Run tests with devtool run: | ./tools/devtool -y test From e82440bfe07fb0690cd1fa3c3e27b53f7c24fe88 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 18:42:19 -0800 Subject: [PATCH 17/18] Use only functional tests for now --- .github/workflows/pr_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 67488029d22..eba89af46ba 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -19,4 +19,4 @@ jobs: - name: Run tests with devtool run: | - ./tools/devtool -y test + ./tools/devtool -y test -- integration_tests/functional/ From a3608adc9dfb2871fcc18505725ea615abf44eac Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 10 Dec 2025 23:48:10 -0800 Subject: [PATCH 18/18] Limit parallelism --- .github/workflows/pr_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index eba89af46ba..5245e19d08d 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -18,5 +18,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Run tests with devtool + env: + PYTEST_ADDOPTS: "-n 2" run: | ./tools/devtool -y test -- integration_tests/functional/