From 42a5058e0fe116c000c9eaa2dde93726197beeb8 Mon Sep 17 00:00:00 2001 From: Jethro Beekman Date: Thu, 15 May 2025 09:27:40 -0700 Subject: [PATCH 1/3] Revert "Bump cargo_toml from 0.20.4 to 0.22.0" This reverts commit 0829b8367e8db8d5a01de5392d3bfaa2b7c54d71. --- Cargo.lock | 4 ++-- fortanix-vme/aws-nitro-enclaves/eif-tools/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fe16685..901a69e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,9 +390,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cargo_toml" -version = "0.22.0" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e98c0c5c444b1eaed8adefa6eca2abdb8f036c897d5b6f49d47bef2f140ba1" +checksum = "ad639525b1c67b6a298f378417b060fbc04618bea559482a8484381cce27d965" dependencies = [ "serde", "toml 0.8.19", diff --git a/fortanix-vme/aws-nitro-enclaves/eif-tools/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/eif-tools/Cargo.toml index 7d51b057..2d4f9b89 100644 --- a/fortanix-vme/aws-nitro-enclaves/eif-tools/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/eif-tools/Cargo.toml @@ -15,7 +15,7 @@ categories = ["command-line-utilities"] [dependencies] anyhow = "1.0" -cargo_toml = "0.22.0" +cargo_toml = "0.20.4" clap = "2.34" elf = "0.0.10" env_logger = "0.9" From 9b51e657c56069687a3dd54e7390469f2843fca8 Mon Sep 17 00:00:00 2001 From: Jethro Beekman Date: Thu, 15 May 2025 11:58:57 -0700 Subject: [PATCH 2/3] TCS entry: clobber register can be used for input --- intel-sgx/enclave-runner/src/tcs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intel-sgx/enclave-runner/src/tcs.rs b/intel-sgx/enclave-runner/src/tcs.rs index a0f7703b..181126c0 100644 --- a/intel-sgx/enclave-runner/src/tcs.rs +++ b/intel-sgx/enclave-runner/src/tcs.rs @@ -197,7 +197,7 @@ pub(crate) fn coenter( inout("r8") p4, inout("r9") p5, inout("r10") debug_buf_ptr => _, - out("r11") _, + lateout("r11") _, options(nostack, att_syntax) ); } From e8d02ab7c01d74f84a446b319740c39ddd7110d1 Mon Sep 17 00:00:00 2001 From: Jethro Beekman Date: Thu, 15 May 2025 12:01:30 -0700 Subject: [PATCH 3/3] track_aex_count feature for enclave-runner --- intel-sgx/enclave-runner/Cargo.toml | 1 + intel-sgx/enclave-runner/src/command.rs | 9 ++- intel-sgx/enclave-runner/src/tcs.rs | 71 +++++++++++++++++-- .../enclave-runner/src/usercalls/interface.rs | 2 +- intel-sgx/enclave-runner/src/usercalls/mod.rs | 47 ++++++++++-- 5 files changed, 119 insertions(+), 11 deletions(-) diff --git a/intel-sgx/enclave-runner/Cargo.toml b/intel-sgx/enclave-runner/Cargo.toml index d58be8a1..bc36a802 100644 --- a/intel-sgx/enclave-runner/Cargo.toml +++ b/intel-sgx/enclave-runner/Cargo.toml @@ -42,3 +42,4 @@ futures = { version = "0.3", features = ["compat", "io-compat"] } # MIT/Apache-2 [features] default = ["crypto-openssl"] crypto-openssl = ["openssl", "sgxs/crypto-openssl"] +instrumentation = [] diff --git a/intel-sgx/enclave-runner/src/command.rs b/intel-sgx/enclave-runner/src/command.rs index ffaefd20..13441311 100644 --- a/intel-sgx/enclave-runner/src/command.rs +++ b/intel-sgx/enclave-runner/src/command.rs @@ -25,6 +25,7 @@ pub struct Command { force_time_usercalls: bool, cmd_args: Vec>, num_worker_threads: usize, + track_aex_count: bool, } impl MappingInfo for Command { @@ -61,6 +62,7 @@ impl Command { force_time_usercalls, cmd_args, num_worker_threads, + track_aex_count: false, } } @@ -68,7 +70,12 @@ impl Command { EnclaveBuilder::new(enclave_path.as_ref()).build(loader) } + #[cfg(feature = "instrumentation")] + pub fn track_aex_count(&mut self) { + self.track_aex_count = true; + } + pub fn run(self) -> Result<(), Error> { - EnclaveState::main_entry(self.main, self.threads, self.usercall_ext, self.forward_panics, self.force_time_usercalls, self.cmd_args, self.num_worker_threads) + EnclaveState::main_entry(self.main, self.threads, self.usercall_ext, self.forward_panics, self.force_time_usercalls, self.track_aex_count, self.cmd_args, self.num_worker_threads) } } diff --git a/intel-sgx/enclave-runner/src/tcs.rs b/intel-sgx/enclave-runner/src/tcs.rs index 181126c0..b84d3061 100644 --- a/intel-sgx/enclave-runner/src/tcs.rs +++ b/intel-sgx/enclave-runner/src/tcs.rs @@ -10,6 +10,9 @@ use std::convert::{TryFrom, TryInto}; use std::fmt; use std::io::Write; use std::os::raw::c_void; +use std::sync::atomic::AtomicUsize; +#[cfg(feature = "instrumentation")] +use std::sync::Arc; use sgx_isa::Enclu; use sgxs::loader::Tcs; @@ -22,10 +25,41 @@ pub enum CoResult { Return(R), } +#[cfg(feature = "instrumentation")] +#[derive(Clone, Debug)] +pub(crate) struct AexCount(Option>); +#[cfg(not(feature = "instrumentation"))] +#[derive(Clone, Debug)] +pub(crate) struct AexCount(()); + +impl AexCount { + pub fn none() -> AexCount { + #[cfg(feature = "instrumentation")] + { AexCount(None) } + #[cfg(not(feature = "instrumentation"))] + { AexCount(()) } + } + + fn get(&self) -> Option<&AtomicUsize> { + #[cfg(feature = "instrumentation")] + { self.0.as_ref().map(|p| &**p) } + #[cfg(not(feature = "instrumentation"))] + { None } + } +} + +#[cfg(feature = "instrumentation")] +impl From> for AexCount { + fn from(v: Arc) -> AexCount { + AexCount(Some(v)) + } +} + #[derive(Debug)] pub struct Usercall { tcs: T, parameters: (u64, u64, u64, u64, u64), + aex_count: AexCount, } pub type ThreadResult = CoResult, (T, u64, u64)>; @@ -40,7 +74,7 @@ impl Usercall { retval: (u64, u64), debug_buf: Option<&RefCell>, ) -> ThreadResult { - coenter(self.tcs, 0, retval.0, retval.1, 0, 0, debug_buf) + coenter(self.tcs, 0, retval.0, retval.1, 0, 0, debug_buf, self.aex_count) } pub fn tcs_address(&self) -> *mut c_void { @@ -56,6 +90,7 @@ pub(crate) fn coenter( mut p4: u64, mut p5: u64, debug_buf: Option<&RefCell>, + aex_count: AexCount, ) -> ThreadResult { /// Check if __vdso_sgx_enter_enclave exists. We're using weak linkage, so /// it might not. @@ -101,7 +136,33 @@ pub(crate) fn coenter( uninit_debug_buf.as_mut_ptr() as *mut _ } }; - if has_vdso_sgx_enter_enclave() { + if let Some(aex_count) = aex_count.get() { + asm!(" + decq ({1}) + push {1} + lea 1f(%rip), %rcx // set SGX AEP + xchg {0}, %rbx +1: mov (%rsp), %r12 + incq (%r12) + enclu + addq $8, %rsp + xchg %rbx, {0} + ", + inout(reg) tcs.address() => _, // rbx is used internally by LLVM and cannot be used as an operand for inline asm (#84658) + in(reg) aex_count, + inout("eax") Enclu::EEnter as u32 => sgx_result, + out("rcx") _, + inout("rdx") p3, + inout("rdi") p1, + inout("rsi") p2, + inout("r8") p4, + inout("r9") p5, + inout("r10") debug_buf_ptr => _, + lateout("r11") _, + lateout("r12") _, + options(nostack, att_syntax) + ); + } else if has_vdso_sgx_enter_enclave() { #[repr(C)] #[derive(Default)] struct SgxEnclaveRun { @@ -172,8 +233,9 @@ pub(crate) fn coenter( eprintln!("Enclave triggered exception, treating as panic: {:?}", run); } return CoResult::Yield(Usercall { - tcs: tcs, + tcs, parameters: (crate::usercalls::abi::UsercallList::exit as _, true as _, 0, 0, 0), + aex_count, }); }, _ => panic!("Error entering enclave (VDSO): ret = success, run = {:?}", run), @@ -211,8 +273,9 @@ pub(crate) fn coenter( CoResult::Return((tcs, p2, p3)) } else { CoResult::Yield(Usercall { - tcs: tcs, + tcs, parameters: (p1, p2, p3, p4, p5), + aex_count, }) } } diff --git a/intel-sgx/enclave-runner/src/usercalls/interface.rs b/intel-sgx/enclave-runner/src/usercalls/interface.rs index b10c7e23..00efc94d 100644 --- a/intel-sgx/enclave-runner/src/usercalls/interface.rs +++ b/intel-sgx/enclave-runner/src/usercalls/interface.rs @@ -178,7 +178,7 @@ impl<'future, 'ioinput: 'future, 'tcs: 'ioinput> Usercalls<'future> for Handler< self, ) -> std::pin::Pin)> + 'future>> { async move { - let ret = Ok(self.0.launch_thread().to_sgx_result()); + let ret = Ok(self.0.launch_thread().await.to_sgx_result()); return (self, ret); }.boxed_local() } diff --git a/intel-sgx/enclave-runner/src/usercalls/mod.rs b/intel-sgx/enclave-runner/src/usercalls/mod.rs index bb7e51a2..81128681 100644 --- a/intel-sgx/enclave-runner/src/usercalls/mod.rs +++ b/intel-sgx/enclave-runner/src/usercalls/mod.rs @@ -611,6 +611,7 @@ impl PendingEvents { struct RunningTcs { tcs_address: TcsAddress, mode: EnclaveEntry, + aex_count: tcs::AexCount, } enum EnclaveKind { @@ -665,6 +666,7 @@ pub(crate) struct EnclaveState { // Once set to Some, the guards should not be dropped for the lifetime of the enclave. fifo_guards: Mutex>, return_queue_tx: Mutex>>, + aex_counts: Option>>>, } struct Work { @@ -682,7 +684,7 @@ impl Work { let buf = RefCell::new([0u8; 1024]); let usercall_send_data = match self.entry { CoEntry::Initial(erased_tcs, p1, p2, p3, p4, p5) => { - let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, Some(&buf)); + let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, Some(&buf), self.tcs.aex_count.clone()); UsercallSendData::Sync(coresult, self.tcs, buf) } CoEntry::Resume(usercall, coresult) => { @@ -736,6 +738,7 @@ impl EnclaveState { threads_vector: Vec, forward_panics: bool, force_time_usercalls: bool, + aex_counts: Option>>>, ) -> Arc { let mut fds = FnvHashMap::default(); @@ -779,6 +782,7 @@ impl EnclaveState { force_time_usercalls, fifo_guards: Mutex::new(None), return_queue_tx: Mutex::new(None), + aex_counts, }) } @@ -1053,7 +1057,10 @@ impl EnclaveState { } }); - local_set.block_on(&mut rt, select_fut.unit_error()).unwrap() + let ret = local_set.block_on(&mut rt, select_fut.unit_error()).unwrap(); + #[cfg(feature = "instrumentation")] + tokio::runtime::Runtime::new().unwrap().block_on(enclave.print_aex_counts()); + ret } fn run( @@ -1105,12 +1112,18 @@ impl EnclaveState { usercall_ext: Option>, forward_panics: bool, force_time_usercalls: bool, + track_aex_count: bool, cmd_args: Vec>, num_of_worker_threads: usize, ) -> StdResult<(), anyhow::Error> { assert!(num_of_worker_threads > 0, "worker_threads cannot be zero"); let mut event_queues = FnvHashMap::with_capacity_and_hasher(threads.len() + 1, Default::default()); + let mut aex_counts: Option>> = if track_aex_count { + Some(FnvHashMap::with_capacity_and_hasher(threads.len() + 1, Default::default())) + } else { + None + }; let main = Self::event_queue_add_tcs(&mut event_queues, main); let mut args = Vec::with_capacity(cmd_args.len()); @@ -1123,10 +1136,16 @@ impl EnclaveState { let argc = args.len(); let argv = Box::into_raw(args.into_boxed_slice()) as *const u8; + let aex_count = match &mut aex_counts { + #[cfg(feature = "instrumentation")] + Some(aex_counts) => tcs::AexCount::from(aex_counts.entry(main.tcs.address()).or_default().clone()), + _ => tcs::AexCount::none(), + }; let main_work = Work { tcs: RunningTcs { tcs_address: main.tcs.address(), mode: EnclaveEntry::ExecutableMain, + aex_count, }, entry: CoEntry::Initial(main.tcs, argv as _, argc as _, 0, 0, 0), }; @@ -1137,7 +1156,7 @@ impl EnclaveState { other_reasons: vec![], }), }); - let enclave = EnclaveState::new(kind, event_queues, usercall_ext, threads, forward_panics, force_time_usercalls); + let enclave = EnclaveState::new(kind, event_queues, usercall_ext, threads, forward_panics, force_time_usercalls, aex_counts.map(Mutex::new)); let main_result = EnclaveState::run(enclave.clone(), num_of_worker_threads, main_work); @@ -1194,7 +1213,7 @@ impl EnclaveState { let kind = EnclaveKind::Library(Library {}); - let enclave = EnclaveState::new(kind, event_queues, usercall_ext, threads, forward_panics, force_time_usercalls); + let enclave = EnclaveState::new(kind, event_queues, usercall_ext, threads, forward_panics, force_time_usercalls, None); return enclave; } @@ -1211,6 +1230,7 @@ impl EnclaveState { tcs: RunningTcs { tcs_address: thread.tcs.address(), mode: EnclaveEntry::Library, + aex_count: tcs::AexCount::none(), }, entry: CoEntry::Initial(thread.tcs, p1, p2, p3, p4, p5), }; @@ -1244,6 +1264,17 @@ impl EnclaveState { pending_events.abort(); } } + + #[cfg(feature = "instrumentation")] + async fn print_aex_counts(&self) { + if let Some(aex_counts) = &self.aex_counts { + eprintln!("===AEX counts per TCS==="); + for (addr, count) in aex_counts.lock().await.iter() { + eprintln!("{:p} {}", *addr, count.load(Ordering::Relaxed)); + } + eprintln!("========================"); + } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -1540,7 +1571,7 @@ impl<'tcs> IOHandlerInput<'tcs> { } #[inline(always)] - fn launch_thread(&self) -> IoResult<()> { + async fn launch_thread(&self) -> IoResult<()> { // check if enclave is of type command self.enclave .kind @@ -1553,10 +1584,16 @@ impl<'tcs> IOHandlerInput<'tcs> { } }; + let aex_count = match &self.enclave.aex_counts { + #[cfg(feature = "instrumentation")] + Some(aex_counts) => tcs::AexCount::from(aex_counts.lock().await.entry(new_tcs.tcs.address()).or_default().clone()), + _ => tcs::AexCount::none(), + }; let ret = self.work_sender.send(Work { tcs: RunningTcs { tcs_address: new_tcs.tcs.address(), mode: EnclaveEntry::ExecutableNonMain, + aex_count, }, entry: CoEntry::Initial(new_tcs.tcs, 0, 0, 0, 0, 0), });