From 97240acc1a5940cb724da9f6f101a307436f0caf Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Mon, 14 Apr 2025 10:06:26 +0200 Subject: [PATCH 01/12] draft for corpus refactoring --- libafl/Cargo.toml | 5 +- libafl/src/corpus/cached.rs | 282 ------------ libafl/src/corpus/combined.rs | 392 +++++++++++++++++ libafl/src/corpus/inmemory.rs | 479 --------------------- libafl/src/corpus/inmemory_ondisk.rs | 519 ----------------------- libafl/src/corpus/mod.rs | 149 ++++--- libafl/src/corpus/nop.rs | 23 +- libafl/src/corpus/ondisk.rs | 279 ------------ libafl/src/corpus/single.rs | 114 +++++ libafl/src/corpus/store/inmemory.rs | 104 +++++ libafl/src/corpus/store/maps.rs | 276 ++++++++++++ libafl/src/corpus/store/mod.rs | 72 ++++ libafl/src/corpus/store/ondisk.rs | 321 ++++++++++++++ libafl/src/corpus/testcase.rs | 351 +++++---------- libafl/src/executors/inprocess/mod.rs | 11 +- libafl/src/feedbacks/mod.rs | 3 - libafl/src/fuzzer/mod.rs | 39 +- libafl/src/mutators/encoded_mutations.rs | 8 +- libafl/src/mutators/gramatron.rs | 7 +- libafl/src/mutators/mutations.rs | 23 +- libafl/src/mutators/numeric.rs | 4 +- libafl/src/state/mod.rs | 18 +- 22 files changed, 1544 insertions(+), 1935 deletions(-) delete mode 100644 libafl/src/corpus/cached.rs create mode 100644 libafl/src/corpus/combined.rs delete mode 100644 libafl/src/corpus/inmemory.rs delete mode 100644 libafl/src/corpus/inmemory_ondisk.rs delete mode 100644 libafl/src/corpus/ondisk.rs create mode 100644 libafl/src/corpus/single.rs create mode 100644 libafl/src/corpus/store/inmemory.rs create mode 100644 libafl/src/corpus/store/maps.rs create mode 100644 libafl/src/corpus/store/mod.rs create mode 100644 libafl/src/corpus/store/ondisk.rs diff --git a/libafl/Cargo.toml b/libafl/Cargo.toml index 2b1f227d000..a44acdf0297 100644 --- a/libafl/Cargo.toml +++ b/libafl/Cargo.toml @@ -227,7 +227,7 @@ hashbrown = { workspace = true, features = [ "ahash", ], default-features = false } # A faster hashmap, nostd compatible num-traits = { workspace = true, default-features = false } -serde = { workspace = true, features = ["alloc"] } # serialization lib +serde = { workspace = true, features = ["alloc", "rc"] } # serialization lib postcard = { workspace = true } # no_std compatible serde serialization format bincode = { version = "1.3.3", optional = true } bitbybit = { workspace = true } @@ -236,9 +236,10 @@ ahash = { workspace = true } # The hash function already used in hashbrown meminterval = { workspace = true, features = ["serde"] } backtrace = { workspace = true, optional = true } # Used to get the stacktrace in StacktraceObserver typed-builder = { workspace = true, optional = true } # Implement the builder pattern at compiletime +fs4 = "0.13.1" serde_json = { workspace = true, optional = true, default-features = false, features = [ - "alloc", + "alloc" ] } nix = { workspace = true, optional = true, features = [ "signal", diff --git a/libafl/src/corpus/cached.rs b/libafl/src/corpus/cached.rs deleted file mode 100644 index c9befc19e28..00000000000 --- a/libafl/src/corpus/cached.rs +++ /dev/null @@ -1,282 +0,0 @@ -//! The [`CachedOnDiskCorpus`] stores [`Testcase`]s to disk, keeping a subset of them in memory/cache, evicting in a FIFO manner. - -use alloc::{collections::vec_deque::VecDeque, string::String}; -use core::cell::{Ref, RefCell, RefMut}; -use std::path::Path; - -use serde::{Deserialize, Serialize}; - -use crate::{ - Error, - corpus::{ - Corpus, CorpusId, HasTestcase, Testcase, inmemory_ondisk::InMemoryOnDiskCorpus, - ondisk::OnDiskMetadataFormat, - }, - inputs::Input, -}; - -/// A corpus that keeps a maximum number of [`Testcase`]s in memory -/// and load them from disk, when they are being used. -/// The eviction policy is FIFO. -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct CachedOnDiskCorpus { - inner: InMemoryOnDiskCorpus, - cached_indexes: RefCell>, - cache_max_len: usize, -} - -impl CachedOnDiskCorpus -where - I: Input, -{ - fn cache_testcase<'a>( - &'a self, - testcase: &'a RefCell>, - id: CorpusId, - ) -> Result<(), Error> { - if testcase.borrow().input().is_none() { - self.load_input_into(&mut testcase.borrow_mut())?; - let mut borrowed_num = 0; - while self.cached_indexes.borrow().len() >= self.cache_max_len { - let removed = self.cached_indexes.borrow_mut().pop_front().unwrap(); - - if let Ok(mut borrowed) = self.inner.get_from_all(removed)?.try_borrow_mut() { - *borrowed.input_mut() = None; - } else { - self.cached_indexes.borrow_mut().push_back(removed); - borrowed_num += 1; - if self.cache_max_len == borrowed_num { - break; - } - } - } - self.cached_indexes.borrow_mut().push_back(id); - } - Ok(()) - } -} - -impl Corpus for CachedOnDiskCorpus -where - I: Input, -{ - /// Returns the number of all enabled entries - #[inline] - fn count(&self) -> usize { - self.inner.count() - } - - /// Returns the number of all disabled entries - fn count_disabled(&self) -> usize { - self.inner.count_disabled() - } - - /// Returns the number of elements including disabled entries - #[inline] - fn count_all(&self) -> usize { - self.inner.count_all() - } - - /// Add an enabled testcase to the corpus and return its index - #[inline] - fn add(&mut self, testcase: Testcase) -> Result { - self.inner.add(testcase) - } - - /// Add a disabled testcase to the corpus and return its index - #[inline] - fn add_disabled(&mut self, testcase: Testcase) -> Result { - self.inner.add_disabled(testcase) - } - - /// Replaces the testcase at the given idx - #[inline] - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - // TODO finish - self.inner.replace(id, testcase) - } - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases. - fn remove(&mut self, id: CorpusId) -> Result, Error> { - let testcase = self.inner.remove(id)?; - self.cached_indexes.borrow_mut().retain(|e| *e != id); - Ok(testcase) - } - - /// Get by id; considers only enabled testcases - #[inline] - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { - let testcase = { self.inner.get(id)? }; - self.cache_testcase(testcase, id)?; - Ok(testcase) - } - /// Get by id; considers both enabled and disabled testcases - #[inline] - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { - let testcase = { self.inner.get_from_all(id)? }; - self.cache_testcase(testcase, id)?; - Ok(testcase) - } - - /// Current testcase scheduled - #[inline] - fn current(&self) -> &Option { - self.inner.current() - } - - /// Current testcase scheduled (mutable) - #[inline] - fn current_mut(&mut self) -> &mut Option { - self.inner.current_mut() - } - - #[inline] - fn next(&self, id: CorpusId) -> Option { - self.inner.next(id) - } - - /// Peek the next free corpus id - #[inline] - fn peek_free_id(&self) -> CorpusId { - self.inner.peek_free_id() - } - - #[inline] - fn prev(&self, id: CorpusId) -> Option { - self.inner.prev(id) - } - - #[inline] - fn first(&self) -> Option { - self.inner.first() - } - - #[inline] - fn last(&self) -> Option { - self.inner.last() - } - - /// Get the nth corpus id; considers only enabled testcases - #[inline] - fn nth(&self, nth: usize) -> CorpusId { - self.inner.nth(nth) - } - /// Get the nth corpus id; considers both enabled and disabled testcases - #[inline] - fn nth_from_all(&self, nth: usize) -> CorpusId { - self.inner.nth_from_all(nth) - } - - #[inline] - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { - self.inner.load_input_into(testcase) - } - - #[inline] - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { - self.inner.store_input_from(testcase) - } -} - -impl HasTestcase for CachedOnDiskCorpus -where - I: Input, -{ - fn testcase(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow()) - } - - fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow_mut()) - } -} - -impl CachedOnDiskCorpus { - /// Creates the [`CachedOnDiskCorpus`]. - /// - /// This corpus stores (and reads) all testcases to/from disk - /// - /// By default, it stores metadata for each [`Testcase`] as prettified json. - /// Metadata will be written to a file named `..metadata` - /// the metadata may include objective reason, specific information for a fuzz job, and more. - /// - /// If you don't want metadata, use [`CachedOnDiskCorpus::no_meta`]. - /// to pick a different metadata format, use [`CachedOnDiskCorpus::with_meta_format`]. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn new

(dir_path: P, cache_max_len: usize) -> Result - where - P: AsRef, - { - Self::_new(InMemoryOnDiskCorpus::new(dir_path)?, cache_max_len) - } - - /// Creates an [`CachedOnDiskCorpus`] that does not store [`Testcase`] metadata to disk. - pub fn no_meta

(dir_path: P, cache_max_len: usize) -> Result - where - P: AsRef, - { - Self::_new(InMemoryOnDiskCorpus::no_meta(dir_path)?, cache_max_len) - } - - /// Creates the [`CachedOnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format

( - dir_path: P, - cache_max_len: usize, - meta_format: Option, - ) -> Result - where - P: AsRef, - { - Self::_new( - InMemoryOnDiskCorpus::with_meta_format(dir_path, meta_format)?, - cache_max_len, - ) - } - - /// Creates the [`CachedOnDiskCorpus`] specifying the metadata format and the prefix to prepend - /// to each testcase. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format_and_prefix

( - dir_path: P, - cache_max_len: usize, - meta_format: Option, - prefix: Option, - locking: bool, - ) -> Result - where - P: AsRef, - { - Self::_new( - InMemoryOnDiskCorpus::with_meta_format_and_prefix( - dir_path, - meta_format, - prefix, - locking, - )?, - cache_max_len, - ) - } - - /// Internal constructor `fn` - fn _new(on_disk_corpus: InMemoryOnDiskCorpus, cache_max_len: usize) -> Result { - if cache_max_len == 0 { - return Err(Error::illegal_argument( - "The max cache len in CachedOnDiskCorpus cannot be 0", - )); - } - Ok(Self { - inner: on_disk_corpus, - cached_indexes: RefCell::new(VecDeque::new()), - cache_max_len, - }) - } - - /// Fetch the inner corpus - pub fn inner(&self) -> &InMemoryOnDiskCorpus { - &self.inner - } -} diff --git a/libafl/src/corpus/combined.rs b/libafl/src/corpus/combined.rs new file mode 100644 index 00000000000..e02bbbf92e1 --- /dev/null +++ b/libafl/src/corpus/combined.rs @@ -0,0 +1,392 @@ +use core::{cell::RefCell, marker::PhantomData}; +use std::{collections::VecDeque, rc::Rc, vec::Vec}; + +use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; + +use super::{Corpus, CorpusCounter, CorpusId, Testcase, store::Store}; + +/// A [`CombinedCorpus`] tries first to use the main store according to some policy. +/// If it fails, it falls back to the secondary store. +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct CombinedCorpus { + /// The cache store + cache_store: RefCell, + /// The fallback store + fallback_store: FS, + /// The policty taking decisions + cache: RefCell, + /// The corpus ID counter + counter: CorpusCounter, + /// The keys in order (use `Vec::binary_search`) + keys: Vec, + /// The current ID + current: Option, + phantom: PhantomData, +} + +pub struct FifoCache { + cached_ids: VecDeque, + cache_max_len: usize, + phantom: PhantomData<(I, CS, FS)>, +} + +pub struct IdentityCache; + +pub trait Cache { + fn add( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; + + fn add_disabled( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; + + fn replace( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result, Error>; + + fn remove( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result>>, Error>; + + fn get( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result>>, Error>; + + fn get_from_all( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result>>, Error>; +} + +impl FifoCache +where + CS: Store, + FS: Store, + I: Clone, +{ + fn get_inner( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + cache_add_fn: CAF, + cache_get_fn: CGF, + cache_rm_fn: CRF, + fallback_get_fn: FGF, + ) -> Result>>, Error> + where + CAF: FnOnce(&mut CS, CorpusId, Testcase) -> Result<(), Error>, + CGF: FnOnce(&CS, CorpusId) -> Result>>, Error>, + CRF: FnOnce(&mut CS, CorpusId) -> Result>>, Error>, + FGF: FnOnce(&FS, CorpusId) -> Result>>, Error>, + { + if self.cached_ids.contains(&id) { + cache_get_fn(cache_store, id) + } else { + // tescase is not cached, fetch it from fallback + let fb_tc = fallback_get_fn(&fallback_store, id)?; + cache_add_fn(cache_store, id, fb_tc.borrow().clone())?; + + if self.cached_ids.len() == self.cache_max_len { + let to_evict = self.cached_ids.pop_back().unwrap(); + cache_rm_fn(cache_store, to_evict)?; + } + + debug_assert!(self.cached_ids.len() < self.cache_max_len); + + self.cached_ids.push_front(id); + + Ok(fb_tc) + } + } +} + +impl Cache for IdentityCache +where + CS: Store, + FS: Store, + I: Clone, +{ + fn add( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + cache_store.add(id, testcase.clone())?; + fallback_store.add(id, testcase.clone()) + } + + fn add_disabled( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + cache_store.add_disabled(id, testcase.clone())?; + fallback_store.add_disabled(id, testcase.clone()) + } + + fn replace( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result, Error> { + cache_store.replace(id, testcase.clone())?; + fallback_store.replace(id, testcase.clone()) + } + + fn remove( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result>>, Error> { + cache_store.remove(id)?; + fallback_store.remove(id) + } + + fn get( + &mut self, + id: CorpusId, + cache_store: &mut CS, + _fallback_store: &FS, + ) -> Result>>, Error> { + cache_store.get(id) + } + + fn get_from_all( + &mut self, + id: CorpusId, + cache_store: &mut CS, + _fallback_store: &FS, + ) -> Result>>, Error> { + cache_store.get_from_all(id) + } +} + +impl Cache for FifoCache +where + CS: Store, + FS: Store, + I: Clone, +{ + fn get( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result>>, Error> { + self.get_inner( + id, + cache_store, + fallback_store, + |cache_store, corpus_id, testcase| cache_store.add(corpus_id, testcase), + |cache_store, corpus_id| cache_store.get(corpus_id), + |cache_store, corpus_id| cache_store.remove(corpus_id), + |fallback_store, corpus_id| fallback_store.get(corpus_id), + ) + } + + fn get_from_all( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result>>, Error> { + self.get_inner( + id, + cache_store, + fallback_store, + |cache_store, corpus_id, testcase| cache_store.add(corpus_id, testcase), + |cache_store, corpus_id| cache_store.get(corpus_id), + |cache_store, corpus_id| cache_store.remove(corpus_id), + |fallback_store, corpus_id| fallback_store.get_from_all(corpus_id), + ) + } + + fn add( + &mut self, + id: CorpusId, + testcase: Testcase, + _cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + fallback_store.add(id, testcase) + } + + fn add_disabled( + &mut self, + id: CorpusId, + testcase: Testcase, + _cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + fallback_store.add_disabled(id, testcase) + } + + fn replace( + &mut self, + id: CorpusId, + testcase: Testcase, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result, Error> { + if self.cached_ids.contains(&id) { + cache_store.replace(id, testcase.clone())?; + } + + fallback_store.replace(id, testcase) + } + + fn remove( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result>>, Error> { + if self.cached_ids.contains(&id) { + self.cached_ids.retain(|elt| *elt != id); + cache_store.remove(id)?; + } + + fallback_store.remove(id) + } +} + +impl Corpus for CombinedCorpus +where + C: Cache, + CS: Store, + FS: Store, + I: Clone, +{ + fn count(&self) -> usize { + self.fallback_store.count() + } + + fn count_disabled(&self) -> usize { + self.fallback_store.count_disabled() + } + + fn count_all(&self) -> usize { + self.fallback_store.count_all() + } + + fn add(&mut self, testcase: Testcase) -> Result { + let new_id = self.counter.new_id(); + + self.cache.borrow_mut().add( + new_id, + testcase, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store, + )?; + + Ok(new_id) + } + + fn add_disabled(&mut self, testcase: Testcase) -> Result { + let new_id = self.counter.new_id(); + + self.cache.borrow_mut().add_disabled( + new_id, + testcase, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store, + )?; + + Ok(new_id) + } + + fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { + self.cache.borrow_mut().replace( + id, + testcase, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store, + ) + } + + fn remove(&mut self, id: CorpusId) -> Result>>, Error> { + self.cache.borrow_mut().remove( + id, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store, + ) + } + + fn get(&self, id: CorpusId) -> Result>>, Error> { + self.cache.borrow_mut().get( + id, + &mut *self.cache_store.borrow_mut(), + &self.fallback_store, + ) + } + + fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { + self.cache.borrow_mut().get_from_all( + id, + &mut *self.cache_store.borrow_mut(), + &self.fallback_store, + ) + } + + fn current(&self) -> &Option { + &self.current + } + + fn current_mut(&mut self) -> &mut Option { + &mut self.current + } + + fn next(&self, id: CorpusId) -> Option { + self.fallback_store.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.fallback_store.prev(id) + } + + fn first(&self) -> Option { + self.fallback_store.first() + } + + fn last(&self) -> Option { + self.fallback_store.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.fallback_store.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.fallback_store.nth_from_all(nth) + } +} diff --git a/libafl/src/corpus/inmemory.rs b/libafl/src/corpus/inmemory.rs deleted file mode 100644 index d4760386bf6..00000000000 --- a/libafl/src/corpus/inmemory.rs +++ /dev/null @@ -1,479 +0,0 @@ -//! In-memory corpus, keeps all test cases in memory at all times - -use alloc::vec::Vec; -use core::cell::{Ref, RefCell, RefMut}; - -use serde::{Deserialize, Serialize}; - -use super::HasTestcase; -use crate::{ - Error, - corpus::{Corpus, CorpusId, Testcase}, -}; - -/// Keep track of the stored `Testcase` and the siblings ids (insertion order) -#[cfg(not(feature = "corpus_btreemap"))] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestcaseStorageItem { - /// The stored testcase - pub testcase: RefCell>, - /// Previously inserted id - pub prev: Option, - /// Following inserted id - pub next: Option, -} - -/// The map type in which testcases are stored (disable the feature `corpus_btreemap` to use a `HashMap` instead of `BTreeMap`) -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct TestcaseStorageMap { - #[cfg(not(feature = "corpus_btreemap"))] - /// A map of `CorpusId` to `TestcaseStorageItem` - pub map: hashbrown::HashMap>, - #[cfg(feature = "corpus_btreemap")] - /// A map of `CorpusId` to `Testcase`. - pub map: alloc::collections::btree_map::BTreeMap>>, - /// The keys in order (use `Vec::binary_search`) - pub keys: Vec, - /// First inserted id - #[cfg(not(feature = "corpus_btreemap"))] - first_id: Option, - /// Last inserted id - #[cfg(not(feature = "corpus_btreemap"))] - last_id: Option, -} - -impl TestcaseStorageMap { - /// Insert a key in the keys set - fn insert_key(&mut self, id: CorpusId) { - if let Err(idx) = self.keys.binary_search(&id) { - self.keys.insert(idx, id); - } - } - - /// Remove a key from the keys set - fn remove_key(&mut self, id: CorpusId) { - if let Ok(idx) = self.keys.binary_search(&id) { - self.keys.remove(idx); - } - } - - /// Replace a testcase given a `CorpusId` - #[cfg(not(feature = "corpus_btreemap"))] - pub fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Option> { - match self.map.get_mut(&id) { - Some(entry) => Some(entry.testcase.replace(testcase)), - _ => None, - } - } - - /// Replace a testcase given a `CorpusId` - #[cfg(feature = "corpus_btreemap")] - pub fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Option> { - self.map.get_mut(&id).map(|entry| entry.replace(testcase)) - } - - /// Remove a testcase given a [`CorpusId`] - #[cfg(not(feature = "corpus_btreemap"))] - pub fn remove(&mut self, id: CorpusId) -> Option>> { - match self.map.remove(&id) { - Some(item) => { - self.remove_key(id); - match item.prev { - Some(prev) => { - self.map.get_mut(&prev).unwrap().next = item.next; - } - _ => { - // first elem - self.first_id = item.next; - } - } - match item.next { - Some(next) => { - self.map.get_mut(&next).unwrap().prev = item.prev; - } - _ => { - // last elem - self.last_id = item.prev; - } - } - Some(item.testcase) - } - _ => None, - } - } - - /// Remove a testcase given a [`CorpusId`] - #[cfg(feature = "corpus_btreemap")] - pub fn remove(&mut self, id: CorpusId) -> Option>> { - self.remove_key(id); - self.map.remove(&id) - } - - /// Get a testcase given a `CorpusId` - #[cfg(not(feature = "corpus_btreemap"))] - #[must_use] - pub fn get(&self, id: CorpusId) -> Option<&RefCell>> { - self.map.get(&id).as_ref().map(|x| &x.testcase) - } - - /// Get a testcase given a `CorpusId` - #[cfg(feature = "corpus_btreemap")] - #[must_use] - pub fn get(&self, id: CorpusId) -> Option<&RefCell>> { - self.map.get(&id) - } - - /// Get the next id given a `CorpusId` (creation order) - #[cfg(not(feature = "corpus_btreemap"))] - #[must_use] - pub fn next(&self, id: CorpusId) -> Option { - match self.map.get(&id) { - Some(item) => item.next, - _ => None, - } - } - - /// Get the next id given a `CorpusId` (creation order) - #[cfg(feature = "corpus_btreemap")] - #[must_use] - pub fn next(&self, id: CorpusId) -> Option { - // TODO see if using self.keys is faster - let mut range = self - .map - .range((core::ops::Bound::Included(id), core::ops::Bound::Unbounded)); - if let Some((this_id, _)) = range.next() { - if id != *this_id { - return None; - } - } - if let Some((next_id, _)) = range.next() { - Some(*next_id) - } else { - None - } - } - - /// Get the previous id given a `CorpusId` (creation order) - #[cfg(not(feature = "corpus_btreemap"))] - #[must_use] - pub fn prev(&self, id: CorpusId) -> Option { - match self.map.get(&id) { - Some(item) => item.prev, - _ => None, - } - } - - /// Get the previous id given a `CorpusId` (creation order) - #[cfg(feature = "corpus_btreemap")] - #[must_use] - pub fn prev(&self, id: CorpusId) -> Option { - // TODO see if using self.keys is faster - let mut range = self - .map - .range((core::ops::Bound::Unbounded, core::ops::Bound::Included(id))); - if let Some((this_id, _)) = range.next_back() { - if id != *this_id { - return None; - } - } - if let Some((prev_id, _)) = range.next_back() { - Some(*prev_id) - } else { - None - } - } - - /// Get the first created id - #[cfg(not(feature = "corpus_btreemap"))] - #[must_use] - pub fn first(&self) -> Option { - self.first_id - } - - /// Get the first created id - #[cfg(feature = "corpus_btreemap")] - #[must_use] - pub fn first(&self) -> Option { - self.map.iter().next().map(|x| *x.0) - } - - /// Get the last created id - #[cfg(not(feature = "corpus_btreemap"))] - #[must_use] - pub fn last(&self) -> Option { - self.last_id - } - - /// Get the last created id - #[cfg(feature = "corpus_btreemap")] - #[must_use] - pub fn last(&self) -> Option { - self.map.iter().next_back().map(|x| *x.0) - } - - fn new() -> Self { - Self { - #[cfg(not(feature = "corpus_btreemap"))] - map: hashbrown::HashMap::default(), - #[cfg(feature = "corpus_btreemap")] - map: alloc::collections::BTreeMap::default(), - keys: Vec::default(), - #[cfg(not(feature = "corpus_btreemap"))] - first_id: None, - #[cfg(not(feature = "corpus_btreemap"))] - last_id: None, - } - } -} -/// Storage map for the testcases (used in `Corpus` implementations) with an incremental index -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct TestcaseStorage { - /// The map in which enabled testcases are stored - pub enabled: TestcaseStorageMap, - /// The map in which disabled testcases are stored - pub disabled: TestcaseStorageMap, - /// The progressive id for both maps - progressive_id: usize, -} - -impl TestcaseStorage { - /// Insert a testcase assigning a `CorpusId` to it - pub fn insert(&mut self, testcase: RefCell>) -> CorpusId { - self.insert_inner(testcase, false) - } - - #[must_use] - /// Peek the next free corpus id - pub fn peek_free_id(&self) -> CorpusId { - CorpusId::from(self.progressive_id) - } - - /// Insert a testcase assigning a `CorpusId` to it - pub fn insert_disabled(&mut self, testcase: RefCell>) -> CorpusId { - self.insert_inner(testcase, true) - } - - /// Insert a testcase assigning a `CorpusId` to it - #[cfg(not(feature = "corpus_btreemap"))] - fn insert_inner(&mut self, testcase: RefCell>, is_disabled: bool) -> CorpusId { - let id = CorpusId::from(self.progressive_id); - self.progressive_id += 1; - let corpus = if is_disabled { - &mut self.disabled - } else { - &mut self.enabled - }; - let prev = if let Some(last_id) = corpus.last_id { - corpus.map.get_mut(&last_id).unwrap().next = Some(id); - Some(last_id) - } else { - None - }; - if corpus.first_id.is_none() { - corpus.first_id = Some(id); - } - corpus.last_id = Some(id); - corpus.insert_key(id); - corpus.map.insert( - id, - TestcaseStorageItem { - testcase, - prev, - next: None, - }, - ); - id - } - - /// Insert a testcase assigning a `CorpusId` to it - #[cfg(feature = "corpus_btreemap")] - fn insert_inner(&mut self, testcase: RefCell>, is_disabled: bool) -> CorpusId { - let id = CorpusId::from(self.progressive_id); - self.progressive_id += 1; - let corpus = if is_disabled { - &mut self.disabled - } else { - &mut self.enabled - }; - corpus.insert_key(id); - corpus.map.insert(id, testcase); - id - } - - /// Create new `TestcaseStorage` - #[must_use] - pub fn new() -> Self { - Self { - enabled: TestcaseStorageMap::new(), - disabled: TestcaseStorageMap::new(), - progressive_id: 0, - } - } -} - -/// A corpus handling all in memory. -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct InMemoryCorpus { - storage: TestcaseStorage, - current: Option, -} - -impl Corpus for InMemoryCorpus { - /// Returns the number of all enabled entries - #[inline] - fn count(&self) -> usize { - self.storage.enabled.map.len() - } - - /// Returns the number of all disabled entries - fn count_disabled(&self) -> usize { - self.storage.disabled.map.len() - } - - /// Returns the number of elements including disabled entries - #[inline] - fn count_all(&self) -> usize { - self.storage - .enabled - .map - .len() - .saturating_add(self.storage.disabled.map.len()) - } - - /// Add an enabled testcase to the corpus and return its index - #[inline] - fn add(&mut self, testcase: Testcase) -> Result { - Ok(self.storage.insert(RefCell::new(testcase))) - } - - /// Add a disabled testcase to the corpus and return its index - #[inline] - fn add_disabled(&mut self, testcase: Testcase) -> Result { - Ok(self.storage.insert_disabled(RefCell::new(testcase))) - } - - /// Replaces the testcase at the given id - #[inline] - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - self.storage.enabled.replace(id, testcase).ok_or_else(|| { - Error::key_not_found(format!("Index {id} not found, could not replace.")) - }) - } - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - #[inline] - fn remove(&mut self, id: CorpusId) -> Result, Error> { - let mut testcase = self.storage.enabled.remove(id); - if testcase.is_none() { - testcase = self.storage.disabled.remove(id); - } - testcase - .map(|x| x.take()) - .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) - } - - /// Get by id; considers only enabled testcases - #[inline] - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.storage - .enabled - .get(id) - .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) - } - /// Get by id; considers both enabled and disabled testcases - #[inline] - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { - let mut testcase = self.storage.enabled.get(id); - if testcase.is_none() { - testcase = self.storage.disabled.get(id); - } - testcase.ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) - } - - /// Current testcase scheduled - #[inline] - fn current(&self) -> &Option { - &self.current - } - - /// Current testcase scheduled (mutable) - #[inline] - fn current_mut(&mut self) -> &mut Option { - &mut self.current - } - - #[inline] - fn next(&self, id: CorpusId) -> Option { - self.storage.enabled.next(id) - } - - /// Peek the next free corpus id - #[inline] - fn peek_free_id(&self) -> CorpusId { - self.storage.peek_free_id() - } - - #[inline] - fn prev(&self, id: CorpusId) -> Option { - self.storage.enabled.prev(id) - } - - #[inline] - fn first(&self) -> Option { - self.storage.enabled.first() - } - - #[inline] - fn last(&self) -> Option { - self.storage.enabled.last() - } - - /// Get the nth corpus id; considers only enabled testcases - #[inline] - fn nth(&self, nth: usize) -> CorpusId { - self.storage.enabled.keys[nth] - } - - /// Get the nth corpus id; considers both enabled and disabled testcases - #[inline] - fn nth_from_all(&self, nth: usize) -> CorpusId { - let enabled_count = self.count(); - if nth >= enabled_count { - return self.storage.disabled.keys[nth.saturating_sub(enabled_count)]; - } - self.storage.enabled.keys[nth] - } - - #[inline] - fn load_input_into(&self, _: &mut Testcase) -> Result<(), Error> { - // Inputs never get evicted, nothing to load here. - Ok(()) - } - - #[inline] - fn store_input_from(&self, _: &Testcase) -> Result<(), Error> { - Ok(()) - } -} - -impl HasTestcase for InMemoryCorpus { - fn testcase(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow()) - } - - fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow_mut()) - } -} - -impl InMemoryCorpus { - /// Creates a new [`InMemoryCorpus`], keeping all [`Testcase`]`s` in memory. - /// This is the simplest and fastest option, however test progress will be lost on exit or on OOM. - #[must_use] - pub fn new() -> Self { - Self { - storage: TestcaseStorage::new(), - current: None, - } - } -} diff --git a/libafl/src/corpus/inmemory_ondisk.rs b/libafl/src/corpus/inmemory_ondisk.rs deleted file mode 100644 index 70b48bfc081..00000000000 --- a/libafl/src/corpus/inmemory_ondisk.rs +++ /dev/null @@ -1,519 +0,0 @@ -//! The [`InMemoryOnDiskCorpus`] stores [`Testcase`]s to disk. -//! -//! Additionally, _all_ of them are kept in memory. -//! For a lower memory footprint, consider using [`crate::corpus::CachedOnDiskCorpus`] -//! which only stores a certain number of [`Testcase`]s and removes additional ones in a FIFO manner. - -use alloc::string::{String, ToString}; -use core::cell::{Ref, RefCell, RefMut}; -use std::{ - fs, - fs::{File, OpenOptions}, - io, - io::{Read, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, -}; - -use fs2::FileExt; -#[cfg(feature = "gzip")] -use libafl_bolts::compress::GzipCompressor; -use serde::{Deserialize, Serialize}; - -use super::{ - HasTestcase, - ondisk::{OnDiskMetadata, OnDiskMetadataFormat}, -}; -use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase}, - inputs::Input, -}; - -/// Creates the given `path` and returns an error if it fails. -/// If the create succeeds, it will return the file. -/// If the create fails for _any_ reason, including, but not limited to, a preexisting existing file of that name, -/// it will instead return the respective [`io::Error`]. -fn create_new>(path: P) -> Result { - OpenOptions::new() - .write(true) - .read(true) - .create_new(true) - .open(path) -} - -/// Tries to create the given `path` and returns `None` _only_ if the file already existed. -/// If the create succeeds, it will return the file. -/// If the create fails for some other reason, it will instead return the respective [`io::Error`]. -fn try_create_new>(path: P) -> Result, io::Error> { - match create_new(path) { - Ok(ret) => Ok(Some(ret)), - Err(err) if err.kind() == io::ErrorKind::AlreadyExists => Ok(None), - Err(err) => Err(err), - } -} - -/// A corpus able to store [`Testcase`]s to disk, while also keeping all of them in memory. -/// -/// Metadata is written to a `..metadata` file in the same folder by default. -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct InMemoryOnDiskCorpus { - inner: InMemoryCorpus, - dir_path: PathBuf, - meta_format: Option, - prefix: Option, - locking: bool, -} - -impl Corpus for InMemoryOnDiskCorpus -where - I: Input, -{ - /// Returns the number of all enabled entries - #[inline] - fn count(&self) -> usize { - self.inner.count() - } - - /// Returns the number of all disabled entries - fn count_disabled(&self) -> usize { - self.inner.count_disabled() - } - - /// Returns the number of elements including disabled entries - #[inline] - fn count_all(&self) -> usize { - self.inner.count_all() - } - - /// Add an enabled testcase to the corpus and return its index - #[inline] - fn add(&mut self, testcase: Testcase) -> Result { - let id = self.inner.add(testcase)?; - let testcase = &mut self.get(id).unwrap().borrow_mut(); - self.save_testcase(testcase, Some(id))?; - *testcase.input_mut() = None; - Ok(id) - } - - /// Add a disabled testcase to the corpus and return its index - #[inline] - fn add_disabled(&mut self, testcase: Testcase) -> Result { - let id = self.inner.add_disabled(testcase)?; - let testcase = &mut self.get_from_all(id).unwrap().borrow_mut(); - self.save_testcase(testcase, Some(id))?; - *testcase.input_mut() = None; - Ok(id) - } - - /// Replaces the testcase at the given idx - #[inline] - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - let entry = self.inner.replace(id, testcase)?; - self.remove_testcase(&entry)?; - let testcase = &mut self.get(id).unwrap().borrow_mut(); - self.save_testcase(testcase, Some(id))?; - *testcase.input_mut() = None; - Ok(entry) - } - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled corpus - #[inline] - fn remove(&mut self, id: CorpusId) -> Result, Error> { - let entry = self.inner.remove(id)?; - self.remove_testcase(&entry)?; - Ok(entry) - } - - /// Get by id; considers only enabled testcases - #[inline] - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.inner.get(id) - } - - /// Get by id; considers both enabled and disabled testcases - #[inline] - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.inner.get_from_all(id) - } - - /// Current testcase scheduled - #[inline] - fn current(&self) -> &Option { - self.inner.current() - } - - /// Current testcase scheduled (mutable) - #[inline] - fn current_mut(&mut self) -> &mut Option { - self.inner.current_mut() - } - - #[inline] - fn next(&self, id: CorpusId) -> Option { - self.inner.next(id) - } - - /// Peek the next free corpus id - #[inline] - fn peek_free_id(&self) -> CorpusId { - self.inner.peek_free_id() - } - - #[inline] - fn prev(&self, id: CorpusId) -> Option { - self.inner.prev(id) - } - - #[inline] - fn first(&self) -> Option { - self.inner.first() - } - - #[inline] - fn last(&self) -> Option { - self.inner.last() - } - - /// Get the nth corpus id; considers only enabled testcases - #[inline] - fn nth(&self, nth: usize) -> CorpusId { - self.inner.nth(nth) - } - /// Get the nth corpus id; considers both enabled and disabled testcases - #[inline] - fn nth_from_all(&self, nth: usize) -> CorpusId { - self.inner.nth_from_all(nth) - } - - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { - if testcase.input_mut().is_none() { - let Some(file_path) = testcase.file_path().as_ref() else { - return Err(Error::illegal_argument( - "No file path set for testcase. Could not load inputs.", - )); - }; - let input = I::from_file(file_path)?; - testcase.set_input(input); - } - Ok(()) - } - - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { - // Store the input to disk - let Some(file_path) = testcase.file_path() else { - return Err(Error::illegal_argument( - "No file path set for testcase. Could not store input to disk.", - )); - }; - let Some(input) = testcase.input() else { - return Err(Error::illegal_argument( - "No input available for testcase. Could not store anything.", - )); - }; - input.to_file(file_path) - } -} - -impl HasTestcase for InMemoryOnDiskCorpus -where - I: Input, -{ - fn testcase(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow()) - } - - fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow_mut()) - } -} - -impl InMemoryOnDiskCorpus { - /// Creates an [`InMemoryOnDiskCorpus`]. - /// - /// This corpus stores all testcases to disk, and keeps all of them in memory, as well. - /// - /// By default, it stores metadata for each [`Testcase`] as prettified json. - /// Metadata will be written to a file named `..metadata` - /// The metadata may include objective reason, specific information for a fuzz job, and more. - /// - /// If you don't want metadata, use [`InMemoryOnDiskCorpus::no_meta`]. - /// To pick a different metadata format, use [`InMemoryOnDiskCorpus::with_meta_format`]. - /// - /// Will error, if [`fs::create_dir_all()`] failed for `dir_path`. - pub fn new

(dir_path: P) -> Result - where - P: AsRef, - { - Self::_new( - dir_path.as_ref(), - Some(OnDiskMetadataFormat::JsonPretty), - None, - true, - ) - } - - /// Creates the [`InMemoryOnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk. - /// - /// Will error, if [`fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format

( - dir_path: P, - meta_format: Option, - ) -> Result - where - P: AsRef, - { - Self::_new(dir_path.as_ref(), meta_format, None, true) - } - - /// Creates the [`InMemoryOnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk - /// and the prefix for the filenames. - /// - /// Will error, if [`fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format_and_prefix

( - dir_path: P, - meta_format: Option, - prefix: Option, - locking: bool, - ) -> Result - where - P: AsRef, - { - Self::_new(dir_path.as_ref(), meta_format, prefix, locking) - } - - /// Creates an [`InMemoryOnDiskCorpus`] that will not store .metadata files - /// - /// Will error, if [`fs::create_dir_all()`] failed for `dir_path`. - pub fn no_meta

(dir_path: P) -> Result - where - P: AsRef, - { - Self::_new(dir_path.as_ref(), None, None, true) - } - - /// Private fn to crate a new corpus at the given (non-generic) path with the given optional `meta_format` - fn _new( - dir_path: &Path, - meta_format: Option, - prefix: Option, - locking: bool, - ) -> Result { - match fs::create_dir_all(dir_path) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(e.into()), - } - Ok(InMemoryOnDiskCorpus { - inner: InMemoryCorpus::new(), - dir_path: dir_path.into(), - meta_format, - prefix, - locking, - }) - } - - /// Sets the filename for a [`Testcase`]. - /// If an error gets returned from the corpus (i.e., file exists), we'll have to retry with a different filename. - /// Renaming testcases will most likely cause duplicate testcases to not be handled correctly - /// if testcases with the same input are not given the same filename. - /// Only rename when you know what you are doing. - #[inline] - pub fn rename_testcase( - &self, - testcase: &mut Testcase, - filename: String, - id: Option, - ) -> Result<(), Error> - where - I: Input, - { - if testcase.filename().is_some() { - // We are renaming! - - let old_filename = testcase.filename_mut().take().unwrap(); - let new_filename = filename; - - // Do operations below when new filename is specified - if old_filename == new_filename { - *testcase.filename_mut() = Some(old_filename); - return Ok(()); - } - - let new_file_path = self.dir_path.join(&new_filename); - self.remove_testcase(testcase)?; - *testcase.filename_mut() = Some(new_filename); - self.save_testcase(testcase, id)?; - *testcase.file_path_mut() = Some(new_file_path); - - Ok(()) - } else { - Err(Error::illegal_argument( - "Cannot rename testcase without name!", - )) - } - } - - fn save_testcase(&self, testcase: &mut Testcase, id: Option) -> Result<(), Error> - where - I: Input, - { - let file_name = testcase.filename_mut().take().unwrap_or_else(|| { - // TODO walk entry metadata to ask for pieces of filename (e.g. :havoc in AFL) - testcase.input().as_ref().unwrap().generate_name(id) - }); - - let mut ctr = 1; - if self.locking { - let lockfile_name = format!(".{file_name}"); - let lockfile_path = self.dir_path.join(lockfile_name); - - let mut lockfile = try_create_new(&lockfile_path)?.unwrap_or( - OpenOptions::new() - .write(true) - .read(true) - .open(&lockfile_path)?, - ); - lockfile.lock_exclusive()?; - - let mut old_ctr = String::new(); - lockfile.read_to_string(&mut old_ctr)?; - if !old_ctr.is_empty() { - ctr = old_ctr.trim().parse::()? + 1; - } - - lockfile.seek(SeekFrom::Start(0))?; - lockfile.write_all(ctr.to_string().as_bytes())?; - } - - if testcase.file_path().is_none() { - *testcase.file_path_mut() = Some(self.dir_path.join(&file_name)); - } - *testcase.filename_mut() = Some(file_name); - - if self.meta_format.is_some() { - let metafile_name = if self.locking { - format!( - ".{}_{}.metadata", - testcase.filename().as_ref().unwrap(), - ctr - ) - } else { - format!(".{}.metadata", testcase.filename().as_ref().unwrap()) - }; - let metafile_path = self.dir_path.join(&metafile_name); - let mut tmpfile_path = metafile_path.clone(); - tmpfile_path.set_file_name(format!(".{metafile_name}.tmp",)); - - let ondisk_meta = OnDiskMetadata { - metadata: testcase.metadata_map(), - exec_time: testcase.exec_time(), - executions: testcase.executions(), - }; - - let mut tmpfile = File::create(&tmpfile_path)?; - - let json_error = - |err| Error::serialize(format!("Failed to json-ify metadata: {err:?}")); - - let serialized = match self.meta_format.as_ref().unwrap() { - OnDiskMetadataFormat::Postcard => postcard::to_allocvec(&ondisk_meta)?, - OnDiskMetadataFormat::Json => { - serde_json::to_vec(&ondisk_meta).map_err(json_error)? - } - OnDiskMetadataFormat::JsonPretty => { - serde_json::to_vec_pretty(&ondisk_meta).map_err(json_error)? - } - #[cfg(feature = "gzip")] - OnDiskMetadataFormat::JsonGzip => GzipCompressor::new() - .compress(&serde_json::to_vec_pretty(&ondisk_meta).map_err(json_error)?), - }; - tmpfile.write_all(&serialized)?; - fs::rename(&tmpfile_path, &metafile_path)?; - *testcase.metadata_path_mut() = Some(metafile_path); - } - - // Only try to write the data if the counter is 1. - // Otherwise we already have a file with this name, and - // we can assume the data has already been written. - if ctr == 1 { - if let Err(err) = self.store_input_from(testcase) { - if self.locking { - return Err(err); - } - log::error!( - "An error occurred when trying to write a testcase without locking: {err}" - ); - } - } - Ok(()) - } - - fn remove_testcase(&self, testcase: &Testcase) -> Result<(), Error> { - if let Some(filename) = testcase.filename() { - let mut ctr = String::new(); - if self.locking { - let lockfile_path = self.dir_path.join(format!(".{filename}")); - let mut lockfile = OpenOptions::new() - .write(true) - .read(true) - .open(&lockfile_path)?; - - lockfile.lock_exclusive()?; - lockfile.read_to_string(&mut ctr)?; - ctr = ctr.trim().to_string(); - - if ctr == "1" { - FileExt::unlock(&lockfile)?; - drop(fs::remove_file(lockfile_path)); - } else { - lockfile.seek(SeekFrom::Start(0))?; - lockfile.write_all(&(ctr.parse::()? - 1).to_le_bytes())?; - return Ok(()); - } - } - - fs::remove_file(self.dir_path.join(filename))?; - if self.meta_format.is_some() { - if self.locking { - fs::remove_file(self.dir_path.join(format!(".{filename}_{ctr}.metadata")))?; - } else { - fs::remove_file(self.dir_path.join(format!(".{filename}.metadata")))?; - } - } - } - Ok(()) - } - - /// Path to the corpus directory associated with this corpus - #[must_use] - pub fn dir_path(&self) -> &PathBuf { - &self.dir_path - } -} - -#[cfg(test)] -mod tests { - #[cfg(not(miri))] - use std::{env, fs, io::Write}; - - #[cfg(not(miri))] - use super::{create_new, try_create_new}; - - #[test] - #[cfg(not(miri))] - fn test() { - let tmp = env::temp_dir(); - let path = tmp.join("testfile.tmp"); - _ = fs::remove_file(&path); - let mut f = create_new(&path).unwrap(); - f.write_all(&[0; 1]).unwrap(); - - match try_create_new(&path) { - Ok(None) => (), - Ok(_) => panic!("File {path:?} did not exist even though it should have?"), - Err(e) => panic!("An unexpected error occurred: {e}"), - } - drop(f); - fs::remove_file(path).unwrap(); - } -} diff --git a/libafl/src/corpus/mod.rs b/libafl/src/corpus/mod.rs index d7670d1a101..59668d7cd7b 100644 --- a/libafl/src/corpus/mod.rs +++ b/libafl/src/corpus/mod.rs @@ -1,6 +1,7 @@ //! Corpuses contain the testcases, either in memory, on disk, or somewhere else. use core::{cell::RefCell, fmt, marker::PhantomData}; +use std::{rc::Rc, string::String}; use serde::{Deserialize, Serialize}; @@ -9,60 +10,67 @@ use crate::Error; pub mod testcase; pub use testcase::{HasTestcase, SchedulerTestcaseMetadata, Testcase}; -pub mod inmemory; -pub use inmemory::InMemoryCorpus; +pub mod single; +pub use single::SingleCorpus; -#[cfg(feature = "std")] -pub mod inmemory_ondisk; -#[cfg(feature = "std")] -pub use inmemory_ondisk::InMemoryOnDiskCorpus; - -#[cfg(feature = "std")] -pub mod ondisk; -#[cfg(feature = "std")] -pub use ondisk::OnDiskCorpus; - -#[cfg(feature = "std")] -pub mod cached; -#[cfg(feature = "std")] -pub use cached::CachedOnDiskCorpus; +pub mod combined; +pub use combined::{CombinedCorpus, FifoCache, IdentityCache}; #[cfg(all(feature = "cmin", unix))] pub mod minimizer; - -pub mod nop; #[cfg(all(feature = "cmin", unix))] pub use minimizer::*; + +pub mod nop; pub use nop::NopCorpus; +pub mod store; +pub use store::{InMemoryStore, OnDiskStore, maps}; + +#[cfg(not(feature = "corpus_btreemap"))] +pub type InMemoryCorpusMap = maps::HashCorpusMap>; + +#[cfg(feature = "corpus_btreemap")] +pub type InMemoryCorpusMap = maps::BtreeCorpusMap>; + +pub type InMemoryCorpus = SingleCorpus>>; + +#[cfg(feature = "std")] +pub type OnDiskCorpus = SingleCorpus>>; + +pub type InMemoryOnDiskCorpus = CombinedCorpus< + IdentityCache, + InMemoryStore>, + OnDiskStore>, + I, +>; + +pub type CachedOnDiskCorpus = CombinedCorpus< + FifoCache>, OnDiskStore>, I>, + InMemoryStore>, + OnDiskStore>, + I, +>; + /// An abstraction for the index that identify a testcase in the corpus #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[repr(transparent)] pub struct CorpusId(pub usize); -impl fmt::Display for CorpusId { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for CorpusId { - fn from(id: usize) -> Self { - Self(id) - } -} - -impl From for CorpusId { - fn from(id: u64) -> Self { - Self(id as usize) - } +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct CorpusCounter { + /// A fresh, progressive ID + /// It stores the next available ID. + current_id: usize, } -impl From for usize { - /// Not that the `CorpusId` is not necessarily stable in the corpus (if we remove [`Testcase`]s, for example). - fn from(id: CorpusId) -> Self { - id.0 - } +/// [`Iterator`] over the ids of a [`Corpus`] +#[derive(Debug)] +pub struct CorpusIdIterator<'a, C, I> { + corpus: &'a C, + cur: Option, + cur_back: Option, + phantom: PhantomData, } /// Utility macro to call `Corpus::random_id`; fetches only enabled [`Testcase`]`s` @@ -128,13 +136,13 @@ pub trait Corpus: Sized { fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error>; /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - fn remove(&mut self, id: CorpusId) -> Result, Error>; + fn remove(&mut self, id: CorpusId) -> Result>>, Error>; /// Get by id; considers only enabled testcases fn get(&self, id: CorpusId) -> Result<&RefCell>, Error>; /// Get by id; considers both enabled and disabled testcases - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error>; + fn get_from_all(&self, id: CorpusId) -> Result>>, Error>; /// Current testcase scheduled fn current(&self) -> &Option; @@ -145,8 +153,8 @@ pub trait Corpus: Sized { /// Get the next corpus id fn next(&self, id: CorpusId) -> Option; - /// Peek the next free corpus id - fn peek_free_id(&self) -> CorpusId; + // /// Peek the next free corpus id + // fn peek_free_id(&self) -> CorpusId; /// Get the prev corpus id fn prev(&self, id: CorpusId) -> Option; @@ -176,23 +184,6 @@ pub trait Corpus: Sized { /// Get the nth corpus id; considers both enabled and disabled testcases fn nth_from_all(&self, nth: usize) -> CorpusId; - - /// Method to load the input for this [`Testcase`] from persistent storage, - /// if necessary, and if was not already loaded (`== Some(input)`). - /// After this call, `testcase.input()` must always return `Some(input)`. - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error>; - - /// Method to store the input of this `Testcase` to persistent storage, if necessary. - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error>; - - /// Loads the `Input` for a given [`CorpusId`] from the [`Corpus`], and returns the clone. - fn cloned_input_for_id(&self, id: CorpusId) -> Result - where - I: Clone, - { - let mut testcase = self.get(id)?.borrow_mut(); - Ok(testcase.load_input(self)?.clone()) - } } /// Trait for types which track the current corpus index @@ -207,13 +198,29 @@ pub trait HasCurrentCorpusId { fn current_corpus_id(&self) -> Result, Error>; } -/// [`Iterator`] over the ids of a [`Corpus`] -#[derive(Debug)] -pub struct CorpusIdIterator<'a, C, I> { - corpus: &'a C, - cur: Option, - cur_back: Option, - phantom: PhantomData, +impl fmt::Display for CorpusId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for CorpusId { + fn from(id: usize) -> Self { + Self(id) + } +} + +impl From for CorpusId { + fn from(id: u64) -> Self { + Self(id as usize) + } +} + +impl From for usize { + /// Not that the `CorpusId` is not necessarily stable in the corpus (if we remove [`Testcase`]s, for example). + fn from(id: CorpusId) -> Self { + id.0 + } } impl Iterator for CorpusIdIterator<'_, C, I> @@ -245,3 +252,11 @@ where } } } + +impl CorpusCounter { + fn new_id(&mut self) -> CorpusId { + let old = self.current_id; + self.current_id.saturating_add(1); + CorpusId(old) + } +} diff --git a/libafl/src/corpus/nop.rs b/libafl/src/corpus/nop.rs index a6e0f902e4f..768094d804e 100644 --- a/libafl/src/corpus/nop.rs +++ b/libafl/src/corpus/nop.rs @@ -1,5 +1,6 @@ //! The null corpus does not store any [`Testcase`]s. use core::{cell::RefCell, marker::PhantomData}; +use std::rc::Rc; use serde::{Deserialize, Serialize}; @@ -53,19 +54,19 @@ impl Corpus for NopCorpus { /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases #[inline] - fn remove(&mut self, _id: CorpusId) -> Result, Error> { + fn remove(&mut self, _id: CorpusId) -> Result>>, Error> { Err(Error::unsupported("Unsupported by NopCorpus")) } /// Get by id; considers only enabled testcases #[inline] - fn get(&self, _id: CorpusId) -> Result<&RefCell>, Error> { + fn get(&self, _id: CorpusId) -> Result>>, Error> { Err(Error::unsupported("Unsupported by NopCorpus")) } /// Get by id; considers both enabled and disabled testcases #[inline] - fn get_from_all(&self, _id: CorpusId) -> Result<&RefCell>, Error> { + fn get_from_all(&self, _id: CorpusId) -> Result>>, Error> { Err(Error::unsupported("Unsupported by NopCorpus")) } @@ -86,12 +87,6 @@ impl Corpus for NopCorpus { None } - /// Peek the next free corpus id - #[inline] - fn peek_free_id(&self) -> CorpusId { - CorpusId::from(0_usize) - } - #[inline] fn prev(&self, _id: CorpusId) -> Option { None @@ -118,16 +113,6 @@ impl Corpus for NopCorpus { fn nth_from_all(&self, _nth: usize) -> CorpusId { CorpusId::from(0_usize) } - - #[inline] - fn load_input_into(&self, _testcase: &mut Testcase) -> Result<(), Error> { - Err(Error::unsupported("Unsupported by NopCorpus")) - } - - #[inline] - fn store_input_from(&self, _testcase: &Testcase) -> Result<(), Error> { - Err(Error::unsupported("Unsupported by NopCorpus")) - } } impl NopCorpus { diff --git a/libafl/src/corpus/ondisk.rs b/libafl/src/corpus/ondisk.rs deleted file mode 100644 index 72179a65313..00000000000 --- a/libafl/src/corpus/ondisk.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! The [`OnDiskCorpus`] stores all [`Testcase`]s to disk. -//! -//! It _never_ keeps any of them in memory. -//! This is a good solution for solutions that are never reused, or for *very* memory-constraint environments. -//! For any other occasions, consider using [`CachedOnDiskCorpus`] -//! which stores a certain number of [`Testcase`]s in memory and removes additional ones in a FIFO manner. - -use alloc::string::String; -use core::{ - cell::{Ref, RefCell, RefMut}, - time::Duration, -}; -use std::path::{Path, PathBuf}; - -use libafl_bolts::serdeany::SerdeAnyMap; -use serde::{Deserialize, Serialize}; - -use crate::{ - Error, - corpus::{CachedOnDiskCorpus, Corpus, CorpusId, HasTestcase, Testcase}, - inputs::Input, -}; - -/// Options for the the format of the on-disk metadata -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -pub enum OnDiskMetadataFormat { - /// A binary-encoded postcard - Postcard, - /// JSON - Json, - /// JSON formatted for readability - #[default] - JsonPretty, - /// The same as [`OnDiskMetadataFormat::JsonPretty`], but compressed - #[cfg(feature = "gzip")] - JsonGzip, -} - -/// The [`Testcase`] metadata that'll be stored to disk -#[derive(Debug, Serialize)] -pub struct OnDiskMetadata<'a> { - /// The dynamic metadata [`SerdeAnyMap`] stored to disk - pub metadata: &'a SerdeAnyMap, - /// The exec time for this [`Testcase`] - pub exec_time: &'a Option, - /// The executions of this [`Testcase`] - pub executions: &'a u64, -} - -/// A corpus able to store [`Testcase`]s to disk, and load them from disk, when they are being used. -/// -/// Metadata is written to a `..metadata` file in the same folder by default. -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct OnDiskCorpus { - /// The root directory backing this corpus - dir_path: PathBuf, - /// We wrapp a cached corpus and set its size to 1. - inner: CachedOnDiskCorpus, -} - -impl Corpus for OnDiskCorpus -where - I: Input, -{ - /// Returns the number of all enabled entries - #[inline] - fn count(&self) -> usize { - self.inner.count() - } - - /// Returns the number of all disabled entries - fn count_disabled(&self) -> usize { - self.inner.count_disabled() - } - - /// Returns the number of all entries - #[inline] - fn count_all(&self) -> usize { - self.inner.count_all() - } - - /// Add an enabled testcase to the corpus and return its index - #[inline] - fn add(&mut self, testcase: Testcase) -> Result { - self.inner.add(testcase) - } - - /// Add a disabled testcase to the corpus and return its index - #[inline] - fn add_disabled(&mut self, testcase: Testcase) -> Result { - self.inner.add_disabled(testcase) - } - - /// Replaces the testcase at the given idx - #[inline] - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - self.inner.replace(id, testcase) - } - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - #[inline] - fn remove(&mut self, id: CorpusId) -> Result, Error> { - self.inner.remove(id) - } - - /// Get by id; will check the disabled corpus if not available in the enabled - #[inline] - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.inner.get(id) - } - - /// Get by id; considers both enabled and disabled testcases - #[inline] - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { - self.inner.get_from_all(id) - } - - /// Current testcase scheduled - #[inline] - fn current(&self) -> &Option { - self.inner.current() - } - - /// Current testcase scheduled (mutable) - #[inline] - fn current_mut(&mut self) -> &mut Option { - self.inner.current_mut() - } - - #[inline] - fn next(&self, id: CorpusId) -> Option { - self.inner.next(id) - } - - /// Peek the next free corpus id - #[inline] - fn peek_free_id(&self) -> CorpusId { - self.inner.peek_free_id() - } - - #[inline] - fn prev(&self, id: CorpusId) -> Option { - self.inner.prev(id) - } - - #[inline] - fn first(&self) -> Option { - self.inner.first() - } - - #[inline] - fn last(&self) -> Option { - self.inner.last() - } - - /// Get the nth corpus id; considers only enabled testcases - #[inline] - fn nth(&self, nth: usize) -> CorpusId { - self.inner.nth(nth) - } - /// Get the nth corpus id; considers both enabled and disabled testcases - #[inline] - fn nth_from_all(&self, nth: usize) -> CorpusId { - self.inner.nth_from_all(nth) - } - - #[inline] - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { - self.inner.load_input_into(testcase) - } - - #[inline] - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { - self.inner.store_input_from(testcase) - } -} - -impl HasTestcase for OnDiskCorpus -where - I: Input, -{ - fn testcase(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow()) - } - - fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { - Ok(self.get(id)?.borrow_mut()) - } -} - -impl OnDiskCorpus { - /// Creates an [`OnDiskCorpus`]. - /// - /// This corpus stores all testcases to disk. - /// - /// By default, it stores metadata for each [`Testcase`] as prettified json. - /// Metadata will be written to a file named `..metadata` - /// The metadata may include objective reason, specific information for a fuzz job, and more. - /// - /// To pick a different metadata format, use [`OnDiskCorpus::with_meta_format`]. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn new

(dir_path: P) -> Result - where - P: AsRef, - { - Self::with_meta_format_and_prefix( - dir_path.as_ref(), - Some(OnDiskMetadataFormat::JsonPretty), - None, - true, - ) - } - - /// Creates the [`OnDiskCorpus`] with a filename prefix. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn with_prefix

(dir_path: P, prefix: Option) -> Result - where - P: AsRef, - { - Self::with_meta_format_and_prefix( - dir_path.as_ref(), - Some(OnDiskMetadataFormat::JsonPretty), - prefix, - true, - ) - } - - /// Creates the [`OnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format

( - dir_path: P, - meta_format: OnDiskMetadataFormat, - ) -> Result - where - P: AsRef, - { - Self::with_meta_format_and_prefix(dir_path.as_ref(), Some(meta_format), None, true) - } - - /// Creates an [`OnDiskCorpus`] that will not store .metadata files - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn no_meta

(dir_path: P) -> Result - where - P: AsRef, - { - Self::with_meta_format_and_prefix(dir_path.as_ref(), None, None, true) - } - - /// Creates a new corpus at the given (non-generic) path with the given optional `meta_format` - /// and `prefix`. - /// - /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`. - pub fn with_meta_format_and_prefix( - dir_path: &Path, - meta_format: Option, - prefix: Option, - locking: bool, - ) -> Result { - Ok(OnDiskCorpus { - dir_path: dir_path.into(), - inner: CachedOnDiskCorpus::with_meta_format_and_prefix( - dir_path, - 1, - meta_format, - prefix, - locking, - )?, - }) - } - - /// Path to the corpus directory associated with this corpus - pub fn dir_path(&self) -> &PathBuf { - &self.dir_path - } -} diff --git a/libafl/src/corpus/single.rs b/libafl/src/corpus/single.rs new file mode 100644 index 00000000000..1a4ee44917f --- /dev/null +++ b/libafl/src/corpus/single.rs @@ -0,0 +1,114 @@ +use core::{cell::RefCell, marker::PhantomData}; +use std::{rc::Rc, vec::Vec}; + +use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; + +use super::{Corpus, CorpusCounter, CorpusId, Testcase, store::Store}; + +/// You average corpus. +/// It has one backing store, used to store / retrieve testcases. +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct SingleCorpus { + /// The backing testcase store + store: S, + /// The corpus ID counter + counter: CorpusCounter, + /// The keys in order (use `Vec::binary_search`) + keys: Vec, + /// The current ID + current: Option, + phantom: PhantomData, +} + +impl SingleCorpus +where + S: Default, +{ + pub fn new() -> Self { + Self { + store: S::default(), + counter: CorpusCounter::default(), + keys: Vec::new(), + current: None, + phantom: PhantomData, + } + } +} + +impl Corpus for SingleCorpus +where + S: Store, +{ + fn count(&self) -> usize { + self.store.count() + } + + fn count_disabled(&self) -> usize { + self.store.count_disabled() + } + + fn count_all(&self) -> usize { + self.store.count_all() + } + + fn add(&mut self, testcase: Testcase) -> Result { + let new_id = self.counter.new_id(); + self.store.add(new_id, testcase)?; + Ok(new_id) + } + + fn add_disabled(&mut self, testcase: Testcase) -> Result { + let new_id = self.counter.new_id(); + self.store.add_disabled(new_id, testcase)?; + Ok(new_id) + } + + fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { + self.store.replace(id, testcase) + } + + fn remove(&mut self, id: CorpusId) -> Result>>, Error> { + self.store.remove(id) + } + + fn get(&self, id: CorpusId) -> Result>>, Error> { + self.store.get(id) + } + + fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { + self.store.get_from_all(id) + } + + fn current(&self) -> &Option { + &self.current + } + + fn current_mut(&mut self) -> &mut Option { + &mut self.current + } + + fn next(&self, id: CorpusId) -> Option { + self.store.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.store.prev(id) + } + + fn first(&self) -> Option { + self.store.first() + } + + fn last(&self) -> Option { + self.store.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.store.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.store.nth_from_all(nth) + } +} diff --git a/libafl/src/corpus/store/inmemory.rs b/libafl/src/corpus/store/inmemory.rs new file mode 100644 index 00000000000..cc6af6ac76f --- /dev/null +++ b/libafl/src/corpus/store/inmemory.rs @@ -0,0 +1,104 @@ +use core::{cell::RefCell, marker::PhantomData}; +use std::rc::Rc; + +use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; + +use super::{InMemoryCorpusMap, Store}; +use crate::corpus::{CorpusId, Testcase}; + +/// The map type in which testcases are stored (disable the feature `corpus_btreemap` to use a `HashMap` instead of `BTreeMap`) +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct InMemoryStore { + enabled_map: M, + disabled_map: M, + phantom: PhantomData, +} + +impl Store for InMemoryStore +where + M: InMemoryCorpusMap>, +{ + fn count(&self) -> usize { + self.enabled_map.count() + } + + fn count_disabled(&self) -> usize { + self.disabled_map.count() + } + + fn is_empty(&self) -> bool { + self.enabled_map.is_empty() + } + + fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { + Ok(self.enabled_map.add(id, testcase)) + } + + fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { + Ok(self.disabled_map.add(id, testcase)) + } + + fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error> { + self.enabled_map.replace(id, new_testcase).ok_or_else(|| { + Error::key_not_found(format!("Index {id} not found, could not replace.")) + }) + } + + fn remove(&mut self, id: CorpusId) -> Result>>, Error> { + let mut testcase = self.enabled_map.remove(id); + + if testcase.is_none() { + testcase = self.disabled_map.remove(id); + } + + testcase + .map(|x| x.clone()) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) + } + + fn get(&self, id: CorpusId) -> Result>>, Error> { + self.enabled_map + .get(id) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) + } + + fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { + let mut testcase = self.enabled_map.get(id); + + if testcase.is_none() { + testcase = self.disabled_map.get(id); + } + + testcase.ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) + } + + fn prev(&self, id: CorpusId) -> Option { + self.enabled_map.prev(id) + } + + fn next(&self, id: CorpusId) -> Option { + self.enabled_map.next(id) + } + + fn first(&self) -> Option { + self.enabled_map.first() + } + + fn last(&self) -> Option { + self.enabled_map.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.enabled_map.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + let nb_enabled = self.enabled_map.count(); + if nth >= nb_enabled { + self.disabled_map.nth(nth.saturating_sub(nb_enabled)) + } else { + self.enabled_map.nth(nth) + } + } +} diff --git a/libafl/src/corpus/store/maps.rs b/libafl/src/corpus/store/maps.rs new file mode 100644 index 00000000000..86b2213d9a7 --- /dev/null +++ b/libafl/src/corpus/store/maps.rs @@ -0,0 +1,276 @@ +//! Multiple map implementations for the in-memory store. + +use core::cell::RefCell; +use std::{collections::BTreeMap, rc::Rc, vec::Vec}; + +use num_traits::Zero; +use serde::{Deserialize, Serialize}; + +use crate::corpus::CorpusId; + +pub trait InMemoryCorpusMap { + /// Returns the number of testcases + fn count(&self) -> usize; + + /// Returns true, if no elements are in this corpus yet + fn is_empty(&self) -> bool { + self.count().is_zero() + } + + /// Store the testcase associated to corpus_id. + fn add(&mut self, id: CorpusId, testcase: T); + + /// Replaces the [`Testcase`] at the given idx, returning the existing. + fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option; + + /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases + fn remove(&mut self, id: CorpusId) -> Option>>; + + /// Get by id; considers only enabled testcases + fn get(&self, id: CorpusId) -> Option>>; + + /// Get the prev corpus id in chronological order + fn prev(&self, id: CorpusId) -> Option; + + /// Get the next corpus id in chronological order + fn next(&self, id: CorpusId) -> Option; + + /// Get the first inserted corpus id + fn first(&self) -> Option; + + /// Get the last inserted corpus id + fn last(&self) -> Option; + + /// Get the nth inserted item + fn nth(&self, nth: usize) -> CorpusId; +} + +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct CorpusIdHistory { + keys: Vec, +} + +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct BtreeCorpusMap { + /// A map of `CorpusId` to `Testcase`. + map: BTreeMap>>, + /// A list of available corpus ids + history: CorpusIdHistory, +} + +/// Keep track of the stored `Testcase` and the siblings ids (insertion order) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestcaseStorageItem { + /// The stored testcase + pub testcase: Rc>, + /// Previously inserted id + pub prev: Option, + /// Following inserted id + pub next: Option, +} + +#[derive(Default, Serialize, Deserialize, Clone, Debug)] +pub struct HashCorpusMap { + /// A map of `CorpusId` to `TestcaseStorageItem` + map: hashbrown::HashMap>, + /// First inserted id + first_id: Option, + /// Last inserted id + last_id: Option, + /// A list of available corpus ids + history: CorpusIdHistory, +} + +impl CorpusIdHistory { + /// Add a key to the history + pub fn add(&mut self, id: CorpusId) { + if let Err(idx) = self.keys.binary_search(&id) { + self.keys.insert(idx, id); + } + } + + /// Remove a key from the history + fn remove(&mut self, id: CorpusId) { + if let Ok(idx) = self.keys.binary_search(&id) { + self.keys.remove(idx); + } + } + + // Get the nth item from the map + fn nth(&self, idx: usize) -> CorpusId { + self.keys[idx] + } +} + +impl InMemoryCorpusMap for HashCorpusMap { + fn count(&self) -> usize { + self.map.len() + } + + fn is_empty(&self) -> bool { + self.map.is_empty() + } + + fn add(&mut self, id: CorpusId, testcase: T) { + let prev = if let Some(last_id) = self.last_id { + self.map.get_mut(&last_id).unwrap().next = Some(id); + Some(last_id) + } else { + None + }; + + if self.first_id.is_none() { + self.first_id = Some(id); + } + + self.last_id = Some(id); + + self.history.add(id); + + self.map.insert( + id, + TestcaseStorageItem { + testcase: Rc::new(RefCell::new(testcase)), + prev, + next: None, + }, + ); + } + + fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { + match self.map.get_mut(&id) { + Some(entry) => Some(entry.testcase.replace(new_testcase)), + _ => None, + } + } + + fn remove(&mut self, id: CorpusId) -> Option>> { + if let Some(item) = self.map.remove(&id) { + if let Some(prev) = item.prev { + self.history.remove(id); + self.map.get_mut(&prev).unwrap().next = item.next; + } else { + // first elem + self.first_id = item.next; + } + + if let Some(next) = item.next { + self.map.get_mut(&next).unwrap().prev = item.prev; + } else { + // last elem + self.last_id = item.prev; + } + + Some(item.testcase) + } else { + None + } + } + + fn get(&self, id: CorpusId) -> Option>> { + self.map.get(&id).map(|inner| inner.testcase.clone()) + } + + fn prev(&self, id: CorpusId) -> Option { + match self.map.get(&id) { + Some(item) => item.prev, + _ => None, + } + } + + fn next(&self, id: CorpusId) -> Option { + match self.map.get(&id) { + Some(item) => item.next, + _ => None, + } + } + + fn first(&self) -> Option { + self.first_id + } + + fn last(&self) -> Option { + self.last_id + } + + fn nth(&self, nth: usize) -> CorpusId { + self.history.nth(nth) + } +} + +impl InMemoryCorpusMap for BtreeCorpusMap { + fn count(&self) -> usize { + self.map.len() + } + + fn is_empty(&self) -> bool { + self.map.is_empty() + } + + fn add(&mut self, id: CorpusId, testcase: T) { + // corpus.insert_key(id); + self.map.insert(id, Rc::new(RefCell::new(testcase))); + self.history.add(id); + } + + fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { + self.map + .get_mut(&id) + .map(|entry| entry.replace(new_testcase)) + } + + fn remove(&mut self, id: CorpusId) -> Option>> { + self.history.remove(id); + self.map.remove(&id) + } + + fn get(&self, id: CorpusId) -> Option>> { + self.map.get(&id).cloned() + } + + fn prev(&self, id: CorpusId) -> Option { + // TODO see if using self.keys is faster + let mut range = self + .map + .range((core::ops::Bound::Unbounded, core::ops::Bound::Included(id))); + if let Some((this_id, _)) = range.next_back() { + if id != *this_id { + return None; + } + } + if let Some((prev_id, _)) = range.next_back() { + Some(*prev_id) + } else { + None + } + } + + fn next(&self, id: CorpusId) -> Option { + // TODO see if using self.keys is faster + let mut range = self + .map + .range((core::ops::Bound::Included(id), core::ops::Bound::Unbounded)); + if let Some((this_id, _)) = range.next() { + if id != *this_id { + return None; + } + } + if let Some((next_id, _)) = range.next() { + Some(*next_id) + } else { + None + } + } + + fn first(&self) -> Option { + self.map.iter().next().map(|x| *x.0) + } + + fn last(&self) -> Option { + self.map.iter().next_back().map(|x| *x.0) + } + + fn nth(&self, nth: usize) -> CorpusId { + self.history.nth(nth) + } +} diff --git a/libafl/src/corpus/store/mod.rs b/libafl/src/corpus/store/mod.rs new file mode 100644 index 00000000000..ffb6239f899 --- /dev/null +++ b/libafl/src/corpus/store/mod.rs @@ -0,0 +1,72 @@ +//! Stores are collections managing testcases + +use core::cell::RefCell; +use std::rc::Rc; + +use libafl_bolts::Error; + +use super::{CorpusId, Testcase}; + +pub mod maps; +pub use maps::{BtreeCorpusMap, HashCorpusMap, InMemoryCorpusMap}; + +pub mod inmemory; +pub use inmemory::InMemoryStore; + +pub mod ondisk; +pub use ondisk::OnDiskStore; + +/// A store is responsible for storing and retrieving [`Testcase`]s, ordered by add time. +pub trait Store { + /// Returns the number of all enabled entries + fn count(&self) -> usize; + + /// Returns the number of all disabled entries + fn count_disabled(&self) -> usize; + + /// Returns the number of elements including disabled entries + fn count_all(&self) -> usize { + self.count().saturating_add(self.count_disabled()) + } + + /// Returns true, if no elements are in this corpus yet + fn is_empty(&self) -> bool { + self.count() == 0 + } + + /// Store the testcase associated to corpus_id to the enabled set. + fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error>; + + /// Store the testcase associated to corpus_id to the disabled set. + fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error>; + + /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. + fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error>; + + /// Removes an entry from the corpus, returning it; considers both enabled and disabled testcases + fn remove(&mut self, id: CorpusId) -> Result>>, Error>; + + /// Get by id; considers only enabled testcases + fn get(&self, id: CorpusId) -> Result>>, Error>; + + /// Get by id; considers both enabled and disabled testcases + fn get_from_all(&self, id: CorpusId) -> Result>>, Error>; + + /// Get the prev corpus id in chronological order + fn prev(&self, id: CorpusId) -> Option; + + /// Get the next corpus id in chronological order + fn next(&self, id: CorpusId) -> Option; + + /// Get the first inserted corpus id + fn first(&self) -> Option; + + /// Get the last inserted corpus id + fn last(&self) -> Option; + + /// Get the nth corpus id; considers only enabled testcases + fn nth(&self, nth: usize) -> CorpusId; + + /// Get the nth corpus id; considers both enabled and disabled testcases + fn nth_from_all(&self, nth: usize) -> CorpusId; +} diff --git a/libafl/src/corpus/store/ondisk.rs b/libafl/src/corpus/store/ondisk.rs new file mode 100644 index 00000000000..ee54adde908 --- /dev/null +++ b/libafl/src/corpus/store/ondisk.rs @@ -0,0 +1,321 @@ +use core::{cell::RefCell, marker::PhantomData}; +use std::{ + fs::{self, File}, + io::{self, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + rc::Rc, + string::String, + vec::Vec, +}; + +use fs4::fs_std::FileExt; +use libafl_bolts::{Error, compress::GzipCompressor}; +use serde::{Deserialize, Serialize}; + +use super::{InMemoryCorpusMap, Store}; +use crate::{ + corpus::{CorpusId, Testcase, testcase::TestcaseMetadata}, + inputs::Input, +}; + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct OnDiskStore { + disk_mgr: DiskMgr, + enabled_map: M, + disabled_map: M, + first: Option, + last: Option, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct DiskMgr { + root_dir: PathBuf, + md_format: OnDiskMetadataFormat, + phantom: PhantomData, +} + +/// Options for the the format of the on-disk metadata +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub enum OnDiskMetadataFormat { + /// A binary-encoded postcard + Postcard, + /// JSON + Json, + /// JSON formatted for readability + #[default] + JsonPretty, + /// The same as [`OnDiskMetadataFormat::JsonPretty`], but compressed + #[cfg(feature = "gzip")] + JsonGzip, +} + +impl OnDiskMetadataFormat { + pub fn to_vec(&self, testcase_md: &TestcaseMetadata) -> Result, Error> { + let json_error = |err| Error::serialize(format!("Failed to json-ify metadata: {err:?}")); + + Ok(match self { + OnDiskMetadataFormat::Postcard => postcard::to_allocvec(testcase_md)?, + OnDiskMetadataFormat::Json => serde_json::to_vec(&testcase_md).map_err(json_error)?, + OnDiskMetadataFormat::JsonPretty => { + serde_json::to_vec_pretty(&testcase_md).map_err(json_error)? + } + #[cfg(feature = "gzip")] + OnDiskMetadataFormat::JsonGzip => GzipCompressor::new() + .compress(&serde_json::to_vec_pretty(&testcase_md).map_err(json_error)?), + }) + } + + pub fn from_file(&self, md_path: &Path) -> Result { + let json_error = |err| Error::serialize(format!("Failed to parse metadata: {err:?}")); + let md_serialized = fs::read(md_path)?; + + Ok(match self { + OnDiskMetadataFormat::Postcard => postcard::from_bytes(&md_serialized)?, + OnDiskMetadataFormat::Json => { + serde_json::from_slice(&md_serialized).map_err(json_error)? + } + OnDiskMetadataFormat::JsonPretty => { + serde_json::from_slice(&md_serialized).map_err(json_error)? + } + #[cfg(feature = "gzip")] + OnDiskMetadataFormat::JsonGzip => { + serde_json::from_slice(&GzipCompressor::new().decompress(&md_serialized)?) + .map_err(json_error)? + } + }) + } +} + +#[derive(Debug)] +struct TestcaseLockfile { + lockfile: File, + nb_used: u32, +} + +impl TestcaseLockfile { + pub fn new(ondisk_mgr: &DiskMgr, testcase_id: &String) -> Result { + let lockfile_path = ondisk_mgr.root_dir.join(format!(".{}.lock", testcase_id)); + + let mut lockfile = match File::create_new(lockfile_path.as_path()) { + Ok(f) => f, + Err(e) => match e.kind() { + io::ErrorKind::AlreadyExists => File::open(lockfile_path.as_path()).unwrap(), + + _ => return Err(e.into()), + }, + }; + + lockfile.lock_exclusive()?; + + let mut nb_used_buf: [u8; 4] = [0; 4]; + let nb_used: u32 = match lockfile.read_exact(&mut nb_used_buf) { + Ok(()) => u32::from_le_bytes(nb_used_buf), + Err(e) => match e.kind() { + io::ErrorKind::UnexpectedEof => 0, + + _ => return Err(e.into()), + }, + }; + + Ok(Self { lockfile, nb_used }) + } + + /// returns true if it is the first use + pub fn inc_used(&mut self) -> bool { + self.nb_used += 1; + self.nb_used == 1 + } + + /// returns true if not in used anymore + /// can be safely deleted + pub fn dec_used(&mut self) -> bool { + if self.nb_used == 0 { + true + } else { + self.nb_used -= 1; + self.nb_used == 0 + } + } +} + +impl Drop for TestcaseLockfile { + fn drop(&mut self) { + let nb_used_buf = self.nb_used.to_le_bytes(); + + self.lockfile.seek(SeekFrom::Start(0)); + self.lockfile.write_all(&nb_used_buf).unwrap(); + + FileExt::unlock(&self.lockfile); + } +} + +impl DiskMgr +where + I: Input, +{ + fn testcase_path(&self, testcase_id: &String) -> PathBuf { + self.root_dir.join(&testcase_id) + } + + fn testcase_md_path(&self, testcase_id: &String) -> PathBuf { + self.root_dir.join(format!(".{}.metadata", testcase_id)) + } + + fn save_testcase(&self, testcase: &Testcase) -> Result { + let testcase_id = testcase.id(); + let testcase_path = self.testcase_path(testcase_id); + let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; + + if lockfile.inc_used() { + // save md to file + let ser_fmt = self.md_format.clone(); + let testcase_md_path = self.testcase_md_path(testcase_id); + + let mut testcase_md_f = File::create_new(testcase_md_path.as_path()).unwrap(); + let testcase_md_ser = ser_fmt.to_vec(testcase.metadata())?; + + testcase_md_f.write_all(&testcase_md_ser)?; + + // testcase_f.write_all(testcase.input().target_bytes().as_ref())?; + testcase.input().to_file(testcase_path.as_path())?; + } + + Ok(testcase_id.clone()) + } + + /// prerequisite: the testcase should not have been "removed" before. + /// also, it should only happen if it has been saved before. + fn load_testcase(&self, testcase_id: &String) -> Result, Error> { + let testcase_path = self.testcase_path(testcase_id); + let testcase_md_path = self.testcase_md_path(testcase_id); + let ser_fmt = self.md_format.clone(); + + let _lockfile = TestcaseLockfile::new(self, testcase_id)?; + + let input = I::from_file(testcase_path.as_path())?; + let md = ser_fmt.from_file(testcase_md_path.as_path())?; + + Ok(Testcase::new(input, md)) + } + + fn remove_testcase(&self, testcase_id: &String) -> Result<(), Error> { + let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; + + if lockfile.dec_used() { + fs::remove_file(self.testcase_path(testcase_id))?; + fs::remove_file(self.testcase_md_path(testcase_id))?; + } + + Ok(()) + } +} + +impl Store for OnDiskStore +where + I: Input, + M: InMemoryCorpusMap, +{ + fn count_all(&self) -> usize { + self.count().saturating_add(self.count_disabled()) + } + + fn is_empty(&self) -> bool { + self.count() == 0 + } + + fn count(&self) -> usize { + self.enabled_map.count() + } + + fn count_disabled(&self) -> usize { + self.disabled_map.count() + } + + fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { + let testcase_id = self.disk_mgr.save_testcase(&testcase)?; + self.enabled_map.add(id, testcase_id); + Ok(()) + } + + fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { + let testcase_id = self.disk_mgr.save_testcase(&testcase)?; + self.disabled_map.add(id, testcase_id); + Ok(()) + } + + fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error> { + let new_tc_id = self.disk_mgr.save_testcase(&new_testcase)?; + let old_tc_id = self.enabled_map.replace(id, new_tc_id).ok_or_else(|| { + Error::key_not_found(format!("Index {id} not found, could not replace.")) + })?; + + let old_tc = self.disk_mgr.load_testcase(&old_tc_id)?; + self.disk_mgr.remove_testcase(&old_tc_id)?; + Ok(old_tc) + } + + fn remove(&mut self, id: CorpusId) -> Result>>, Error> { + let old_tc_id = self + .enabled_map + .remove(id) + .or_else(|| self.disabled_map.remove(id)) + .ok_or(Error::key_not_found(format!("Index {id} not found")))?; + + let old_tc_id_borrowed = old_tc_id.borrow(); + let old_tc = self.disk_mgr.load_testcase(&old_tc_id_borrowed)?; + self.disk_mgr.remove_testcase(&old_tc_id_borrowed)?; + Ok(Rc::new(RefCell::new(old_tc))) + } + + fn get(&self, id: CorpusId) -> Result>>, Error> { + let tc_id = self + .enabled_map + .get(id) + .ok_or(Error::key_not_found(format!("Index not found: {id}")))?; + + let tc_id_borrowed = tc_id.borrow(); + let tc = self.disk_mgr.load_testcase(&*tc_id_borrowed)?; + Ok(Rc::new(RefCell::new(tc))) + } + + fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { + let tc_id = self + .enabled_map + .get(id) + .or_else(|| self.disabled_map.get(id)) + .ok_or(Error::key_not_found(format!("Index {id} not found")))?; + + let tc_id_borrowed = tc_id.borrow(); + let tc = self.disk_mgr.load_testcase(&*&tc_id_borrowed)?; + Ok(Rc::new(RefCell::new(tc))) + } + + fn prev(&self, id: CorpusId) -> Option { + self.enabled_map.prev(id) + } + + fn next(&self, id: CorpusId) -> Option { + self.enabled_map.next(id) + } + + fn first(&self) -> Option { + self.enabled_map.first() + } + + fn last(&self) -> Option { + self.enabled_map.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.enabled_map.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + let nb_enabled = self.enabled_map.count(); + if nth >= nb_enabled { + self.disabled_map.nth(nth.saturating_sub(nb_enabled)) + } else { + self.enabled_map.nth(nth) + } + } +} diff --git a/libafl/src/corpus/testcase.rs b/libafl/src/corpus/testcase.rs index 3e2763e661d..54b53367713 100644 --- a/libafl/src/corpus/testcase.rs +++ b/libafl/src/corpus/testcase.rs @@ -6,16 +6,15 @@ use alloc::string::String; use alloc::{borrow::Cow, vec::Vec}; use core::{ cell::{Ref, RefMut}, + hash::Hasher, time::Duration, }; -#[cfg(feature = "std")] -use std::path::PathBuf; -use libafl_bolts::{HasLen, serdeany::SerdeAnyMap}; +use libafl_bolts::{hasher_std, serdeany::SerdeAnyMap}; use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; -use super::Corpus; -use crate::{Error, HasMetadata, corpus::CorpusId}; +use crate::{Error, HasMetadata, corpus::CorpusId, inputs::Input}; /// Shorthand to receive a [`Ref`] or [`RefMut`] to a stored [`Testcase`], by [`CorpusId`]. /// For a normal state, this should return a [`Testcase`] in the corpus, not the objectives. @@ -29,34 +28,27 @@ pub trait HasTestcase { fn testcase_mut(&self, id: CorpusId) -> Result>, Error>; } -/// An entry in the [`Testcase`] Corpus -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct Testcase { - /// The [`Input`] of this [`Testcase`], or `None`, if it is not currently in memory - input: Option, - /// The filename for this [`Testcase`] - filename: Option, - /// Complete path to the [`Input`] on disk, if this [`Testcase`] is backed by a file in the filesystem - #[cfg(feature = "std")] - file_path: Option, +#[derive(Serialize, Deserialize, Clone, Debug, TypedBuilder)] +pub struct TestcaseMetadata { /// Map of metadata associated with this [`Testcase`] + #[builder(default)] metadata: SerdeAnyMap, - /// Complete path to the metadata [`SerdeAnyMap`] on disk, if this [`Testcase`] is backed by a file in the filesystem - #[cfg(feature = "std")] - metadata_path: Option, /// Time needed to execute the input + #[builder(default)] exec_time: Option, - /// Cached len of the input, if any - cached_len: Option, /// Number of fuzzing iterations of this particular input updated in `perform_mutational` + #[builder(default = 0)] scheduled_count: usize, /// Number of executions done at discovery time executions: u64, /// Parent [`CorpusId`], if known + #[builder(default)] parent_id: Option, /// If the testcase is "disabled" + #[builder(default = false)] disabled: bool, /// has found crash (or timeout) or not + #[builder(default = 0)] objectives_found: usize, /// Vector of `Feedback` names that deemed this `Testcase` as corpus worthy #[cfg(feature = "track_hit_feedbacks")] @@ -66,37 +58,77 @@ pub struct Testcase { hit_objectives: Vec>, } +/// An entry in the [`Testcase`] Corpus +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Testcase { + /// The [`Input`] of this [`Testcase`], or `None`, if it is not currently in memory + input: I, + /// The unique id for [`Testcase`]. + /// It should uniquely identify input at the very least. + id: String, + /// The metadata linked to the [`Testcase`] + metadata: TestcaseMetadata, + // /// The filename for this [`Testcase`] + // filename: Option, +} + impl HasMetadata for Testcase { /// Get all the metadata into an [`hashbrown::HashMap`] #[inline] fn metadata_map(&self) -> &SerdeAnyMap { - &self.metadata + &self.metadata.metadata } /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) #[inline] fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { - &mut self.metadata + &mut self.metadata.metadata + } +} + +impl Testcase +where + I: Input, +{ + fn gen_id(input: &I, _md: &TestcaseMetadata) -> String { + let mut hasher = hasher_std(); + input.hash(&mut hasher); + let hash = hasher.finish(); + format!("{hash:0>8x}") + } + + /// Create a new Testcase instance given an input + #[inline] + pub fn new(input: I, metadata: TestcaseMetadata) -> Self { + let id = Self::gen_id(&input, &metadata); + + Self { + input, + id, + metadata, + } } } /// Impl of a testcase impl Testcase { - /// Returns this [`Testcase`] with a loaded `Input`] - pub fn load_input>(&mut self, corpus: &C) -> Result<&I, Error> { - corpus.load_input_into(self)?; - Ok(self.input.as_ref().unwrap()) + pub fn id(&self) -> &String { + &self.id + } + + pub fn metadata(&self) -> &TestcaseMetadata { + &self.metadata } /// Get the input, if available any #[inline] - pub fn input(&self) -> &Option { + pub fn input(&self) -> &I { &self.input } /// Get the input, if any (mutable) #[inline] - pub fn input_mut(&mut self) -> &mut Option { + pub fn input_mut(&mut self) -> &mut I { // self.cached_len = None; &mut self.input } @@ -104,303 +136,146 @@ impl Testcase { /// Set the input #[inline] pub fn set_input(&mut self, input: I) { - self.input = Some(input); - } - - /// Get the filename, if any - #[inline] - pub fn filename(&self) -> &Option { - &self.filename - } - - /// Get the filename, if any (mutable) - #[inline] - pub fn filename_mut(&mut self) -> &mut Option { - &mut self.filename - } - - /// Get the filename path, if any - #[inline] - #[cfg(feature = "std")] - pub fn file_path(&self) -> &Option { - &self.file_path - } - - /// Get the filename path, if any (mutable) - #[inline] - #[cfg(feature = "std")] - pub fn file_path_mut(&mut self) -> &mut Option { - &mut self.file_path - } - - /// Get the metadata path, if any - #[inline] - #[cfg(feature = "std")] - pub fn metadata_path(&self) -> &Option { - &self.metadata_path - } - - /// Get the metadata path, if any (mutable) - #[inline] - #[cfg(feature = "std")] - pub fn metadata_path_mut(&mut self) -> &mut Option { - &mut self.metadata_path + self.input = input; } /// Get the executions #[inline] - pub fn executions(&self) -> &u64 { - &self.executions + pub fn executions(&self) -> u64 { + self.metadata.executions } /// Get the executions (mutable) #[inline] pub fn executions_mut(&mut self) -> &mut u64 { - &mut self.executions + &mut self.metadata.executions } /// Set the executions #[inline] pub fn set_executions(&mut self, executions: u64) { - self.executions = executions; + self.metadata.executions = executions; } /// Get the execution time of the testcase #[inline] pub fn exec_time(&self) -> &Option { - &self.exec_time + &self.metadata.exec_time } /// Get the execution time of the testcase (mutable) #[inline] pub fn exec_time_mut(&mut self) -> &mut Option { - &mut self.exec_time + &mut self.metadata.exec_time } /// Sets the execution time of the current testcase #[inline] pub fn set_exec_time(&mut self, time: Duration) { - self.exec_time = Some(time); + self.metadata.exec_time = Some(time); } /// Get the `scheduled_count` #[inline] pub fn scheduled_count(&self) -> usize { - self.scheduled_count + self.metadata.scheduled_count } /// Set the `scheduled_count` #[inline] pub fn set_scheduled_count(&mut self, scheduled_count: usize) { - self.scheduled_count = scheduled_count; + self.metadata.scheduled_count = scheduled_count; } /// Get `disabled` #[inline] pub fn disabled(&mut self) -> bool { - self.disabled + self.metadata.disabled } /// Set the testcase as disabled #[inline] pub fn set_disabled(&mut self, disabled: bool) { - self.disabled = disabled; + self.metadata.disabled = disabled; } /// Get the hit feedbacks #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_feedbacks(&self) -> &Vec> { - &self.hit_feedbacks + &self.metadata.hit_feedbacks } /// Get the hit feedbacks (mutable) #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_feedbacks_mut(&mut self) -> &mut Vec> { - &mut self.hit_feedbacks + &mut self.metadata.hit_feedbacks } /// Get the hit objectives #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_objectives(&self) -> &Vec> { - &self.hit_objectives + &self.metadata.hit_objectives } /// Get the hit objectives (mutable) #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_objectives_mut(&mut self) -> &mut Vec> { - &mut self.hit_objectives - } - - /// Create a new Testcase instance given an input - #[inline] - pub fn new(input: I) -> Self { - Self { - input: Some(input), - filename: None, - #[cfg(feature = "std")] - file_path: None, - metadata: SerdeAnyMap::default(), - #[cfg(feature = "std")] - metadata_path: None, - exec_time: None, - cached_len: None, - executions: 0, - scheduled_count: 0, - parent_id: None, - disabled: false, - objectives_found: 0, - #[cfg(feature = "track_hit_feedbacks")] - hit_feedbacks: Vec::new(), - #[cfg(feature = "track_hit_feedbacks")] - hit_objectives: Vec::new(), - } - } - - /// Creates a testcase, attaching the id of the parent - /// that this [`Testcase`] was derived from on creation - pub fn with_parent_id(input: I, parent_id: CorpusId) -> Self { - Testcase { - input: Some(input), - filename: None, - #[cfg(feature = "std")] - file_path: None, - metadata: SerdeAnyMap::default(), - #[cfg(feature = "std")] - metadata_path: None, - exec_time: None, - cached_len: None, - executions: 0, - scheduled_count: 0, - parent_id: Some(parent_id), - disabled: false, - objectives_found: 0, - #[cfg(feature = "track_hit_feedbacks")] - hit_feedbacks: Vec::new(), - #[cfg(feature = "track_hit_feedbacks")] - hit_objectives: Vec::new(), - } - } - - /// Create a new Testcase instance given an input and a `filename` - /// If locking is enabled, make sure that testcases with the same input have the same filename - /// to prevent ending up with duplicate testcases - #[inline] - pub fn with_filename(input: I, filename: String) -> Self { - Self { - input: Some(input), - filename: Some(filename), - #[cfg(feature = "std")] - file_path: None, - metadata: SerdeAnyMap::default(), - #[cfg(feature = "std")] - metadata_path: None, - exec_time: None, - cached_len: None, - executions: 0, - scheduled_count: 0, - parent_id: None, - disabled: false, - objectives_found: 0, - #[cfg(feature = "track_hit_feedbacks")] - hit_feedbacks: Vec::new(), - #[cfg(feature = "track_hit_feedbacks")] - hit_objectives: Vec::new(), - } - } + &mut self.metadata.hit_objectives + } + + // /// Creates a testcase, attaching the id of the parent + // /// that this [`Testcase`] was derived from on creation + // pub fn with_parent_id(input: I, parent_id: CorpusId) -> Self { + // Testcase { + // input: Some(input), + // filename: None, + // #[cfg(feature = "std")] + // file_path: None, + // metadata: SerdeAnyMap::default(), + // #[cfg(feature = "std")] + // metadata_path: None, + // exec_time: None, + // cached_len: None, + // executions: 0, + // scheduled_count: 0, + // parent_id: Some(parent_id), + // disabled: false, + // objectives_found: 0, + // #[cfg(feature = "track_hit_feedbacks")] + // hit_feedbacks: Vec::new(), + // #[cfg(feature = "track_hit_feedbacks")] + // hit_objectives: Vec::new(), + // } + // } /// Get the id of the parent, that this testcase was derived from #[must_use] pub fn parent_id(&self) -> Option { - self.parent_id + self.metadata.parent_id } /// Sets the id of the parent, that this testcase was derived from pub fn set_parent_id(&mut self, parent_id: CorpusId) { - self.parent_id = Some(parent_id); + self.metadata.parent_id = Some(parent_id); } /// Sets the id of the parent, that this testcase was derived from pub fn set_parent_id_optional(&mut self, parent_id: Option) { - self.parent_id = parent_id; + self.metadata.parent_id = parent_id; } /// Gets how many objectives were found by mutating this testcase pub fn objectives_found(&self) -> usize { - self.objectives_found + self.metadata.objectives_found } /// Adds one objectives to the `objectives_found` counter. Mostly called from crash handler or executor. pub fn found_objective(&mut self) { - self.objectives_found = self.objectives_found.saturating_add(1); - } -} - -impl Default for Testcase { - /// Create a new default Testcase - #[inline] - fn default() -> Self { - Testcase { - input: None, - filename: None, - metadata: SerdeAnyMap::new(), - exec_time: None, - cached_len: None, - scheduled_count: 0, - parent_id: None, - #[cfg(feature = "std")] - file_path: None, - #[cfg(feature = "std")] - metadata_path: None, - disabled: false, - executions: 0, - objectives_found: 0, - #[cfg(feature = "track_hit_feedbacks")] - hit_feedbacks: Vec::new(), - #[cfg(feature = "track_hit_feedbacks")] - hit_objectives: Vec::new(), - } - } -} - -/// Impl of a testcase when the input has len -impl Testcase -where - I: HasLen, -{ - /// Get the cached `len`. Will `Error::EmptyOptional` if `len` is not yet cached. - #[inline] - pub fn cached_len(&mut self) -> Option { - self.cached_len - } - - /// Get the `len` or calculate it, if not yet calculated. - pub fn load_len>(&mut self, corpus: &C) -> Result { - match &self.input { - Some(i) => { - let l = i.len(); - self.cached_len = Some(l); - Ok(l) - } - None => { - if let Some(l) = self.cached_len { - Ok(l) - } else { - corpus.load_input_into(self)?; - self.load_len(corpus) - } - } - } - } -} - -/// Create a testcase from an input -impl From for Testcase { - fn from(input: I) -> Self { - Testcase::new(input) + self.metadata.objectives_found = self.metadata.objectives_found.saturating_add(1); } } @@ -515,15 +390,3 @@ impl SchedulerTestcaseMetadata { } libafl_bolts::impl_serdeany!(SchedulerTestcaseMetadata); - -#[cfg(feature = "std")] -impl Drop for Testcase { - fn drop(&mut self) { - if let Some(filename) = &self.filename { - let mut path = PathBuf::from(filename); - let lockname = format!(".{}.lafl_lock", path.file_name().unwrap().to_str().unwrap()); - path.set_file_name(lockname); - let _ = std::fs::remove_file(path); - } - } -} diff --git a/libafl/src/executors/inprocess/mod.rs b/libafl/src/executors/inprocess/mod.rs index d3942a9cb48..59d59bbbd8d 100644 --- a/libafl/src/executors/inprocess/mod.rs +++ b/libafl/src/executors/inprocess/mod.rs @@ -16,7 +16,7 @@ use libafl_bolts::tuples::{RefIndexable, tuple_list}; use crate::{ Error, HasMetadata, - corpus::{Corpus, Testcase}, + corpus::{Corpus, Testcase, testcase::TestcaseMetadata}, events::{Event, EventFirer, EventRestarter, EventWithStats}, executors::{ Executor, ExitKind, HasObservers, @@ -344,10 +344,13 @@ pub fn run_observers_and_save_state( .expect("In run_observers_and_save_state objective failure."); if is_solution { - let mut new_testcase = Testcase::from(input.clone()); - new_testcase.set_executions(*state.executions()); + let testcase_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .parent_id(*state.corpus().current()) + .build(); + + let mut new_testcase = Testcase::new(input.clone(), testcase_md); new_testcase.add_metadata(exitkind); - new_testcase.set_parent_id_optional(*state.corpus().current()); if let Ok(mut tc) = state.current_testcase_mut() { tc.found_objective(); diff --git a/libafl/src/feedbacks/mod.rs b/libafl/src/feedbacks/mod.rs index 9460be35ee7..627c9729738 100644 --- a/libafl/src/feedbacks/mod.rs +++ b/libafl/src/feedbacks/mod.rs @@ -35,9 +35,6 @@ pub use bool::BoolValueFeedback; #[cfg(feature = "std")] pub mod concolic; -#[cfg(feature = "std")] -/// The module for `CustomFilenameToTestcaseFeedback` -pub mod custom_filename; pub mod differential; /// The module for list feedback pub mod list; diff --git a/libafl/src/fuzzer/mod.rs b/libafl/src/fuzzer/mod.rs index 203cb7e500c..689413e4f94 100644 --- a/libafl/src/fuzzer/mod.rs +++ b/libafl/src/fuzzer/mod.rs @@ -14,7 +14,9 @@ use serde::{Serialize, de::DeserializeOwned}; use crate::monitors::stats::PerfFeature; use crate::{ Error, HasMetadata, - corpus::{Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, Testcase}, + corpus::{ + Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, Testcase, testcase::TestcaseMetadata, + }, events::{ Event, EventConfig, EventFirer, EventReceiver, EventWithStats, ProgressReporter, SendExiting, @@ -420,13 +422,18 @@ where ) -> Result, Error> { let corpus = if exec_res.is_corpus() { // Add the input to the main corpus - let mut testcase = Testcase::from(input.clone()); - testcase.set_executions(*state.executions()); + let tc_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .build(); + + let testcase = Testcase::new(input.clone(), tc_md); + #[cfg(feature = "track_hit_feedbacks")] self.feedback_mut() .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; self.feedback_mut() .append_metadata(state, manager, observers, &mut testcase)?; + let id = state.corpus_mut().add(testcase)?; self.scheduler_mut().on_add(state, id)?; Ok(Some(id)) @@ -436,10 +443,14 @@ where if exec_res.is_solution() { // The input is a solution, add it to the respective corpus - let mut testcase = Testcase::from(input.clone()); - testcase.set_executions(*state.executions()); + let tc_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .parent_id(*state.corpus().current()) + .build(); + + let mut testcase = Testcase::new(input.clone(), tc_md); testcase.add_metadata(*exit_kind); - testcase.set_parent_id_optional(*state.corpus().current()); + if let Ok(mut tc) = state.current_testcase_mut() { tc.found_objective(); } @@ -677,8 +688,10 @@ where let exit_kind = self.execute_input(state, executor, manager, &input)?; let observers = executor.observers(); // Always consider this to be "interesting" - let mut testcase = Testcase::from(input.clone()); - testcase.set_executions(*state.executions()); + let tc_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .build(); + let mut testcase = Testcase::new(input.clone(), tc_md); // Maybe a solution #[cfg(not(feature = "introspection"))] @@ -766,9 +779,13 @@ where } fn add_disabled_input(&mut self, state: &mut S, input: I) -> Result { - let mut testcase = Testcase::from(input.clone()); - testcase.set_executions(*state.executions()); - testcase.set_disabled(true); + let tc_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .disabled(true) + .build(); + + let mut testcase = Testcase::new(input.clone(), tc_md); + // Add the disabled input to the main corpus let id = state.corpus_mut().add_disabled(testcase)?; Ok(id) diff --git a/libafl/src/mutators/encoded_mutations.rs b/libafl/src/mutators/encoded_mutations.rs index a3a02c9657b..2a12225f087 100644 --- a/libafl/src/mutators/encoded_mutations.rs +++ b/libafl/src/mutators/encoded_mutations.rs @@ -332,7 +332,7 @@ where let other_size = { // new scope to make the borrow checker happy let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - other_testcase.load_input(state.corpus())?.codes().len() + other_testcase.input().codes().len() }; if other_size < 2 { @@ -362,7 +362,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // no need to `load_input` again - we did that above already. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); input.codes_mut().resize(size + len, 0); unsafe { @@ -411,7 +411,7 @@ where let other_size = { // new scope to make the borrow checker happy let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - other_testcase.load_input(state.corpus())?.codes().len() + other_testcase.input().codes().len() }; if other_size < 2 { @@ -437,7 +437,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // no need to load the input again, it'll already be present at this point. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); unsafe { buffer_copy(input.codes_mut(), other.codes(), from, to, len); diff --git a/libafl/src/mutators/gramatron.rs b/libafl/src/mutators/gramatron.rs index 58412723fa0..146a929c133 100644 --- a/libafl/src/mutators/gramatron.rs +++ b/libafl/src/mutators/gramatron.rs @@ -128,17 +128,18 @@ where let rand_num = state.rand_mut().next(); - let mut other_testcase = state.corpus().get(id)?.borrow_mut(); + let other_testcase_ref = state.corpus().get(id)?; + let mut other_testcase = other_testcase_ref.borrow_mut(); if !other_testcase.has_metadata::() { - let meta = GramatronIdxMapMetadata::new(other_testcase.load_input(state.corpus())?); + let meta = GramatronIdxMapMetadata::new(other_testcase.input()); other_testcase.add_metadata(meta); } let meta = other_testcase .metadata_map() .get::() .unwrap(); - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); meta.map.get(&input.terminals()[insert_at].state).map_or( Ok(MutationResult::Skipped), diff --git a/libafl/src/mutators/mutations.rs b/libafl/src/mutators/mutations.rs index 5b3d1373de7..be1380b4af0 100644 --- a/libafl/src/mutators/mutations.rs +++ b/libafl/src/mutators/mutations.rs @@ -1171,10 +1171,7 @@ where let other_size = { let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - other_testcase - .load_input(state.corpus())? - .mutator_bytes() - .len() + other_testcase.input_mut().mutator_bytes().len() }; if other_size < 2 { @@ -1191,7 +1188,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // No need to load the input again, it'll still be cached. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); Ok(Self::crossover_insert( input, @@ -1266,7 +1263,7 @@ where let other_size = { let mut testcase = state.corpus().get_from_all(id)?.borrow_mut(); - testcase.load_input(state.corpus())?.mutator_bytes().len() + testcase.input_mut().mutator_bytes().len() }; if other_size < 2 { @@ -1287,7 +1284,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // No need to load the input again, it'll still be cached. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); Ok(Self::crossover_replace( input, @@ -1371,7 +1368,7 @@ where let other_size = { let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - let other_input = other_testcase.load_input(state.corpus())?; + let other_input = other_testcase.input(); let input_mapped = (self.input_mapper)(other_input).map_to_option_bytes(); input_mapped.map_or(0, >::len) }; @@ -1394,7 +1391,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // No need to load the input again, it'll still be cached. - let other_input = &mut other_testcase.input().as_ref().unwrap(); + let other_input = &mut other_testcase.input(); let wrapped_mapped_other_input = (self.input_mapper)(other_input).map_to_option_bytes(); if wrapped_mapped_other_input.is_none() { return Ok(MutationResult::Skipped); @@ -1458,7 +1455,7 @@ where let other_size = { let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - let other_input = other_testcase.load_input(state.corpus())?; + let other_input = other_testcase.input(); let input_mapped = (self.input_mapper)(other_input).map_to_option_bytes(); input_mapped.map_or(0, >::len) }; @@ -1481,7 +1478,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // No need to load the input again, it'll still be cached. - let other_input = &mut other_testcase.input().as_ref().unwrap(); + let other_input = &mut other_testcase.input(); let wrapped_mapped_other_input = (self.input_mapper)(other_input).map_to_option_bytes(); if wrapped_mapped_other_input.is_none() { return Ok(MutationResult::Skipped); @@ -1541,7 +1538,7 @@ where let (first_diff, last_diff) = { let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - let other = other_testcase.load_input(state.corpus())?; + let other = other_testcase.input(); let (f, l) = locate_diffs(input.mutator_bytes(), other.mutator_bytes()); @@ -1556,7 +1553,7 @@ where let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); // Input will already be loaded. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); input.splice( split_at.., diff --git a/libafl/src/mutators/numeric.rs b/libafl/src/mutators/numeric.rs index 6a4c92e93b4..189f57c703a 100644 --- a/libafl/src/mutators/numeric.rs +++ b/libafl/src/mutators/numeric.rs @@ -379,7 +379,7 @@ where } let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - *input = *other_testcase.input().as_ref().unwrap(); + *input = *other_testcase.input(); Ok(MutationResult::Mutated) } } @@ -420,7 +420,7 @@ where } let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - let other_input = other_testcase.input().as_ref().unwrap(); + let other_input = other_testcase.input(); let mapped_input = (self.input_mapper)(other_input).clone(); *input = mapped_input; Ok(MutationResult::Mutated) diff --git a/libafl/src/state/mod.rs b/libafl/src/state/mod.rs index 6bde85520be..95f3f330cff 100644 --- a/libafl/src/state/mod.rs +++ b/libafl/src/state/mod.rs @@ -18,6 +18,7 @@ use std::{ #[cfg(feature = "std")] use libafl_bolts::core_affinity::{CoreId, Cores}; use libafl_bolts::{ + current_time, rands::{Rand, StdRand}, serdeany::{NamedSerdeAnyMap, SerdeAnyMap}, }; @@ -159,6 +160,13 @@ pub trait HasStartTime { /// The starting time (mutable) fn start_time_mut(&mut self) -> &mut Duration; + + fn time_since_start(&self) -> Duration { + let start_time = self.start_time(); + let current_time = current_time(); + + current_time - *start_time + } } /// Trait for the last report time, the last time this node reported progress @@ -296,12 +304,14 @@ where { /// To get the testcase fn testcase(&self, id: CorpusId) -> Result>, Error> { - Ok(self.corpus().get(id)?.borrow()) + let tc = self.corpus().get(id)?; + Ok(tc.borrow()) } /// To get mutable testcase fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { - Ok(self.corpus().get(id)?.borrow_mut()) + let tc = self.corpus().get(id)?; + Ok(tc.borrow_mut()) } } @@ -1129,7 +1139,7 @@ where rand, executions: 0, imported: 0, - start_time: libafl_bolts::current_time(), + start_time: current_time(), metadata: SerdeAnyMap::default(), named_metadata: NamedSerdeAnyMap::default(), corpus, @@ -1143,7 +1153,7 @@ where #[cfg(feature = "std")] dont_reenter: None, last_report_time: None, - last_found_time: libafl_bolts::current_time(), + last_found_time: current_time(), corpus_id: None, stage_stack: StageStack::default(), phantom: PhantomData, From 8a0ab9b97c73c5cca1214f55ed13282b032d64f5 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Wed, 17 Sep 2025 14:42:58 +0200 Subject: [PATCH 02/12] compiling --- crates/libafl/Cargo.toml | 2 +- crates/libafl/src/common/mod.rs | 97 +-- crates/libafl/src/corpus/cache.rs | 260 ++++++++ crates/libafl/src/corpus/cached.rs | 0 crates/libafl/src/corpus/combined.rs | 319 +--------- crates/libafl/src/corpus/dynamic.rs | 57 +- crates/libafl/src/corpus/inmemory.rs | 0 crates/libafl/src/corpus/inmemory_ondisk.rs | 0 crates/libafl/src/corpus/mod.rs | 90 ++- crates/libafl/src/corpus/nop.rs | 50 +- crates/libafl/src/corpus/ondisk.rs | 0 crates/libafl/src/corpus/single.rs | 61 +- crates/libafl/src/corpus/store/inmemory.rs | 112 ++-- crates/libafl/src/corpus/store/maps.rs | 116 ++-- crates/libafl/src/corpus/store/mod.rs | 54 +- crates/libafl/src/corpus/store/ondisk.rs | 239 ++++---- crates/libafl/src/corpus/testcase.rs | 559 ++++++++++++++---- crates/libafl/src/executors/inprocess/mod.rs | 21 +- crates/libafl/src/feedbacks/bool.rs | 3 +- .../libafl/src/feedbacks/capture_feedback.rs | 8 +- crates/libafl/src/feedbacks/concolic.rs | 17 +- crates/libafl/src/feedbacks/list.rs | 13 +- crates/libafl/src/feedbacks/map.rs | 16 +- crates/libafl/src/feedbacks/mod.rs | 75 +-- .../libafl/src/feedbacks/new_hash_feedback.rs | 8 +- crates/libafl/src/feedbacks/simd.rs | 15 +- crates/libafl/src/feedbacks/stdio.rs | 22 +- crates/libafl/src/feedbacks/transferred.rs | 7 +- crates/libafl/src/fuzzer/mod.rs | 61 +- crates/libafl/src/inputs/generalized.rs | 8 +- .../libafl/src/mutators/encoded_mutations.rs | 8 +- crates/libafl/src/mutators/gramatron.rs | 18 +- crates/libafl/src/mutators/grimoire.rs | 8 +- crates/libafl/src/mutators/mopt_mutator.rs | 12 +- crates/libafl/src/mutators/mutations.rs | 32 +- crates/libafl/src/mutators/numeric.rs | 6 +- crates/libafl/src/mutators/scheduled.rs | 11 +- crates/libafl/src/mutators/token_mutations.rs | 4 +- crates/libafl/src/mutators/tuneable.rs | 26 +- crates/libafl/src/mutators/unicode/mod.rs | 2 +- crates/libafl/src/observers/cmp.rs | 4 +- crates/libafl/src/schedulers/accounting.rs | 38 +- crates/libafl/src/schedulers/minimizer.rs | 78 +-- crates/libafl/src/schedulers/mod.rs | 62 +- crates/libafl/src/schedulers/powersched.rs | 30 +- .../src/schedulers/probabilistic_sampling.rs | 24 +- crates/libafl/src/schedulers/queue.rs | 17 +- .../libafl/src/schedulers/testcase_score.rs | 68 ++- crates/libafl/src/schedulers/tuneable.rs | 18 +- crates/libafl/src/schedulers/weighted.rs | 22 +- crates/libafl/src/stages/afl_stats.rs | 20 +- crates/libafl/src/stages/calibrate.rs | 32 +- crates/libafl/src/stages/colorization.rs | 8 +- crates/libafl/src/stages/concolic.rs | 11 +- crates/libafl/src/stages/dump.rs | 156 ++--- crates/libafl/src/stages/generalization.rs | 41 +- crates/libafl/src/stages/mod.rs | 14 +- crates/libafl/src/stages/mutational.rs | 30 +- crates/libafl/src/stages/power.rs | 22 +- crates/libafl/src/stages/push/mod.rs | 5 +- crates/libafl/src/stages/push/mutational.rs | 5 +- crates/libafl/src/stages/replay.rs | 16 +- crates/libafl/src/stages/shadow.rs | 4 +- crates/libafl/src/stages/sync.rs | 10 +- crates/libafl/src/stages/time_tracker.rs | 6 +- crates/libafl/src/stages/tmin.rs | 29 +- crates/libafl/src/stages/tracing.rs | 10 +- crates/libafl/src/stages/tuneable.rs | 59 +- crates/libafl/src/stages/verify_timeouts.rs | 4 +- crates/libafl/src/state/mod.rs | 76 ++- crates/libafl_libfuzzer/runtime/src/merge.rs | 2 +- fuzzers/baby/tutorial/src/metadata.rs | 7 +- .../src/feedback/persistent_record.rs | 2 +- .../libafl-fuzz/src/feedback/seed.rs | 2 +- utils/libafl_repo_tools/Cargo.lock | 2 +- 75 files changed, 1902 insertions(+), 1449 deletions(-) create mode 100644 crates/libafl/src/corpus/cache.rs delete mode 100644 crates/libafl/src/corpus/cached.rs delete mode 100644 crates/libafl/src/corpus/inmemory.rs delete mode 100644 crates/libafl/src/corpus/inmemory_ondisk.rs delete mode 100644 crates/libafl/src/corpus/ondisk.rs diff --git a/crates/libafl/Cargo.toml b/crates/libafl/Cargo.toml index 9c4f8d23fa9..731643cba48 100644 --- a/crates/libafl/Cargo.toml +++ b/crates/libafl/Cargo.toml @@ -230,7 +230,7 @@ hashbrown = { workspace = true, features = [ "ahash", ], default-features = false } # A faster hashmap, nostd compatible num-traits = { workspace = true, default-features = false } -serde = { workspace = true, features = ["alloc"] } # serialization lib +serde = { workspace = true, features = ["alloc", "rc"] } # serialization lib postcard = { workspace = true } # no_std compatible serde serialization format bincode = { version = "2.0.1", optional = true, features = ["serde"] } bitbybit = { workspace = true } diff --git a/crates/libafl/src/common/mod.rs b/crates/libafl/src/common/mod.rs index 9e508061290..dc050389968 100644 --- a/crates/libafl/src/common/mod.rs +++ b/crates/libafl/src/common/mod.rs @@ -3,17 +3,45 @@ use alloc::boxed::Box; use core::any::type_name; -#[cfg(feature = "nautilus")] -pub mod nautilus; - use libafl_bolts::{ Error, serdeany::{NamedSerdeAnyMap, SerdeAny, SerdeAnyMap}, }; + +#[cfg(feature = "nautilus")] +pub mod nautilus; + /// Trait for elements offering metadata pub trait HasMetadata { /// A map, storing all metadata fn metadata_map(&self) -> &SerdeAnyMap; + + /// Check for a metadata + /// + /// # Note + /// For performance reasons, you likely want to use [`Self::metadata_or_insert_with`] instead + #[inline] + fn has_metadata(&self) -> bool + where + M: SerdeAny, + { + self.metadata_map().get::().is_some() + } + + /// To get metadata + #[inline] + fn metadata(&self) -> Result<&M, Error> + where + M: SerdeAny, + { + self.metadata_map() + .get::() + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + } +} + +/// Trait for elements offering mutable metadata +pub trait HasMetadataMut: HasMetadata { /// A map, storing all metadata (mutable) fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap; @@ -53,45 +81,49 @@ pub trait HasMetadata { self.metadata_map_mut().remove::() } - /// Check for a metadata - /// - /// # Note - /// For performance reasons, you likely want to use [`Self::metadata_or_insert_with`] instead + /// To get mutable metadata #[inline] - fn has_metadata(&self) -> bool + fn metadata_mut(&mut self) -> Result<&mut M, Error> where M: SerdeAny, { - self.metadata_map().get::().is_some() + self.metadata_map_mut() + .get_mut::() + .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) } +} - /// To get metadata +/// Trait for elements offering named metadata +pub trait HasNamedMetadata { + /// A map, storing all metadata + fn named_metadata_map(&self) -> &NamedSerdeAnyMap; + + /// Check for a metadata + /// + /// # Note + /// You likely want to use [`Self::named_metadata_or_insert_with`] for performance reasons. #[inline] - fn metadata(&self) -> Result<&M, Error> + fn has_named_metadata(&self, name: &str) -> bool where M: SerdeAny, { - self.metadata_map() - .get::() - .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) + self.named_metadata_map().contains::(name) } - /// To get mutable metadata + /// To get named metadata #[inline] - fn metadata_mut(&mut self) -> Result<&mut M, Error> + fn named_metadata(&self, name: &str) -> Result<&M, Error> where M: SerdeAny, { - self.metadata_map_mut() - .get_mut::() + self.named_metadata_map() + .get::(name) .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) } } /// Trait for elements offering named metadata -pub trait HasNamedMetadata { - /// A map, storing all metadata - fn named_metadata_map(&self) -> &NamedSerdeAnyMap; +pub trait HasNamedMetadataMut: HasNamedMetadata { /// A map, storing all metadata (mutable) fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap; @@ -136,29 +168,6 @@ pub trait HasNamedMetadata { .get_or_insert_with::(name, default) } - /// Check for a metadata - /// - /// # Note - /// You likely want to use [`Self::named_metadata_or_insert_with`] for performance reasons. - #[inline] - fn has_named_metadata(&self, name: &str) -> bool - where - M: SerdeAny, - { - self.named_metadata_map().contains::(name) - } - - /// To get named metadata - #[inline] - fn named_metadata(&self, name: &str) -> Result<&M, Error> - where - M: SerdeAny, - { - self.named_metadata_map() - .get::(name) - .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) - } - /// To get mutable named metadata #[inline] fn named_metadata_mut(&mut self, name: &str) -> Result<&mut M, Error> diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs new file mode 100644 index 00000000000..d1f2a62e444 --- /dev/null +++ b/crates/libafl/src/corpus/cache.rs @@ -0,0 +1,260 @@ +//! A collection of cache policy implementations. +//! They are meant to be used by [`crate::corpus::CombinedCorpus`]. +//! +//! Caches are acting on two [`Store`]s: +//! - a **cache store** holding on the testcases with quick access. +//! - a **backing store** with more expensive access, used when the testcase cannot be found in the cache store. + +use alloc::rc::Rc; +use std::{cell::RefCell, collections::VecDeque, marker::PhantomData}; + +use libafl_bolts::Error; + +use crate::corpus::{ + CorpusId, Testcase, + store::{RemovableStore, Store}, + testcase::{HasTestcaseMetadata, TestcaseMetadata}, +}; + +/// A First In -> First Out cache policy. +#[derive(Debug)] +pub struct FifoCache { + cached_ids: VecDeque, + cache_max_len: usize, + phantom: PhantomData<(I, CS, FS)>, +} + +/// An identity cache, storing everything both in the cache and the backing store. +#[derive(Debug)] +pub struct IdentityCache; + +/// A cache, managing a cache store and a fallback store. +pub trait Cache { + /// A [`TestcaseMetadata`] cell. + type TestcaseMetadataCell: HasTestcaseMetadata; + + /// Add a testcase to the cache + fn add( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; + + /// Add a disabled testcase to the cache + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; + + /// Get a testcase from the cache + fn get_from( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result, Error>; + + /// Replace a testcase in the cache + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result, Error>; + + /// Flush the cache, committing the cached testcase to the fallback store. + fn flush( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; +} + +impl FifoCache +where + CS: RemovableStore, + FS: Store, + I: Clone, +{ + fn get_inner( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + cache_add_fn: CAF, + cache_get_fn: CGF, + cache_rm_fn: CRF, + fallback_get_fn: FGF, + ) -> Result, Error> + where + CAF: FnOnce(&mut CS, CorpusId, Testcase>) -> Result<(), Error>, + CGF: FnOnce(&CS, CorpusId) -> Result, Error>, + CRF: FnOnce(&mut CS, CorpusId) -> Result, Error>, + FGF: FnOnce(&FS, CorpusId) -> Result, Error>, + { + if self.cached_ids.contains(&id) { + cache_get_fn(cache_store, id) + } else { + if self.cached_ids.len() == self.cache_max_len { + let to_evict = self.cached_ids.pop_back().unwrap(); + cache_rm_fn(cache_store, to_evict)?; + } + + debug_assert!(self.cached_ids.len() < self.cache_max_len); + + // tescase is not cached, fetch it from fallback + let fb_tc = fallback_get_fn(&fallback_store, id)?.cloned(); + cache_add_fn(cache_store, id, fb_tc)?; + + self.cached_ids.push_front(id); + + cache_get_fn(cache_store, id) + } + } +} + +impl Cache for IdentityCache +where + CS: RemovableStore, + FS: Store, + I: Clone, +{ + type TestcaseMetadataCell = CS::TestcaseMetadataCell; + + fn add( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + cache_store.add(id, input.clone(), md.clone())?; + fallback_store.add(id, input, md) + } + + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + cache_store.add_disabled(id, input.clone(), md.clone())?; + fallback_store.add_disabled(id, input, md) + } + + fn get_from( + &mut self, + id: CorpusId, + cache_store: &mut CS, + _fallback_store: &FS, + ) -> Result, Error> { + cache_store.get_from::(id) + } + + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result, Error> { + let old_tc = cache_store.replace(id, input.clone(), md.clone())?; + fallback_store.replace(id, input, md)?; + + Ok(old_tc) + } + + fn flush( + &mut self, + _id: CorpusId, + _cache_store: &mut CS, + _fallback_store: &mut FS, + ) -> Result<(), Error> { + todo!() + } +} + +impl Cache for FifoCache +where + CS: RemovableStore, + FS: Store, + I: Clone, +{ + type TestcaseMetadataCell = CS::TestcaseMetadataCell; + + fn add( + &mut self, + id: CorpusId, + input: Rc, + metadata: TestcaseMetadata, + _cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + fallback_store.add(id, input, metadata) + } + + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + metadata: TestcaseMetadata, + _cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { + fallback_store.add_disabled(id, input, metadata) + } + + fn get_from( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + ) -> Result, Error> { + self.get_inner( + id, + cache_store, + fallback_store, + |cache_store, corpus_id, testcase| { + let (input, md) = testcase.into_inner(); + cache_store.add(corpus_id, input, md) + }, + |cache_store, corpus_id| cache_store.get(corpus_id), + |cache_store, corpus_id| cache_store.remove(corpus_id), + |fallback_store, corpus_id| fallback_store.get_from::(corpus_id), + ) + } + + fn replace( + &mut self, + _id: CorpusId, + _input: Rc, + _metadata: TestcaseMetadata, + _cache_store: &mut CS, + _fallback_store: &mut FS, + ) -> Result, Error> { + todo!() + } + + fn flush( + &mut self, + _id: CorpusId, + _cache_store: &mut CS, + _fallback_store: &mut FS, + ) -> Result<(), Error> { + todo!() + } +} diff --git a/crates/libafl/src/corpus/cached.rs b/crates/libafl/src/corpus/cached.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index e02bbbf92e1..85e2574f36b 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -1,10 +1,14 @@ +//! A cached corpus, using a given [`Cache`] policy and two [`Store`]s. + +use alloc::rc::Rc; use core::{cell::RefCell, marker::PhantomData}; -use std::{collections::VecDeque, rc::Rc, vec::Vec}; +use std::vec::Vec; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; -use super::{Corpus, CorpusCounter, CorpusId, Testcase, store::Store}; +use super::{Cache, Corpus, CorpusCounter, CorpusId, Testcase, store::Store}; +use crate::corpus::testcase::TestcaseMetadata; /// A [`CombinedCorpus`] tries first to use the main store according to some policy. /// If it fails, it falls back to the secondary store. @@ -25,268 +29,15 @@ pub struct CombinedCorpus { phantom: PhantomData, } -pub struct FifoCache { - cached_ids: VecDeque, - cache_max_len: usize, - phantom: PhantomData<(I, CS, FS)>, -} - -pub struct IdentityCache; - -pub trait Cache { - fn add( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error>; - - fn add_disabled( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error>; - - fn replace( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result, Error>; - - fn remove( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result>>, Error>; - - fn get( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - ) -> Result>>, Error>; - - fn get_from_all( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - ) -> Result>>, Error>; -} - -impl FifoCache -where - CS: Store, - FS: Store, - I: Clone, -{ - fn get_inner( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - cache_add_fn: CAF, - cache_get_fn: CGF, - cache_rm_fn: CRF, - fallback_get_fn: FGF, - ) -> Result>>, Error> - where - CAF: FnOnce(&mut CS, CorpusId, Testcase) -> Result<(), Error>, - CGF: FnOnce(&CS, CorpusId) -> Result>>, Error>, - CRF: FnOnce(&mut CS, CorpusId) -> Result>>, Error>, - FGF: FnOnce(&FS, CorpusId) -> Result>>, Error>, - { - if self.cached_ids.contains(&id) { - cache_get_fn(cache_store, id) - } else { - // tescase is not cached, fetch it from fallback - let fb_tc = fallback_get_fn(&fallback_store, id)?; - cache_add_fn(cache_store, id, fb_tc.borrow().clone())?; - - if self.cached_ids.len() == self.cache_max_len { - let to_evict = self.cached_ids.pop_back().unwrap(); - cache_rm_fn(cache_store, to_evict)?; - } - - debug_assert!(self.cached_ids.len() < self.cache_max_len); - - self.cached_ids.push_front(id); - - Ok(fb_tc) - } - } -} - -impl Cache for IdentityCache -where - CS: Store, - FS: Store, - I: Clone, -{ - fn add( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - cache_store.add(id, testcase.clone())?; - fallback_store.add(id, testcase.clone()) - } - - fn add_disabled( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - cache_store.add_disabled(id, testcase.clone())?; - fallback_store.add_disabled(id, testcase.clone()) - } - - fn replace( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result, Error> { - cache_store.replace(id, testcase.clone())?; - fallback_store.replace(id, testcase.clone()) - } - - fn remove( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result>>, Error> { - cache_store.remove(id)?; - fallback_store.remove(id) - } - - fn get( - &mut self, - id: CorpusId, - cache_store: &mut CS, - _fallback_store: &FS, - ) -> Result>>, Error> { - cache_store.get(id) - } - - fn get_from_all( - &mut self, - id: CorpusId, - cache_store: &mut CS, - _fallback_store: &FS, - ) -> Result>>, Error> { - cache_store.get_from_all(id) - } -} - -impl Cache for FifoCache -where - CS: Store, - FS: Store, - I: Clone, -{ - fn get( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - ) -> Result>>, Error> { - self.get_inner( - id, - cache_store, - fallback_store, - |cache_store, corpus_id, testcase| cache_store.add(corpus_id, testcase), - |cache_store, corpus_id| cache_store.get(corpus_id), - |cache_store, corpus_id| cache_store.remove(corpus_id), - |fallback_store, corpus_id| fallback_store.get(corpus_id), - ) - } - - fn get_from_all( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - ) -> Result>>, Error> { - self.get_inner( - id, - cache_store, - fallback_store, - |cache_store, corpus_id, testcase| cache_store.add(corpus_id, testcase), - |cache_store, corpus_id| cache_store.get(corpus_id), - |cache_store, corpus_id| cache_store.remove(corpus_id), - |fallback_store, corpus_id| fallback_store.get_from_all(corpus_id), - ) - } - - fn add( - &mut self, - id: CorpusId, - testcase: Testcase, - _cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - fallback_store.add(id, testcase) - } - - fn add_disabled( - &mut self, - id: CorpusId, - testcase: Testcase, - _cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - fallback_store.add_disabled(id, testcase) - } - - fn replace( - &mut self, - id: CorpusId, - testcase: Testcase, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result, Error> { - if self.cached_ids.contains(&id) { - cache_store.replace(id, testcase.clone())?; - } - - fallback_store.replace(id, testcase) - } - - fn remove( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result>>, Error> { - if self.cached_ids.contains(&id) { - self.cached_ids.retain(|elt| *elt != id); - cache_store.remove(id)?; - } - - fallback_store.remove(id) - } -} - impl Corpus for CombinedCorpus where - C: Cache, + C: Cache, CS: Store, FS: Store, I: Clone, { + type TestcaseMetadataCell = CS::TestcaseMetadataCell; + fn count(&self) -> usize { self.fallback_store.count() } @@ -299,12 +50,13 @@ where self.fallback_store.count_all() } - fn add(&mut self, testcase: Testcase) -> Result { + fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result { let new_id = self.counter.new_id(); self.cache.borrow_mut().add( new_id, - testcase, + input, + md, &mut *self.cache_store.borrow_mut(), &mut self.fallback_store, )?; @@ -312,12 +64,13 @@ where Ok(new_id) } - fn add_disabled(&mut self, testcase: Testcase) -> Result { + fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result { let new_id = self.counter.new_id(); self.cache.borrow_mut().add_disabled( new_id, - testcase, + input, + md, &mut *self.cache_store.borrow_mut(), &mut self.fallback_store, )?; @@ -325,36 +78,28 @@ where Ok(new_id) } - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - self.cache.borrow_mut().replace( - id, - testcase, - &mut *self.cache_store.borrow_mut(), - &mut self.fallback_store, - ) - } - - fn remove(&mut self, id: CorpusId) -> Result>>, Error> { - self.cache.borrow_mut().remove( - id, - &mut *self.cache_store.borrow_mut(), - &mut self.fallback_store, - ) - } + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + let mut cache = self.cache.borrow_mut(); + let cache_store = &mut *self.cache_store.borrow_mut(); - fn get(&self, id: CorpusId) -> Result>>, Error> { - self.cache.borrow_mut().get( - id, - &mut *self.cache_store.borrow_mut(), - &self.fallback_store, - ) + cache.get_from::(id, cache_store, &self.fallback_store) } - fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { - self.cache.borrow_mut().get_from_all( + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result, Error> { + self.cache.borrow_mut().replace( id, + input, + md, &mut *self.cache_store.borrow_mut(), - &self.fallback_store, + &mut self.fallback_store, ) } diff --git a/crates/libafl/src/corpus/dynamic.rs b/crates/libafl/src/corpus/dynamic.rs index 5b2fb3e950b..c20fa863296 100644 --- a/crates/libafl/src/corpus/dynamic.rs +++ b/crates/libafl/src/corpus/dynamic.rs @@ -1,6 +1,7 @@ //! Dynamic corpus that allows users to switch corpus types at runtime. -use core::{cell::RefCell, marker::PhantomData}; +use alloc::rc::Rc; +use core::marker::PhantomData; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; @@ -38,13 +39,6 @@ where C1: Corpus, C2: Corpus, { - fn peek_free_id(&self) -> CorpusId { - match self { - Self::Corpus1(c1, _) => c1.peek_free_id(), - Self::Corpus2(c2, _) => c2.peek_free_id(), - } - } - fn add(&mut self, testcase: Testcase) -> Result { match self { Self::Corpus1(c1, _) => c1.add(testcase), @@ -59,16 +53,6 @@ where } } - fn cloned_input_for_id(&self, idx: CorpusId) -> Result - where - I: Clone, - { - match self { - Self::Corpus1(c1, _) => c1.cloned_input_for_id(idx), - Self::Corpus2(c2, _) => c2.cloned_input_for_id(idx), - } - } - fn count(&self) -> usize { match self { Self::Corpus1(c1, _) => c1.count(), @@ -111,14 +95,14 @@ where } } - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error> { + fn get(&self, id: CorpusId) -> Result>, Error> { match self { Self::Corpus1(c1, _) => c1.get(id), Self::Corpus2(c2, _) => c2.get(id), } } - fn get_from_all(&self, id: CorpusId) -> Result<&RefCell>, Error> { + fn get_from_all(&self, id: CorpusId) -> Result>, Error> { match self { Self::Corpus1(c1, _) => c1.get_from_all(id), Self::Corpus2(c2, _) => c2.get_from_all(id), @@ -139,13 +123,6 @@ where } } - fn load_input_into(&self, testcase: &mut Testcase) -> Result<(), Error> { - match self { - Self::Corpus1(c1, _) => c1.load_input_into(testcase), - Self::Corpus2(c2, _) => c2.load_input_into(testcase), - } - } - fn next(&self, id: CorpusId) -> Option { match self { Self::Corpus1(c1, _) => c1.next(id), @@ -174,24 +151,10 @@ where } } - fn remove(&mut self, id: CorpusId) -> Result, Error> { - match self { - Self::Corpus1(c1, _) => c1.remove(id), - Self::Corpus2(c2, _) => c2.remove(id), - } - } - - fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { - match self { - Self::Corpus1(c1, _) => c1.replace(idx, testcase), - Self::Corpus2(c2, _) => c2.replace(idx, testcase), - } - } - - fn store_input_from(&self, testcase: &Testcase) -> Result<(), Error> { - match self { - Self::Corpus1(c1, _) => c1.store_input_from(testcase), - Self::Corpus2(c2, _) => c2.store_input_from(testcase), - } - } + // fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { + // match self { + // Self::Corpus1(c1, _) => c1.replace(idx, testcase), + // Self::Corpus2(c2, _) => c2.replace(idx, testcase), + // } + // } } diff --git a/crates/libafl/src/corpus/inmemory.rs b/crates/libafl/src/corpus/inmemory.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/crates/libafl/src/corpus/inmemory_ondisk.rs b/crates/libafl/src/corpus/inmemory_ondisk.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index 77446f57ff2..84b74bf4b3d 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -1,14 +1,20 @@ //! Corpuses contain the testcases, either in memory, on disk, or somewhere else. -use core::{cell::RefCell, fmt, marker::PhantomData}; -use std::{rc::Rc, string::String}; +use alloc::rc::Rc; +use core::{fmt, marker::PhantomData}; +use std::{cell::RefCell, string::String}; use serde::{Deserialize, Serialize}; use crate::Error; pub mod testcase; -pub use testcase::{HasTestcase, SchedulerTestcaseMetadata, Testcase}; +pub use testcase::{ + HasTestcase, HasTestcaseMetadata, SchedulerTestcaseMetadata, Testcase, TestcaseMetadata, +}; + +pub mod cache; +pub use cache::{Cache, FifoCache, IdentityCache}; pub mod single; pub use single::SingleCorpus; @@ -17,7 +23,7 @@ pub mod dynamic; pub use dynamic::DynamicCorpus; pub mod combined; -pub use combined::{CombinedCorpus, FifoCache, IdentityCache}; +pub use combined::CombinedCorpus; #[cfg(all(feature = "cmin", unix))] pub mod minimizer; @@ -30,28 +36,38 @@ pub use nop::NopCorpus; pub mod store; pub use store::{InMemoryStore, OnDiskStore, maps}; +/// The standard fully in-memory corpus map. #[cfg(not(feature = "corpus_btreemap"))] -pub type InMemoryCorpusMap = maps::HashCorpusMap>; +pub type StdInMemoryCorpusMap = maps::HashCorpusMap>>>; +/// The standard fully in-memory corpus map. #[cfg(feature = "corpus_btreemap")] -pub type InMemoryCorpusMap = maps::BtreeCorpusMap>; +pub type StdInMemoryCorpusMap = maps::BtreeCorpusMap>>>; + +/// The standard fully in-memory store. +pub type StdInMemoryStore = + InMemoryStore, Rc>>; + +/// The standard fully on-disk store. +pub type StdOnDiskStore = OnDiskStore>; -pub type InMemoryCorpus = SingleCorpus>>; +/// The standard in-memory corpus. +pub type InMemoryCorpus = SingleCorpus>; +/// The standard fully on-disk corpus. #[cfg(feature = "std")] pub type OnDiskCorpus = SingleCorpus>>; -pub type InMemoryOnDiskCorpus = CombinedCorpus< - IdentityCache, - InMemoryStore>, - OnDiskStore>, - I, ->; +/// The standard corpus for storing on disk and in-memory. +pub type InMemoryOnDiskCorpus = + CombinedCorpus, StdOnDiskStore, I>; +/// The standard corpus for storing on disk and in-memory with a cache. +/// Useful for very large corpuses. pub type CachedOnDiskCorpus = CombinedCorpus< - FifoCache>, OnDiskStore>, I>, - InMemoryStore>, - OnDiskStore>, + FifoCache, StdOnDiskStore, I>, + StdInMemoryStore, + StdOnDiskStore, I, >; @@ -60,6 +76,8 @@ pub type CachedOnDiskCorpus = CombinedCorpus< #[repr(transparent)] pub struct CorpusId(pub usize); +/// A counter for [`Corpus`] implementors. +/// Useful to generate fresh [`CorpusId`]s. #[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct CorpusCounter { /// A fresh, progressive ID @@ -115,6 +133,9 @@ macro_rules! random_corpus_id_with_disabled { /// Corpus with all current [`Testcase`]s, or solutions pub trait Corpus: Sized { + /// A [`TestcaseMetadata`] cell. + type TestcaseMetadataCell: HasTestcaseMetadata; + /// Returns the number of all enabled entries fn count(&self) -> usize; @@ -130,22 +151,34 @@ pub trait Corpus: Sized { } /// Add an enabled testcase to the corpus and return its index - fn add(&mut self, testcase: Testcase) -> Result; + fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result; /// Add a disabled testcase to the corpus and return its index - fn add_disabled(&mut self, testcase: Testcase) -> Result; + fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result; - /// Replaces the [`Testcase`] at the given idx, returning the existing. - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error>; + /// Get testcase by id; considers only enabled testcases + fn get(&self, id: CorpusId) -> Result, Error> { + Self::get_from::(self, id) + } - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - fn remove(&mut self, id: CorpusId) -> Result>>, Error>; + /// Get testcase by id, looking at the enabled and disabled stores. + fn get_from_all(&self, id: CorpusId) -> Result, Error> { + Self::get_from::(self, id) + } - /// Get by id; considers only enabled testcases - fn get(&self, id: CorpusId) -> Result<&RefCell>, Error>; + /// Get testcase by id + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error>; - /// Get by id; considers both enabled and disabled testcases - fn get_from_all(&self, id: CorpusId) -> Result>>, Error>; + /// Replace a [`Testcase`] by another one. + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result, Error>; /// Current testcase scheduled fn current(&self) -> &Option; @@ -156,9 +189,6 @@ pub trait Corpus: Sized { /// Get the next corpus id fn next(&self, id: CorpusId) -> Option; - // /// Peek the next free corpus id - // fn peek_free_id(&self) -> CorpusId; - /// Get the prev corpus id fn prev(&self, id: CorpusId) -> Option; @@ -268,7 +298,7 @@ where impl CorpusCounter { fn new_id(&mut self) -> CorpusId { let old = self.current_id; - self.current_id.saturating_add(1); + self.current_id += 1; CorpusId(old) } } diff --git a/crates/libafl/src/corpus/nop.rs b/crates/libafl/src/corpus/nop.rs index 768094d804e..5af696f9bb8 100644 --- a/crates/libafl/src/corpus/nop.rs +++ b/crates/libafl/src/corpus/nop.rs @@ -1,12 +1,16 @@ //! The null corpus does not store any [`Testcase`]s. -use core::{cell::RefCell, marker::PhantomData}; -use std::rc::Rc; + +use alloc::rc::Rc; +use core::marker::PhantomData; use serde::{Deserialize, Serialize}; use crate::{ Error, - corpus::{Corpus, CorpusId, Testcase}, + corpus::{ + Corpus, CorpusId, Testcase, + testcase::{NopTestcaseMetadataCell, TestcaseMetadata}, + }, }; /// A corpus which does not store any [`Testcase`]s. @@ -17,6 +21,8 @@ pub struct NopCorpus { } impl Corpus for NopCorpus { + type TestcaseMetadataCell = NopTestcaseMetadataCell; + /// Returns the number of all enabled entries #[inline] fn count(&self) -> usize { @@ -36,37 +42,20 @@ impl Corpus for NopCorpus { /// Add an enabled testcase to the corpus and return its index #[inline] - fn add(&mut self, _testcase: Testcase) -> Result { + fn add(&mut self, _input: Rc, _md: TestcaseMetadata) -> Result { Err(Error::unsupported("Unsupported by NopCorpus")) } /// Add a disabled testcase to the corpus and return its index #[inline] - fn add_disabled(&mut self, _testcase: Testcase) -> Result { - Err(Error::unsupported("Unsupported by NopCorpus")) - } - - /// Replaces the testcase with the given id - #[inline] - fn replace(&mut self, _id: CorpusId, _testcase: Testcase) -> Result, Error> { - Err(Error::unsupported("Unsupported by NopCorpus")) - } - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - #[inline] - fn remove(&mut self, _id: CorpusId) -> Result>>, Error> { + fn add_disabled(&mut self, _input: Rc, _md: TestcaseMetadata) -> Result { Err(Error::unsupported("Unsupported by NopCorpus")) } - /// Get by id; considers only enabled testcases - #[inline] - fn get(&self, _id: CorpusId) -> Result>>, Error> { - Err(Error::unsupported("Unsupported by NopCorpus")) - } - - /// Get by id; considers both enabled and disabled testcases - #[inline] - fn get_from_all(&self, _id: CorpusId) -> Result>>, Error> { + fn get_from( + &self, + _id: CorpusId, + ) -> Result, Error> { Err(Error::unsupported("Unsupported by NopCorpus")) } @@ -113,6 +102,15 @@ impl Corpus for NopCorpus { fn nth_from_all(&self, _nth: usize) -> CorpusId { CorpusId::from(0_usize) } + + fn replace( + &mut self, + _id: CorpusId, + _input: Rc, + _md: TestcaseMetadata, + ) -> Result, Error> { + Err(Error::unsupported("Unsupported by NopCorpus")) + } } impl NopCorpus { diff --git a/crates/libafl/src/corpus/ondisk.rs b/crates/libafl/src/corpus/ondisk.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 1a4ee44917f..86c3537d2f8 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -1,14 +1,20 @@ -use core::{cell::RefCell, marker::PhantomData}; -use std::{rc::Rc, vec::Vec}; +//! A simple corpus, with a backing store. +//! +//! A [`SingleCorpus`] owns a single store, in which every testcase is added. + +use alloc::rc::Rc; +use core::marker::PhantomData; +use std::vec::Vec; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; use super::{Corpus, CorpusCounter, CorpusId, Testcase, store::Store}; +use crate::corpus::testcase::TestcaseMetadata; /// You average corpus. /// It has one backing store, used to store / retrieve testcases. -#[derive(Default, Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct SingleCorpus { /// The backing testcase store store: S, @@ -21,11 +27,11 @@ pub struct SingleCorpus { phantom: PhantomData, } -impl SingleCorpus +impl Default for SingleCorpus where S: Default, { - pub fn new() -> Self { + fn default() -> Self { Self { store: S::default(), counter: CorpusCounter::default(), @@ -36,10 +42,22 @@ where } } +impl SingleCorpus +where + S: Default, +{ + /// Create a new [`SingleCorpus`] + pub fn new() -> Self { + Self::default() + } +} + impl Corpus for SingleCorpus where S: Store, { + type TestcaseMetadataCell = S::TestcaseMetadataCell; + fn count(&self) -> usize { self.store.count() } @@ -52,32 +70,33 @@ where self.store.count_all() } - fn add(&mut self, testcase: Testcase) -> Result { + fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result { let new_id = self.counter.new_id(); - self.store.add(new_id, testcase)?; + self.store.add(new_id, input, md)?; Ok(new_id) } - fn add_disabled(&mut self, testcase: Testcase) -> Result { + fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result { let new_id = self.counter.new_id(); - self.store.add_disabled(new_id, testcase)?; + self.store.add_disabled(new_id, input, md)?; Ok(new_id) } - fn replace(&mut self, id: CorpusId, testcase: Testcase) -> Result, Error> { - self.store.replace(id, testcase) - } - - fn remove(&mut self, id: CorpusId) -> Result>>, Error> { - self.store.remove(id) - } - - fn get(&self, id: CorpusId) -> Result>>, Error> { - self.store.get(id) + /// Get testcase by id + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.store.get_from::(id) } - fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { - self.store.get_from_all(id) + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result, Error> { + self.store.replace(id, input, md) } fn current(&self) -> &Option { diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index cc6af6ac76f..92bff8c1d4b 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -1,24 +1,49 @@ -use core::{cell::RefCell, marker::PhantomData}; -use std::rc::Rc; +//! An in-memory store + +use alloc::rc::Rc; +use core::marker::PhantomData; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; use super::{InMemoryCorpusMap, Store}; -use crate::corpus::{CorpusId, Testcase}; +use crate::{ + corpus::{ + CorpusId, Testcase, + testcase::{HasInstantiableTestcaseMetadata, TestcaseMetadata}, + }, + inputs::Input, +}; /// The map type in which testcases are stored (disable the feature `corpus_btreemap` to use a `HashMap` instead of `BTreeMap`) -#[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct InMemoryStore { +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct InMemoryStore { enabled_map: M, disabled_map: M, - phantom: PhantomData, + phantom: PhantomData<(I, TMC)>, +} + +impl Default for InMemoryStore +where + M: Default, +{ + fn default() -> Self { + Self { + enabled_map: M::default(), + disabled_map: M::default(), + phantom: PhantomData, + } + } } -impl Store for InMemoryStore +impl Store for InMemoryStore where - M: InMemoryCorpusMap>, + M: InMemoryCorpusMap>, + TMC: HasInstantiableTestcaseMetadata + Clone, + I: Input, { + type TestcaseMetadataCell = TMC; + fn count(&self) -> usize { self.enabled_map.count() } @@ -31,46 +56,55 @@ where self.enabled_map.is_empty() } - fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { - Ok(self.enabled_map.add(id, testcase)) + fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error> { + Ok(self + .enabled_map + .add(id, Testcase::new(input, TMC::instantiate(md)))) } - fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { - Ok(self.disabled_map.add(id, testcase)) + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result<(), Error> { + Ok(self + .disabled_map + .add(id, Testcase::new(input, TMC::instantiate(md)))) } - fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error> { - self.enabled_map.replace(id, new_testcase).ok_or_else(|| { - Error::key_not_found(format!("Index {id} not found, could not replace.")) - }) - } + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + if ENABLED { + self.enabled_map + .get(id) + .cloned() + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) + } else { + let mut testcase = self.enabled_map.get(id); - fn remove(&mut self, id: CorpusId) -> Result>>, Error> { - let mut testcase = self.enabled_map.remove(id); + if testcase.is_none() { + testcase = self.disabled_map.get(id); + } - if testcase.is_none() { - testcase = self.disabled_map.remove(id); + testcase + .cloned() + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) } - - testcase - .map(|x| x.clone()) - .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) - } - - fn get(&self, id: CorpusId) -> Result>>, Error> { - self.enabled_map - .get(id) - .ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) } - fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { - let mut testcase = self.enabled_map.get(id); - - if testcase.is_none() { - testcase = self.disabled_map.get(id); - } - - testcase.ok_or_else(|| Error::key_not_found(format!("Index {id} not found"))) + fn replace( + &mut self, + id: CorpusId, + input: Rc, + metadata: TestcaseMetadata, + ) -> Result, Error> { + Ok(self + .enabled_map + .replace(id, Testcase::new(input, TMC::instantiate(metadata))) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?) } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/store/maps.rs b/crates/libafl/src/corpus/store/maps.rs index 86b2213d9a7..bd0d6941aa6 100644 --- a/crates/libafl/src/corpus/store/maps.rs +++ b/crates/libafl/src/corpus/store/maps.rs @@ -1,13 +1,13 @@ //! Multiple map implementations for the in-memory store. -use core::cell::RefCell; -use std::{collections::BTreeMap, rc::Rc, vec::Vec}; +use std::{collections::BTreeMap, vec::Vec}; use num_traits::Zero; use serde::{Deserialize, Serialize}; use crate::corpus::CorpusId; +/// A trait implemented by in-memory corpus maps pub trait InMemoryCorpusMap { /// Returns the number of testcases fn count(&self) -> usize; @@ -20,14 +20,14 @@ pub trait InMemoryCorpusMap { /// Store the testcase associated to corpus_id. fn add(&mut self, id: CorpusId, testcase: T); - /// Replaces the [`Testcase`] at the given idx, returning the existing. - fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option; - - /// Removes an entry from the corpus, returning it if it was present; considers both enabled and disabled testcases - fn remove(&mut self, id: CorpusId) -> Option>>; + /// Get by id; considers only enabled testcases + fn get(&self, id: CorpusId) -> Option<&T>; /// Get by id; considers only enabled testcases - fn get(&self, id: CorpusId) -> Option>>; + fn get_mut(&mut self, id: CorpusId) -> Option<&mut T>; + + /// Replace a testcase by another one + fn replace(&mut self, id: CorpusId, testcase: T) -> Option; /// Get the prev corpus id in chronological order fn prev(&self, id: CorpusId) -> Option; @@ -45,31 +45,34 @@ pub trait InMemoryCorpusMap { fn nth(&self, nth: usize) -> CorpusId; } +/// A history for [`CorpusId`] #[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct CorpusIdHistory { keys: Vec, } -#[derive(Default, Serialize, Deserialize, Clone, Debug)] +/// A [`BTreeMap`] based [`InMemoryCorpusMap`] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct BtreeCorpusMap { /// A map of `CorpusId` to `Testcase`. - map: BTreeMap>>, + map: BTreeMap, /// A list of available corpus ids history: CorpusIdHistory, } /// Keep track of the stored `Testcase` and the siblings ids (insertion order) -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestcaseStorageItem { /// The stored testcase - pub testcase: Rc>, + pub testcase: T, /// Previously inserted id pub prev: Option, /// Following inserted id pub next: Option, } -#[derive(Default, Serialize, Deserialize, Clone, Debug)] +/// A [`hashbrown::HashMap`] based [`InMemoryCorpusMap`] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct HashCorpusMap { /// A map of `CorpusId` to `TestcaseStorageItem` map: hashbrown::HashMap>, @@ -81,6 +84,26 @@ pub struct HashCorpusMap { history: CorpusIdHistory, } +impl Default for BtreeCorpusMap { + fn default() -> Self { + Self { + map: BTreeMap::default(), + history: CorpusIdHistory::default(), + } + } +} + +impl Default for HashCorpusMap { + fn default() -> Self { + Self { + map: hashbrown::HashMap::default(), + first_id: None, + last_id: None, + history: CorpusIdHistory::default(), + } + } +} + impl CorpusIdHistory { /// Add a key to the history pub fn add(&mut self, id: CorpusId) { @@ -90,7 +113,7 @@ impl CorpusIdHistory { } /// Remove a key from the history - fn remove(&mut self, id: CorpusId) { + pub fn remove(&mut self, id: CorpusId) { if let Ok(idx) = self.keys.binary_search(&id) { self.keys.remove(idx); } @@ -130,45 +153,31 @@ impl InMemoryCorpusMap for HashCorpusMap { self.map.insert( id, TestcaseStorageItem { - testcase: Rc::new(RefCell::new(testcase)), + testcase, prev, next: None, }, ); } - fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { - match self.map.get_mut(&id) { - Some(entry) => Some(entry.testcase.replace(new_testcase)), - _ => None, - } + fn get(&self, id: CorpusId) -> Option<&T> { + self.map.get(&id).map(|inner| &inner.testcase) } - fn remove(&mut self, id: CorpusId) -> Option>> { - if let Some(item) = self.map.remove(&id) { - if let Some(prev) = item.prev { - self.history.remove(id); - self.map.get_mut(&prev).unwrap().next = item.next; - } else { - // first elem - self.first_id = item.next; - } + fn get_mut(&mut self, id: CorpusId) -> Option<&mut T> { + self.map.get_mut(&id).map(|storage| &mut storage.testcase) + } - if let Some(next) = item.next { - self.map.get_mut(&next).unwrap().prev = item.prev; - } else { - // last elem - self.last_id = item.prev; - } + fn replace(&mut self, id: CorpusId, testcase: T) -> Option { + let old_tc = self.map.get_mut(&id)?; - Some(item.testcase) - } else { - None - } - } + let new_tc = TestcaseStorageItem { + testcase, + prev: old_tc.prev, + next: old_tc.next, + }; - fn get(&self, id: CorpusId) -> Option>> { - self.map.get(&id).map(|inner| inner.testcase.clone()) + self.map.insert(id, new_tc).map(|storage| storage.testcase) } fn prev(&self, id: CorpusId) -> Option { @@ -209,23 +218,26 @@ impl InMemoryCorpusMap for BtreeCorpusMap { fn add(&mut self, id: CorpusId, testcase: T) { // corpus.insert_key(id); - self.map.insert(id, Rc::new(RefCell::new(testcase))); + self.map.insert(id, testcase); self.history.add(id); } - fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { - self.map - .get_mut(&id) - .map(|entry| entry.replace(new_testcase)) + // fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { + // self.map + // .get_mut(&id) + // .map(|entry| entry.replace(new_testcase)) + // } + + fn get(&self, id: CorpusId) -> Option<&T> { + self.map.get(&id) } - fn remove(&mut self, id: CorpusId) -> Option>> { - self.history.remove(id); - self.map.remove(&id) + fn get_mut(&mut self, id: CorpusId) -> Option<&mut T> { + self.map.get_mut(&id) } - fn get(&self, id: CorpusId) -> Option>> { - self.map.get(&id).cloned() + fn replace(&mut self, id: CorpusId, testcase: T) -> Option { + self.map.insert(id, testcase) } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index ffb6239f899..0ca252b0dc2 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -1,11 +1,11 @@ //! Stores are collections managing testcases -use core::cell::RefCell; -use std::rc::Rc; +use alloc::rc::Rc; use libafl_bolts::Error; use super::{CorpusId, Testcase}; +use crate::corpus::testcase::{HasTestcaseMetadata, TestcaseMetadata}; pub mod maps; pub use maps::{BtreeCorpusMap, HashCorpusMap, InMemoryCorpusMap}; @@ -14,10 +14,13 @@ pub mod inmemory; pub use inmemory::InMemoryStore; pub mod ondisk; -pub use ondisk::OnDiskStore; +pub use ondisk::{OnDiskMetadataFormat, OnDiskStore}; /// A store is responsible for storing and retrieving [`Testcase`]s, ordered by add time. pub trait Store { + /// A [`TestcaseMetadata`] cell. + type TestcaseMetadataCell: HasTestcaseMetadata; + /// Returns the number of all enabled entries fn count(&self) -> usize; @@ -35,22 +38,39 @@ pub trait Store { } /// Store the testcase associated to corpus_id to the enabled set. - fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error>; + fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error>; /// Store the testcase associated to corpus_id to the disabled set. - fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error>; - - /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. - fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error>; + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result<(), Error>; + + /// Get testcase by id; considers only enabled testcases + fn get(&self, id: CorpusId) -> Result, Error> { + Self::get_from::(self, id) + } - /// Removes an entry from the corpus, returning it; considers both enabled and disabled testcases - fn remove(&mut self, id: CorpusId) -> Result>>, Error>; + /// Get testcase by id; considers both enabled and disabled testcases + fn get_from_all(&self, id: CorpusId) -> Result, Error> { + Self::get_from::(self, id) + } - /// Get by id; considers only enabled testcases - fn get(&self, id: CorpusId) -> Result>>, Error>; + /// Get testcase by id + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error>; - /// Get by id; considers both enabled and disabled testcases - fn get_from_all(&self, id: CorpusId) -> Result>>, Error>; + /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. + fn replace( + &mut self, + id: CorpusId, + input: Rc, + metadata: TestcaseMetadata, + ) -> Result, Error>; /// Get the prev corpus id in chronological order fn prev(&self, id: CorpusId) -> Option; @@ -70,3 +90,9 @@ pub trait Store { /// Get the nth corpus id; considers both enabled and disabled testcases fn nth_from_all(&self, nth: usize) -> CorpusId; } + +/// A Store with removable entries +pub trait RemovableStore: Store { + /// Removes an entry from the corpus, returning it; considers both enabled and disabled testcases + fn remove(&mut self, id: CorpusId) -> Result, Error>; +} diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index ee54adde908..52d163d4216 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -1,34 +1,41 @@ -use core::{cell::RefCell, marker::PhantomData}; +//! An on-disk store + +use alloc::rc::Rc; +use core::marker::PhantomData; use std::{ + cell::{Ref, RefCell, RefMut}, fs::{self, File}, - io::{self, Read, Seek, SeekFrom, Write}, + io::Write, path::{Path, PathBuf}, - rc::Rc, string::String, vec::Vec, }; -use fs4::fs_std::FileExt; use libafl_bolts::{Error, compress::GzipCompressor}; use serde::{Deserialize, Serialize}; use super::{InMemoryCorpusMap, Store}; use crate::{ - corpus::{CorpusId, Testcase, testcase::TestcaseMetadata}, + corpus::{ + CorpusId, Testcase, + testcase::{HasTestcaseMetadata, TestcaseMetadata}, + }, inputs::Input, }; +/// An on-disk store #[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct OnDiskStore { - disk_mgr: DiskMgr, + disk_mgr: Rc>, enabled_map: M, disabled_map: M, first: Option, last: Option, } +/// A Disk Manager, able to load and store [`Testcase`]s #[derive(Default, Debug, Clone, Serialize, Deserialize)] -struct DiskMgr { +pub struct DiskMgr { root_dir: PathBuf, md_format: OnDiskMetadataFormat, phantom: PhantomData, @@ -50,6 +57,7 @@ pub enum OnDiskMetadataFormat { } impl OnDiskMetadataFormat { + /// Convert a [`TestcaseMetadata`] to the format [`OnDiskMetadataFormat`] and stores it in a [`Vec`]. pub fn to_vec(&self, testcase_md: &TestcaseMetadata) -> Result, Error> { let json_error = |err| Error::serialize(format!("Failed to json-ify metadata: {err:?}")); @@ -65,6 +73,7 @@ impl OnDiskMetadataFormat { }) } + /// Load a [`TestcaseMetadata`] from a file with a format [`OnDiskMetadataFormat`]. pub fn from_file(&self, md_path: &Path) -> Result { let json_error = |err| Error::serialize(format!("Failed to parse metadata: {err:?}")); let md_serialized = fs::read(md_path)?; @@ -86,66 +95,51 @@ impl OnDiskMetadataFormat { } } +/// An on-disk [`Testcase`] cell. #[derive(Debug)] -struct TestcaseLockfile { - lockfile: File, - nb_used: u32, +pub struct OnDiskTestcaseCell { + mgr: Rc>, + testcase_md: RefCell, + modified: bool, } -impl TestcaseLockfile { - pub fn new(ondisk_mgr: &DiskMgr, testcase_id: &String) -> Result { - let lockfile_path = ondisk_mgr.root_dir.join(format!(".{}.lock", testcase_id)); - - let mut lockfile = match File::create_new(lockfile_path.as_path()) { - Ok(f) => f, - Err(e) => match e.kind() { - io::ErrorKind::AlreadyExists => File::open(lockfile_path.as_path()).unwrap(), - - _ => return Err(e.into()), - }, - }; - - lockfile.lock_exclusive()?; - - let mut nb_used_buf: [u8; 4] = [0; 4]; - let nb_used: u32 = match lockfile.read_exact(&mut nb_used_buf) { - Ok(()) => u32::from_le_bytes(nb_used_buf), - Err(e) => match e.kind() { - io::ErrorKind::UnexpectedEof => 0, - - _ => return Err(e.into()), - }, - }; +impl OnDiskTestcaseCell { + /// Get a new [`OnDiskTestcaseCell`]. + pub fn new(mgr: Rc>, testcase_md: TestcaseMetadata) -> Self { + Self { + mgr, + testcase_md: RefCell::new(testcase_md), + modified: false, + } + } +} - Ok(Self { lockfile, nb_used }) +impl HasTestcaseMetadata for OnDiskTestcaseCell { + type TestcaseMetadataRef<'a> + = Ref<'a, TestcaseMetadata> + where + I: 'a; + type TestcaseMetadataRefMut<'a> + = RefMut<'a, TestcaseMetadata> + where + I: 'a; + + fn testcase_metadata<'a>(&'a self) -> Ref<'a, TestcaseMetadata> { + self.testcase_md.borrow() } - /// returns true if it is the first use - pub fn inc_used(&mut self) -> bool { - self.nb_used += 1; - self.nb_used == 1 + fn testcase_metadata_mut<'a>(&'a self) -> RefMut<'a, TestcaseMetadata> { + self.testcase_md.borrow_mut() } - /// returns true if not in used anymore - /// can be safely deleted - pub fn dec_used(&mut self) -> bool { - if self.nb_used == 0 { - true - } else { - self.nb_used -= 1; - self.nb_used == 0 - } + fn into_testcase_metadata(self) -> TestcaseMetadata { + self.testcase_md.clone().into_inner() } } -impl Drop for TestcaseLockfile { +impl Drop for OnDiskTestcaseCell { fn drop(&mut self) { - let nb_used_buf = self.nb_used.to_le_bytes(); - - self.lockfile.seek(SeekFrom::Start(0)); - self.lockfile.write_all(&nb_used_buf).unwrap(); - - FileExt::unlock(&self.lockfile); + todo!() } } @@ -161,52 +155,47 @@ where self.root_dir.join(format!(".{}.metadata", testcase_id)) } - fn save_testcase(&self, testcase: &Testcase) -> Result { - let testcase_id = testcase.id(); - let testcase_path = self.testcase_path(testcase_id); - let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; + fn save_testcase(&self, input: Rc, md: TestcaseMetadata) -> Result { + let testcase_id = Testcase::>::compute_id(input.as_ref()); + let testcase_path = self.testcase_path(&testcase_id); + // let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; - if lockfile.inc_used() { - // save md to file - let ser_fmt = self.md_format.clone(); - let testcase_md_path = self.testcase_md_path(testcase_id); + // if lockfile.inc_used() { + // save md to file + let ser_fmt = self.md_format.clone(); + let testcase_md_path = self.testcase_md_path(&testcase_id); - let mut testcase_md_f = File::create_new(testcase_md_path.as_path()).unwrap(); - let testcase_md_ser = ser_fmt.to_vec(testcase.metadata())?; + let mut testcase_md_f = File::create_new(testcase_md_path.as_path()).unwrap(); + let testcase_md_ser = ser_fmt.to_vec(&md)?; - testcase_md_f.write_all(&testcase_md_ser)?; + testcase_md_f.write_all(&testcase_md_ser)?; - // testcase_f.write_all(testcase.input().target_bytes().as_ref())?; - testcase.input().to_file(testcase_path.as_path())?; - } + // testcase_f.write_all(testcase.input().target_bytes().as_ref())?; + input.as_ref().to_file(testcase_path.as_path())?; + // } - Ok(testcase_id.clone()) + Ok(testcase_id) } /// prerequisite: the testcase should not have been "removed" before. /// also, it should only happen if it has been saved before. - fn load_testcase(&self, testcase_id: &String) -> Result, Error> { + fn load_testcase( + self: &Rc, + testcase_id: &String, + ) -> Result>, Error> { let testcase_path = self.testcase_path(testcase_id); let testcase_md_path = self.testcase_md_path(testcase_id); let ser_fmt = self.md_format.clone(); - let _lockfile = TestcaseLockfile::new(self, testcase_id)?; + // let _lockfile = TestcaseLockfile::new(self, testcase_id)?; let input = I::from_file(testcase_path.as_path())?; let md = ser_fmt.from_file(testcase_md_path.as_path())?; - Ok(Testcase::new(input, md)) - } - - fn remove_testcase(&self, testcase_id: &String) -> Result<(), Error> { - let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; - - if lockfile.dec_used() { - fs::remove_file(self.testcase_path(testcase_id))?; - fs::remove_file(self.testcase_md_path(testcase_id))?; - } - - Ok(()) + Ok(Testcase::new( + Rc::new(input), + OnDiskTestcaseCell::new(self.clone(), md), + )) } } @@ -215,6 +204,8 @@ where I: Input, M: InMemoryCorpusMap, { + type TestcaseMetadataCell = OnDiskTestcaseCell; + fn count_all(&self) -> usize { self.count().saturating_add(self.count_disabled()) } @@ -231,63 +222,53 @@ where self.disabled_map.count() } - fn add(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(&testcase)?; + fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error> { + let testcase_id = self.disk_mgr.save_testcase(input, md)?; self.enabled_map.add(id, testcase_id); Ok(()) } - fn add_disabled(&mut self, id: CorpusId, testcase: Testcase) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(&testcase)?; + fn add_disabled( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result<(), Error> { + let testcase_id = self.disk_mgr.save_testcase(input, md)?; self.disabled_map.add(id, testcase_id); Ok(()) } - fn replace(&mut self, id: CorpusId, new_testcase: Testcase) -> Result, Error> { - let new_tc_id = self.disk_mgr.save_testcase(&new_testcase)?; - let old_tc_id = self.enabled_map.replace(id, new_tc_id).ok_or_else(|| { - Error::key_not_found(format!("Index {id} not found, could not replace.")) - })?; - - let old_tc = self.disk_mgr.load_testcase(&old_tc_id)?; - self.disk_mgr.remove_testcase(&old_tc_id)?; - Ok(old_tc) - } - - fn remove(&mut self, id: CorpusId) -> Result>>, Error> { - let old_tc_id = self - .enabled_map - .remove(id) - .or_else(|| self.disabled_map.remove(id)) - .ok_or(Error::key_not_found(format!("Index {id} not found")))?; - - let old_tc_id_borrowed = old_tc_id.borrow(); - let old_tc = self.disk_mgr.load_testcase(&old_tc_id_borrowed)?; - self.disk_mgr.remove_testcase(&old_tc_id_borrowed)?; - Ok(Rc::new(RefCell::new(old_tc))) - } - - fn get(&self, id: CorpusId) -> Result>>, Error> { - let tc_id = self - .enabled_map - .get(id) - .ok_or(Error::key_not_found(format!("Index not found: {id}")))?; + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + let tc_id = if ENABLED { + self.enabled_map + .get(id) + .ok_or_else(|| Error::key_not_found(format!("Index not found: {id}")))? + } else { + self.enabled_map + .get(id) + .or_else(|| self.disabled_map.get(id)) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))? + }; - let tc_id_borrowed = tc_id.borrow(); - let tc = self.disk_mgr.load_testcase(&*tc_id_borrowed)?; - Ok(Rc::new(RefCell::new(tc))) + self.disk_mgr.load_testcase(&tc_id) } - fn get_from_all(&self, id: CorpusId) -> Result>>, Error> { - let tc_id = self + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result, Error> { + let new_testcase_id = self.disk_mgr.save_testcase(input, md)?; + let old_testcase_id = self .enabled_map - .get(id) - .or_else(|| self.disabled_map.get(id)) - .ok_or(Error::key_not_found(format!("Index {id} not found")))?; - - let tc_id_borrowed = tc_id.borrow(); - let tc = self.disk_mgr.load_testcase(&*&tc_id_borrowed)?; - Ok(Rc::new(RefCell::new(tc))) + .replace(id, new_testcase_id) + .ok_or_else(|| Error::key_not_found(format!("Index not found: {id}")))?; + self.disk_mgr.load_testcase(&old_testcase_id) } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/testcase.rs b/crates/libafl/src/corpus/testcase.rs index 782ae1704d1..c9f9d5f234e 100644 --- a/crates/libafl/src/corpus/testcase.rs +++ b/crates/libafl/src/corpus/testcase.rs @@ -1,34 +1,204 @@ //! The [`Testcase`] is a struct embedded in each [`Corpus`]. //! It will contain a respective input, and metadata. -use alloc::string::String; #[cfg(feature = "track_hit_feedbacks")] use alloc::{borrow::Cow, vec::Vec}; +use alloc::{rc::Rc, string::String}; use core::{ - cell::{Ref, RefMut}, + fmt::{Debug, Formatter}, hash::Hasher, + ops::{Deref, DerefMut}, time::Duration, }; +use std::{ + cell::{Ref, RefCell, RefMut}, + marker::PhantomData, +}; -use libafl_bolts::{hasher_std, serdeany::SerdeAnyMap}; -use serde::{Deserialize, Serialize}; +use libafl_bolts::{ + HasLen, hasher_std, + serdeany::{SerdeAny, SerdeAnyMap}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use typed_builder::TypedBuilder; -use crate::{Error, HasMetadata, corpus::CorpusId, inputs::Input}; +use crate::{ + Error, HasMetadata, HasMetadataMut, + corpus::{Corpus, CorpusId}, + inputs::Input, + state::HasCorpus, +}; + +/// A testcase metadata cell that can be instantiated only from a [`TestcaseMetadata`]. +pub trait HasInstantiableTestcaseMetadata: HasTestcaseMetadata { + /// Instantiate a testcase metadata cell from a [`TestcaseMetadata`]. + fn instantiate(metadata: TestcaseMetadata) -> Self; +} + +/// Trait implemented by possible [`TestcaseMetadata`] cells. +pub trait HasTestcaseMetadata { + /// A reference to a testcase metadata. + type TestcaseMetadataRef<'a>: Deref + where + Self: 'a; + + /// A mutable reference to a testcase metadata. + type TestcaseMetadataRefMut<'a>: DerefMut + where + Self: 'a; + + /// Get a reference to the testcase metadata. + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a>; + /// Get a mutable reference to the testcase metadata. + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a>; + /// Consume the cell, and get the inner testcase metadata. + fn into_testcase_metadata(self) -> TestcaseMetadata; +} + +/// A dummy (empty) [`TestcaseMetadata`] reference. +#[derive(Default, Clone, Copy, Debug)] +pub struct NopTestcaseMetadataRef<'a>(PhantomData<&'a ()>); + +/// A dummy (empty) [`TestcaseMetadata`] cell. +#[derive(Default, Clone, Copy, Debug)] +pub struct NopTestcaseMetadataCell; + +impl<'a> Deref for NopTestcaseMetadataRef<'a> { + type Target = TestcaseMetadata; + + fn deref(&self) -> &Self::Target { + panic!("Invalid testcase metadata ref") + } +} + +impl<'a> DerefMut for NopTestcaseMetadataRef<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + panic!("Invalid testcase metadata ref mut") + } +} + +impl HasTestcaseMetadata for NopTestcaseMetadataCell { + type TestcaseMetadataRef<'a> = NopTestcaseMetadataRef<'a>; + type TestcaseMetadataRefMut<'a> = NopTestcaseMetadataRef<'a>; + + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + NopTestcaseMetadataRef::default() + } + + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + NopTestcaseMetadataRef::default() + } + + fn into_testcase_metadata(self) -> TestcaseMetadata { + panic!("Invalid testcase metadata") + } +} + +impl HasTestcaseMetadata for RefCell { + type TestcaseMetadataRef<'a> = Ref<'a, TestcaseMetadata>; + type TestcaseMetadataRefMut<'a> = RefMut<'a, TestcaseMetadata>; + + // fn new(md: TestcaseMetadata) -> Self { + // RefCell::new(md) + // } + + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + self.borrow() + } + + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + self.borrow_mut() + } + + fn into_testcase_metadata(self) -> TestcaseMetadata { + self.into_inner() + } +} + +impl HasInstantiableTestcaseMetadata for RefCell { + fn instantiate(metadata: TestcaseMetadata) -> Self { + RefCell::new(metadata) + } +} + +impl HasTestcaseMetadata for Rc +where + T: HasTestcaseMetadata + Clone, +{ + type TestcaseMetadataRef<'a> + = T::TestcaseMetadataRef<'a> + where + Self: 'a; + + type TestcaseMetadataRefMut<'a> + = T::TestcaseMetadataRefMut<'a> + where + Self: 'a; + + // fn new(md: TestcaseMetadata) -> Self { + // Rc::new(T::new(md)) + // } + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + self.deref().testcase_metadata() + } + + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + self.deref().testcase_metadata_mut() + } + + fn into_testcase_metadata(self) -> TestcaseMetadata { + self.deref().clone().into_testcase_metadata() + } +} + +impl HasInstantiableTestcaseMetadata for Rc +where + T: HasInstantiableTestcaseMetadata + Clone, +{ + fn instantiate(metadata: TestcaseMetadata) -> Self { + Rc::new(T::instantiate(metadata)) + } +} + +impl HasTestcaseMetadata for Testcase +where + M: HasTestcaseMetadata, +{ + type TestcaseMetadataRef<'a> + = M::TestcaseMetadataRef<'a> + where + Self: 'a; + type TestcaseMetadataRefMut<'a> + = M::TestcaseMetadataRefMut<'a> + where + Self: 'a; + + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + self.metadata.testcase_metadata() + } + + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + self.metadata.testcase_metadata_mut() + } + + fn into_testcase_metadata(self) -> TestcaseMetadata { + self.metadata.into_testcase_metadata() + } +} /// Shorthand to receive a [`Ref`] or [`RefMut`] to a stored [`Testcase`], by [`CorpusId`]. /// For a normal state, this should return a [`Testcase`] in the corpus, not the objectives. -pub trait HasTestcase { +pub trait HasTestcase: HasCorpus { /// Shorthand to receive a [`Ref`] to a stored [`Testcase`], by [`CorpusId`]. /// For a normal state, this should return a [`Testcase`] in the corpus, not the objectives. - fn testcase(&self, id: CorpusId) -> Result>, Error>; - - /// Shorthand to receive a [`RefMut`] to a stored [`Testcase`], by [`CorpusId`]. - /// For a normal state, this should return a [`Testcase`] in the corpus, not the objectives. - fn testcase_mut(&self, id: CorpusId) -> Result>, Error>; + fn testcase( + &self, + id: CorpusId, + ) -> Result>::TestcaseMetadataCell>, Error>; } -#[derive(Serialize, Deserialize, Clone, Debug, TypedBuilder)] +/// The [`Testcase`] metadata. +#[derive(Serialize, Deserialize, Clone, Debug, Default, TypedBuilder)] pub struct TestcaseMetadata { /// Map of metadata associated with this [`Testcase`] #[builder(default)] @@ -59,48 +229,112 @@ pub struct TestcaseMetadata { } /// An entry in the [`Testcase`] Corpus -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct Testcase { +pub struct Testcase { /// The [`Input`] of this [`Testcase`], or `None`, if it is not currently in memory - input: I, + input: Rc, + /// The unique id for [`Testcase`]. - /// It should uniquely identify input at the very least. + /// It should uniquely identify the input. id: String, + /// The metadata linked to the [`Testcase`] - metadata: TestcaseMetadata, - // /// The filename for this [`Testcase`] - // filename: Option, + pub(crate) metadata: M, } -impl HasMetadata for Testcase { - /// Get all the metadata into an [`hashbrown::HashMap`] - #[inline] +impl Clone for Testcase +where + M: Clone, +{ + fn clone(&self) -> Self { + Self { + input: self.input.clone(), + id: self.id.clone(), + metadata: self.metadata.clone(), + } + } +} + +impl Debug for Testcase +where + I: Debug, + M: HasTestcaseMetadata, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Testcase") + .field("input", self.input.as_ref()) + .field("id", &self.id) + .field("metadata", &*self.metadata.testcase_metadata()) + .finish() + } +} + +impl Serialize for Testcase +where + M: HasTestcaseMetadata, +{ + fn serialize(&self, _serializer: S) -> Result + where + S: Serializer, + { + todo!() + } +} + +impl<'de, I, M> Deserialize<'de> for Testcase +where + M: HasTestcaseMetadata, +{ + fn deserialize(_deserializer: D) -> Result + where + D: Deserializer<'de>, + { + todo!() + } +} + +impl HasMetadata for TestcaseMetadata { fn metadata_map(&self) -> &SerdeAnyMap { - &self.metadata.metadata + &self.metadata } +} - /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) - #[inline] +impl HasMetadataMut for TestcaseMetadata { fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { - &mut self.metadata.metadata + &mut self.metadata } } -impl Testcase +impl Testcase { + /// Get the input + #[inline] + pub fn input(&self) -> Rc { + self.input.clone() + } + + /// Get the associated unique ID. + pub fn id(&self) -> &String { + &self.id + } +} + +impl Testcase where - I: Input, + I: HasLen, { - fn gen_id(input: &I, _md: &TestcaseMetadata) -> String { - let mut hasher = hasher_std(); - input.hash(&mut hasher); - let hash = hasher.finish(); - format!("{hash:0>8x}") + /// Get the input length + pub fn input_len(&self) -> usize { + self.input.len() } +} +impl Testcase +where + I: Input, + M: HasTestcaseMetadata, +{ /// Create a new Testcase instance given an input - #[inline] - pub fn new(input: I, metadata: TestcaseMetadata) -> Self { - let id = Self::gen_id(&input, &metadata); + pub fn new(input: Rc, metadata: M) -> Self { + let id = Self::compute_id(&input); Self { input, @@ -108,174 +342,245 @@ where metadata, } } + + /// Get the unique ID associated to an input. + pub fn compute_id(input: &I) -> String { + let mut hasher = hasher_std(); + input.hash(&mut hasher); + let hash = hasher.finish(); + format!("{hash:0>8x}") + } +} + +impl Testcase +where + M: HasTestcaseMetadata, + I: Clone, +{ + /// Clone the input embedded in the [`Testcase`]. + pub fn cloned_input(&self) -> I { + self.input.as_ref().clone() + } } /// Impl of a testcase -impl Testcase { - pub fn id(&self) -> &String { - &self.id +impl Testcase +where + M: HasTestcaseMetadata, +{ + /// Get the same testcase, with an owned [`TestcaseMetadata`]. + pub fn cloned(self) -> Testcase> { + Testcase { + input: self.input, + id: self.id, + metadata: RefCell::new(self.metadata.into_testcase_metadata()), + } } - pub fn metadata(&self) -> &TestcaseMetadata { - &self.metadata + /// Decompose a [`Testcase`] into its inner input and metadata. + pub fn into_inner(self) -> (Rc, TestcaseMetadata) { + (self.input, self.metadata.into_testcase_metadata()) } - /// Get the input, if available any + /// Test whether the metadata map contains a metadata #[inline] - pub fn input(&self) -> &I { - &self.input + pub fn has_metadata(&self) -> bool + where + MT: SerdeAny, + { + self.metadata.testcase_metadata().has_metadata::() } - /// Get the input, if any (mutable) + /// Get the executions #[inline] - pub fn input_mut(&mut self) -> &mut I { - // self.cached_len = None; - &mut self.input + pub fn executions(&self) -> u64 { + self.metadata.testcase_metadata().executions() } - /// Set the input + /// Get the `scheduled_count` #[inline] - pub fn set_input(&mut self, input: I) { - self.input = input; + pub fn scheduled_count(&self) -> usize { + self.metadata.testcase_metadata().scheduled_count() } - /// Get the executions + /// Get `disabled` #[inline] - pub fn executions(&self) -> u64 { - self.metadata.executions + pub fn disabled(&mut self) -> bool { + self.metadata.testcase_metadata_mut().disabled() } - /// Get the executions (mutable) - #[inline] - pub fn executions_mut(&mut self) -> &mut u64 { - &mut self.metadata.executions + /// Get the id of the parent, that this testcase was derived from + #[must_use] + pub fn parent_id(&self) -> Option { + self.metadata.testcase_metadata().parent_id() + } + + /// Gets how many objectives were found by mutating this testcase + pub fn objectives_found(&self) -> usize { + self.metadata.testcase_metadata().objectives_found() } /// Set the executions #[inline] pub fn set_executions(&mut self, executions: u64) { - self.metadata.executions = executions; + self.metadata + .testcase_metadata_mut() + .set_executions(executions); } - /// Get the execution time of the testcase + /// Sets the execution time of the current testcase #[inline] - pub fn exec_time(&self) -> &Option { - &self.metadata.exec_time + pub fn set_exec_time(&mut self, time: Duration) { + self.metadata.testcase_metadata_mut().set_exec_time(time); } - /// Get the execution time of the testcase (mutable) + /// Set the `scheduled_count` #[inline] - pub fn exec_time_mut(&mut self) -> &mut Option { - &mut self.metadata.exec_time + pub fn set_scheduled_count(&mut self, scheduled_count: usize) { + self.metadata + .testcase_metadata_mut() + .set_scheduled_count(scheduled_count); } - /// Sets the execution time of the current testcase + /// Set the testcase as disabled #[inline] - pub fn set_exec_time(&mut self, time: Duration) { - self.metadata.exec_time = Some(time); + pub fn set_disabled(&mut self, disabled: bool) { + self.metadata.testcase_metadata_mut().set_disabled(disabled); } - /// Get the `scheduled_count` + /// Sets the id of the parent, that this testcase was derived from + pub fn set_parent_id(&mut self, parent_id: CorpusId) { + self.metadata + .testcase_metadata_mut() + .set_parent_id(parent_id); + } + + /// Sets the id of the parent, that this testcase was derived from + pub fn set_parent_id_optional(&mut self, parent_id: Option) { + self.metadata + .testcase_metadata_mut() + .set_parent_id_optional(parent_id); + } + + /// Adds one objective to the `objectives_found` counter. Mostly called from crash handler or executor. + pub fn found_objective(&mut self) { + self.metadata.testcase_metadata_mut().found_objective(); + } +} + +impl TestcaseMetadata { + /// Get the executions #[inline] - pub fn scheduled_count(&self) -> usize { - self.metadata.scheduled_count + pub fn executions(&self) -> u64 { + self.executions } - /// Set the `scheduled_count` + /// Get the execution time of the testcase #[inline] - pub fn set_scheduled_count(&mut self, scheduled_count: usize) { - self.metadata.scheduled_count = scheduled_count; + pub fn exec_time(&self) -> &Option { + &self.exec_time } - /// Get `disabled` + /// Get the `scheduled_count` #[inline] - pub fn disabled(&mut self) -> bool { - self.metadata.disabled + pub fn scheduled_count(&self) -> usize { + self.scheduled_count } - /// Set the testcase as disabled + /// Get `disabled` #[inline] - pub fn set_disabled(&mut self, disabled: bool) { - self.metadata.disabled = disabled; + pub fn disabled(&mut self) -> bool { + self.disabled } /// Get the hit feedbacks #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_feedbacks(&self) -> &Vec> { - &self.metadata.hit_feedbacks + &self.hit_feedbacks } - /// Get the hit feedbacks (mutable) + /// Get the hit objectives #[inline] #[cfg(feature = "track_hit_feedbacks")] - pub fn hit_feedbacks_mut(&mut self) -> &mut Vec> { - &mut self.metadata.hit_feedbacks + pub fn hit_objectives(&self) -> &Vec> { + &self.hit_objectives } - /// Get the hit objectives + /// Get the id of the parent, that this testcase was derived from + #[must_use] + pub fn parent_id(&self) -> Option { + self.parent_id + } + + /// Gets how many objectives were found by mutating this testcase + pub fn objectives_found(&self) -> usize { + self.objectives_found + } + + /// Get the executions (mutable) + #[inline] + pub fn executions_mut(&mut self) -> &mut u64 { + &mut self.executions + } + + /// Set the executions + #[inline] + pub fn set_executions(&mut self, executions: u64) { + self.executions = executions; + } + + /// Get a mutable reference to the execution time + pub fn exec_time_mut(&mut self) -> &mut Option { + &mut self.exec_time + } + + /// Sets the execution time of the current testcase + #[inline] + pub fn set_exec_time(&mut self, time: Duration) { + self.exec_time = Some(time); + } + + /// Set the `scheduled_count` + #[inline] + pub fn set_scheduled_count(&mut self, scheduled_count: usize) { + self.scheduled_count = scheduled_count; + } + + /// Set the testcase as disabled + #[inline] + pub fn set_disabled(&mut self, disabled: bool) { + self.disabled = disabled; + } + + /// Get the hit feedbacks (mutable) #[inline] #[cfg(feature = "track_hit_feedbacks")] - pub fn hit_objectives(&self) -> &Vec> { - &self.metadata.hit_objectives + pub fn hit_feedbacks_mut(&mut self) -> &mut Vec> { + &mut self.hit_feedbacks } /// Get the hit objectives (mutable) #[inline] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_objectives_mut(&mut self) -> &mut Vec> { - &mut self.metadata.hit_objectives - } - - // /// Creates a testcase, attaching the id of the parent - // /// that this [`Testcase`] was derived from on creation - // pub fn with_parent_id(input: I, parent_id: CorpusId) -> Self { - // Testcase { - // input: Some(input), - // filename: None, - // #[cfg(feature = "std")] - // file_path: None, - // metadata: SerdeAnyMap::default(), - // #[cfg(feature = "std")] - // metadata_path: None, - // exec_time: None, - // cached_len: None, - // executions: 0, - // scheduled_count: 0, - // parent_id: Some(parent_id), - // disabled: false, - // objectives_found: 0, - // #[cfg(feature = "track_hit_feedbacks")] - // hit_feedbacks: Vec::new(), - // #[cfg(feature = "track_hit_feedbacks")] - // hit_objectives: Vec::new(), - // } - // } - - /// Get the id of the parent, that this testcase was derived from - #[must_use] - pub fn parent_id(&self) -> Option { - self.metadata.parent_id + &mut self.hit_objectives } /// Sets the id of the parent, that this testcase was derived from pub fn set_parent_id(&mut self, parent_id: CorpusId) { - self.metadata.parent_id = Some(parent_id); + self.parent_id = Some(parent_id); } /// Sets the id of the parent, that this testcase was derived from pub fn set_parent_id_optional(&mut self, parent_id: Option) { - self.metadata.parent_id = parent_id; - } - - /// Gets how many objectives were found by mutating this testcase - pub fn objectives_found(&self) -> usize { - self.metadata.objectives_found + self.parent_id = parent_id; } /// Adds one objectives to the `objectives_found` counter. Mostly called from crash handler or executor. pub fn found_objective(&mut self) { - self.metadata.objectives_found = self.metadata.objectives_found.saturating_add(1); + let count = self.objectives_found.saturating_add(1); + self.objectives_found = count; } } diff --git a/crates/libafl/src/executors/inprocess/mod.rs b/crates/libafl/src/executors/inprocess/mod.rs index 59d59bbbd8d..89ad794470f 100644 --- a/crates/libafl/src/executors/inprocess/mod.rs +++ b/crates/libafl/src/executors/inprocess/mod.rs @@ -2,7 +2,7 @@ //! It should usually be paired with extra error-handling, such as a restarting event manager, to be effective. //! //! Needs the `fork` feature flag. -use alloc::boxed::Box; +use alloc::{boxed::Box, rc::Rc}; use core::{ borrow::BorrowMut, ffi::c_void, @@ -15,8 +15,9 @@ use core::{ use libafl_bolts::tuples::{RefIndexable, tuple_list}; use crate::{ - Error, HasMetadata, - corpus::{Corpus, Testcase, testcase::TestcaseMetadata}, + Error, + common::HasMetadataMut, + corpus::{Corpus, HasTestcaseMetadata, testcase::TestcaseMetadata}, events::{Event, EventFirer, EventRestarter, EventWithStats}, executors::{ Executor, ExitKind, HasObservers, @@ -344,25 +345,25 @@ pub fn run_observers_and_save_state( .expect("In run_observers_and_save_state objective failure."); if is_solution { - let testcase_md = TestcaseMetadata::builder() + let mut testcase_md = TestcaseMetadata::builder() .executions(*state.executions()) .parent_id(*state.corpus().current()) .build(); - let mut new_testcase = Testcase::new(input.clone(), testcase_md); - new_testcase.add_metadata(exitkind); + testcase_md.add_metadata(exitkind); - if let Ok(mut tc) = state.current_testcase_mut() { - tc.found_objective(); + if let Ok(tc) = state.current_testcase() { + tc.testcase_metadata_mut().found_objective(); } fuzzer .objective_mut() - .append_metadata(state, event_mgr, &*observers, &mut new_testcase) + .append_metadata(state, event_mgr, &*observers, &mut testcase_md) .expect("Failed adding metadata"); + state .solutions_mut() - .add(new_testcase) + .add(Rc::new(input.clone()), testcase_md) .expect("In run_observers_and_save_state solutions failure."); let event = Event::Objective { diff --git a/crates/libafl/src/feedbacks/bool.rs b/crates/libafl/src/feedbacks/bool.rs index 912f73e887b..e91d8966022 100644 --- a/crates/libafl/src/feedbacks/bool.rs +++ b/crates/libafl/src/feedbacks/bool.rs @@ -9,6 +9,7 @@ use libafl_bolts::{ use crate::{ HasNamedMetadata, + corpus::testcase::TestcaseMetadata, feedbacks::{Feedback, StateInitializer}, observers::{ObserversTuple, ValueObserver}, }; @@ -86,7 +87,7 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, - _testcase: &mut crate::corpus::Testcase, + _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) } diff --git a/crates/libafl/src/feedbacks/capture_feedback.rs b/crates/libafl/src/feedbacks/capture_feedback.rs index 44102131e55..133d1389253 100644 --- a/crates/libafl/src/feedbacks/capture_feedback.rs +++ b/crates/libafl/src/feedbacks/capture_feedback.rs @@ -6,8 +6,8 @@ use libafl_bolts::{Error, Named}; use serde::{Serialize, de::DeserializeOwned}; use crate::{ - HasMetadata, - corpus::Testcase, + HasMetadataMut, + corpus::testcase::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, stages::verify_timeouts::TimeoutsToVerify, @@ -39,7 +39,7 @@ impl StateInitializer for CaptureTimeoutFeedback {} impl Feedback for CaptureTimeoutFeedback where - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, I: Debug + Serialize + DeserializeOwned + Default + 'static + Clone, { #[inline] @@ -64,7 +64,7 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, - _testcase: &mut Testcase, + _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) } diff --git a/crates/libafl/src/feedbacks/concolic.rs b/crates/libafl/src/feedbacks/concolic.rs index 56b29e064b0..29be79df890 100644 --- a/crates/libafl/src/feedbacks/concolic.rs +++ b/crates/libafl/src/feedbacks/concolic.rs @@ -13,8 +13,8 @@ use libafl_bolts::{ }; use crate::{ - Error, HasMetadata, - corpus::Testcase, + Error, HasMetadataMut, + corpus::testcase::TestcaseMetadata, feedbacks::{Feedback, StateInitializer}, observers::concolic::ConcolicObserver, }; @@ -38,18 +38,15 @@ impl<'map> ConcolicFeedback<'map> { } } - fn add_concolic_feedback_to_metadata( - &mut self, - observers: &OT, - testcase: &mut Testcase, - ) where + fn add_concolic_feedback_to_metadata(&mut self, observers: &OT, md: &mut TestcaseMetadata) + where OT: MatchName, { if let Some(metadata) = observers .get(&self.observer_handle) .map(ConcolicObserver::create_metadata_from_current_map) { - testcase.metadata_map_mut().insert(metadata); + md.metadata_map_mut().insert(metadata); } } } @@ -76,9 +73,9 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.add_concolic_feedback_to_metadata(observers, testcase); + self.add_concolic_feedback_to_metadata::(observers, md); Ok(()) } } diff --git a/crates/libafl/src/feedbacks/list.rs b/crates/libafl/src/feedbacks/list.rs index a7eaf89816d..6aef553b01a 100644 --- a/crates/libafl/src/feedbacks/list.rs +++ b/crates/libafl/src/feedbacks/list.rs @@ -9,7 +9,8 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ - HasNamedMetadata, + HasNamedMetadataMut, + corpus::testcase::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, observers::ListObserver, @@ -81,7 +82,7 @@ where ) -> bool where OT: MatchName, - S: HasNamedMetadata, + S: HasNamedMetadataMut, { let observer = observers.get(&self.observer_handle).unwrap(); // TODO register the list content in a testcase metadata @@ -99,7 +100,7 @@ where !self.novelty.is_empty() } - fn append_list_observer_metadata(&mut self, state: &mut S) { + fn append_list_observer_metadata(&mut self, state: &mut S) { let history_set = state .named_metadata_map_mut() .get_mut::>(self.name()) @@ -113,7 +114,7 @@ where impl StateInitializer for ListFeedback where - S: HasNamedMetadata, + S: HasNamedMetadataMut, T: Debug + Eq + Hash + for<'a> Deserialize<'a> + Serialize + Default + Copy + 'static, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { @@ -125,7 +126,7 @@ where impl Feedback for ListFeedback where OT: MatchName, - S: HasNamedMetadata, + S: HasNamedMetadataMut, T: Debug + Eq + Hash + for<'a> Deserialize<'a> + Serialize + Default + Copy + 'static, { fn is_interesting( @@ -149,7 +150,7 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, - _testcase: &mut crate::corpus::Testcase, + _md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.append_list_observer_metadata(state); Ok(()) diff --git a/crates/libafl/src/feedbacks/map.rs b/crates/libafl/src/feedbacks/map.rs index 13caef28fa9..b3c51bd8dad 100644 --- a/crates/libafl/src/feedbacks/map.rs +++ b/crates/libafl/src/feedbacks/map.rs @@ -26,8 +26,8 @@ use super::simd::SimdMapFeedback; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - Error, HasMetadata, HasNamedMetadata, - corpus::Testcase, + Error, HasMetadataMut, HasNamedMetadataMut, + corpus::testcase::TestcaseMetadata, events::{Event, EventFirer, EventWithStats}, executors::ExitKind, feedbacks::{Feedback, HasObserverHandle, StateInitializer}, @@ -320,7 +320,7 @@ impl StateInitializer for MapFeedback where O: MapObserver, O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, - S: HasNamedMetadata, + S: HasNamedMetadataMut, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { // Initialize `MapFeedbackMetadata` with an empty vector and add it to the state. @@ -339,7 +339,7 @@ where O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, OT: MatchName, R: Reducer, - S: HasNamedMetadata + HasExecutions, + S: HasNamedMetadataMut + HasExecutions, { fn is_interesting( &mut self, @@ -368,11 +368,11 @@ where state: &mut S, manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { if let Some(novelties) = self.novelties.as_mut().map(core::mem::take) { let meta = MapNoveltiesMetadata::new(novelties); - testcase.add_metadata(meta); + md.add_metadata(meta); } let observer = observers.get(&self.map_ref).expect("MapObserver not found. This is likely because you entered the crash handler with the wrong executor/observer").as_ref(); let initial = observer.initial(); @@ -403,7 +403,7 @@ where indices.push(i); } let meta = MapIndexesMetadata::new(indices); - if testcase.try_add_metadata(meta).is_err() { + if md.try_add_metadata(meta).is_err() { return Err(Error::key_exists( "MapIndexesMetadata is already attached to this testcase. You should not have more than one observer with tracking.", )); @@ -531,7 +531,7 @@ where { fn is_interesting_default(&mut self, state: &mut S, observers: &OT) -> bool where - S: HasNamedMetadata, + S: HasNamedMetadataMut, OT: MatchName, { let mut interesting = false; diff --git a/crates/libafl/src/feedbacks/mod.rs b/crates/libafl/src/feedbacks/mod.rs index eed41a4a61d..dbad27e8b32 100644 --- a/crates/libafl/src/feedbacks/mod.rs +++ b/crates/libafl/src/feedbacks/mod.rs @@ -1,6 +1,5 @@ //! The feedbacks reduce observer state after each run to a single `is_interesting`-value. //! If a testcase is interesting, it may be added to a Corpus. - // TODO: make S of Feedback an associated type when specialisation + AT is stable use alloc::borrow::Cow; @@ -8,53 +7,60 @@ use alloc::borrow::Cow; use alloc::vec::Vec; use core::{fmt::Debug, marker::PhantomData}; -#[cfg(feature = "std")] -pub use concolic::ConcolicFeedback; -pub use differential::DiffFeedback; use libafl_bolts::{ Named, tuples::{Handle, Handled, MatchName, MatchNameRef}, }; -pub use list::*; -pub use map::*; -#[cfg(feature = "nautilus")] -pub use nautilus::*; -#[cfg(feature = "std")] -pub use new_hash_feedback::NewHashFeedback; -#[cfg(feature = "std")] -pub use new_hash_feedback::NewHashFeedbackMetadata; use serde::{Deserialize, Serialize}; -use crate::{Error, corpus::Testcase, executors::ExitKind, observers::TimeObserver}; +#[cfg(feature = "introspection")] +use crate::state::HasClientPerfMonitor; +use crate::{ + Error, corpus::testcase::TestcaseMetadata, executors::ExitKind, observers::TimeObserver, + state::HasCorpus, +}; #[cfg(feature = "std")] pub mod capture_feedback; +#[cfg(feature = "std")] +pub use capture_feedback::CaptureTimeoutFeedback; pub mod bool; pub use bool::BoolValueFeedback; #[cfg(feature = "std")] pub mod concolic; +#[cfg(feature = "std")] +pub use concolic::ConcolicFeedback; + pub mod differential; +pub use differential::DiffFeedback; + /// The module for list feedback pub mod list; +pub use list::*; + pub mod map; +pub use map::*; + #[cfg(feature = "nautilus")] pub mod nautilus; +#[cfg(feature = "nautilus")] +pub use nautilus::*; + #[cfg(feature = "std")] pub mod new_hash_feedback; +#[cfg(feature = "std")] +pub use new_hash_feedback::NewHashFeedback; +#[cfg(feature = "std")] +pub use new_hash_feedback::NewHashFeedbackMetadata; + #[cfg(feature = "simd")] pub mod simd; #[cfg(feature = "std")] pub mod stdio; pub mod transferred; -#[cfg(feature = "std")] -pub use capture_feedback::CaptureTimeoutFeedback; - -#[cfg(feature = "introspection")] -use crate::state::HasClientPerfMonitor; - #[cfg(feature = "value_bloom_feedback")] pub mod value_bloom; #[cfg(feature = "value_bloom_feedback")] @@ -143,7 +149,7 @@ pub trait Feedback: StateInitializer + Named { _state: &mut S, _manager: &mut EM, _observers: &OT, - _testcase: &mut Testcase, + _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) } @@ -292,12 +298,10 @@ where state: &mut S, manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.first - .append_metadata(state, manager, observers, testcase)?; - self.second - .append_metadata(state, manager, observers, testcase) + self.first.append_metadata(state, manager, observers, md)?; + self.second.append_metadata(state, manager, observers, md) } } @@ -652,10 +656,9 @@ where state: &mut S, manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.inner - .append_metadata(state, manager, observers, testcase) + self.inner.append_metadata(state, manager, observers, md) } } @@ -698,7 +701,7 @@ macro_rules! feedback_and { $crate::feedbacks::EagerAndFeedback::new($head , feedback_and!($($tail),+)) }; } -/// + /// Variadic macro to create a chain of (fast) [`AndFeedback`](FastAndFeedback) #[macro_export] macro_rules! feedback_and_fast { @@ -749,7 +752,10 @@ macro_rules! feedback_not { impl StateInitializer for () {} /// Hack to use () as empty Feedback -impl Feedback for () { +impl Feedback for () +where + S: HasCorpus, +{ #[cfg(feature = "track_hit_feedbacks")] fn last_result(&self) -> Result { Ok(false) @@ -919,7 +925,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { let Some(observer) = observers.get(&self.observer_handle) else { return Err(Error::illegal_state( @@ -927,7 +933,7 @@ where )); }; - *testcase.exec_time_mut() = *observer.last_runtime(); + *md.exec_time_mut() = *observer.last_runtime(); Ok(()) } } @@ -961,7 +967,10 @@ pub enum ConstFeedback { impl StateInitializer for ConstFeedback {} -impl Feedback for ConstFeedback { +impl Feedback for ConstFeedback +where + S: HasCorpus, +{ #[inline] fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/new_hash_feedback.rs b/crates/libafl/src/feedbacks/new_hash_feedback.rs index e7a8f825968..8e2afc3da54 100644 --- a/crates/libafl/src/feedbacks/new_hash_feedback.rs +++ b/crates/libafl/src/feedbacks/new_hash_feedback.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - Error, HasNamedMetadata, + Error, HasNamedMetadataMut, executors::ExitKind, feedbacks::{Feedback, HasObserverHandle, StateInitializer}, observers::ObserverWithHashField, @@ -103,7 +103,7 @@ impl NewHashFeedback where O: ObserverWithHashField + Named, { - fn has_interesting_backtrace_hash_observation( + fn has_interesting_backtrace_hash_observation( &mut self, state: &mut S, observers: &OT, @@ -137,7 +137,7 @@ where impl StateInitializer for NewHashFeedback where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { state.add_named_metadata_checked( @@ -152,7 +152,7 @@ impl Feedback for NewHashFeedback where O: ObserverWithHashField + Named, OT: MatchName, - S: HasNamedMetadata, + S: HasNamedMetadataMut, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/simd.rs b/crates/libafl/src/feedbacks/simd.rs index f892ccffb79..ef31517b0ee 100644 --- a/crates/libafl/src/feedbacks/simd.rs +++ b/crates/libafl/src/feedbacks/simd.rs @@ -20,8 +20,8 @@ use super::{DifferentIsNovel, Feedback, HasObserverHandle, MapFeedback, StateIni #[cfg(feature = "introspection")] use crate::state::HasClientPerfMonitor; use crate::{ - HasNamedMetadata, - corpus::Testcase, + HasNamedMetadataMut, + corpus::testcase::TestcaseMetadata, events::EventFirer, executors::ExitKind, feedbacks::MapFeedbackMetadata, @@ -49,7 +49,7 @@ where { fn is_interesting_u8_simd_optimized(&mut self, state: &mut S, observers: &OT) -> bool where - S: HasNamedMetadata, + S: HasNamedMetadataMut, OT: MatchName, { // TODO Replace with match_name_type when stable @@ -155,7 +155,7 @@ impl StateInitializer for SimdMapFeedback where O: MapObserver, O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, - S: HasNamedMetadata, + S: HasNamedMetadataMut, R: SimdReducer, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { @@ -192,7 +192,7 @@ where EM: EventFirer, O: MapObserver + for<'a> AsSlice<'a, Entry = u8> + for<'a> AsIter<'a, Item = u8>, OT: MatchName, - S: HasNamedMetadata + HasExecutions, + S: HasNamedMetadataMut + HasExecutions, R: SimdReducer, V: VectorType + Copy + Eq, R::PrimitiveReducer: Reducer, @@ -253,9 +253,8 @@ where state: &mut S, manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.map - .append_metadata(state, manager, observers, testcase) + self.map.append_metadata(state, manager, observers, md) } } diff --git a/crates/libafl/src/feedbacks/stdio.rs b/crates/libafl/src/feedbacks/stdio.rs index 9474175965e..41735391058 100644 --- a/crates/libafl/src/feedbacks/stdio.rs +++ b/crates/libafl/src/feedbacks/stdio.rs @@ -9,8 +9,8 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, - corpus::Testcase, + Error, HasMetadataMut, + corpus::testcase::TestcaseMetadata, feedbacks::{Feedback, StateInitializer}, observers::{StdErrObserver, StdOutObserver}, }; @@ -33,10 +33,10 @@ pub struct StdOutToMetadataFeedback { impl StdOutToMetadataFeedback { /// Append to the testcase the generated metadata in case of a new corpus item. #[inline] - fn append_stdout_observation_to_testcase( + fn append_stdout_observation_to_testcase( &mut self, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> where OT: MatchName, @@ -50,9 +50,7 @@ impl StdOutToMetadataFeedback { .ok_or(Error::illegal_state("StdOutObserver has no stdout"))?; let stdout = String::from_utf8_lossy(buffer).into_owned(); - testcase - .metadata_map_mut() - .insert(StdOutMetadata { stdout }); + md.metadata_map_mut().insert(StdOutMetadata { stdout }); Ok(()) } @@ -76,9 +74,9 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.append_stdout_observation_to_testcase(observers, testcase) + self.append_stdout_observation_to_testcase(observers, md) } } @@ -132,7 +130,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { let observer = observers .get(&self.o_ref) @@ -143,9 +141,7 @@ where .ok_or(Error::illegal_state("StdErrObserver has no stderr"))?; let stderr = String::from_utf8_lossy(buffer).into_owned(); - testcase - .metadata_map_mut() - .insert(StdErrMetadata { stderr }); + md.metadata_map_mut().insert(StdErrMetadata { stderr }); Ok(()) } diff --git a/crates/libafl/src/feedbacks/transferred.rs b/crates/libafl/src/feedbacks/transferred.rs index a1259faeb3d..ed801306bed 100644 --- a/crates/libafl/src/feedbacks/transferred.rs +++ b/crates/libafl/src/feedbacks/transferred.rs @@ -9,9 +9,10 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - HasMetadata, + HasMetadataMut, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, + state::HasCorpus, }; /// Constant name of the [`TransferringMetadata`]. @@ -55,7 +56,7 @@ impl Named for TransferredFeedback { impl StateInitializer for TransferredFeedback where - S: HasMetadata, + S: HasMetadataMut, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { state.add_metadata(TransferringMetadata { transferring: true }); @@ -65,7 +66,7 @@ where impl Feedback for TransferredFeedback where - S: HasMetadata, + S: HasCorpus + HasMetadataMut, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/fuzzer/mod.rs b/crates/libafl/src/fuzzer/mod.rs index 8fb9f04b719..161bed6ce56 100644 --- a/crates/libafl/src/fuzzer/mod.rs +++ b/crates/libafl/src/fuzzer/mod.rs @@ -1,6 +1,6 @@ //! The `Fuzzer` is the main struct for a fuzz campaign. -use alloc::{string::ToString, vec::Vec}; +use alloc::{rc::Rc, string::ToString, vec::Vec}; #[cfg(feature = "std")] use core::hash::Hash; use core::{fmt::Debug, time::Duration}; @@ -13,9 +13,10 @@ use serde::{Serialize, de::DeserializeOwned}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, + Error, HasMetadata, HasMetadataMut, corpus::{ - Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, Testcase, testcase::TestcaseMetadata, + Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, + testcase::{HasTestcaseMetadata, TestcaseMetadata}, }, events::{ Event, EventConfig, EventFirer, EventReceiver, EventWithStats, ProgressReporter, @@ -451,19 +452,17 @@ where ) -> Result, Error> { let corpus = if exec_res.is_corpus() { // Add the input to the main corpus - let tc_md = TestcaseMetadata::builder() + let mut tc_md = TestcaseMetadata::builder() .executions(*state.executions()) .build(); - let testcase = Testcase::new(input.clone(), tc_md); - #[cfg(feature = "track_hit_feedbacks")] self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + .append_hit_feedbacks(tc_md.hit_feedbacks_mut())?; self.feedback_mut() - .append_metadata(state, manager, observers, &mut testcase)?; + .append_metadata(state, manager, observers, &mut tc_md)?; - let id = state.corpus_mut().add(testcase)?; + let id = state.corpus_mut().add(Rc::new(input.clone()), tc_md)?; self.scheduler_mut().on_add(state, id)?; Ok(Some(id)) } else { @@ -472,23 +471,22 @@ where if exec_res.is_solution() { // The input is a solution, add it to the respective corpus - let tc_md = TestcaseMetadata::builder() + let mut tc_md = TestcaseMetadata::builder() .executions(*state.executions()) .parent_id(*state.corpus().current()) .build(); - let mut testcase = Testcase::new(input.clone(), tc_md); - testcase.add_metadata(*exit_kind); + tc_md.add_metadata(*exit_kind); - if let Ok(mut tc) = state.current_testcase_mut() { - tc.found_objective(); + if let Ok(tc) = state.current_testcase() { + tc.testcase_metadata_mut().found_objective(); } #[cfg(feature = "track_hit_feedbacks")] self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; + .append_hit_feedbacks(tc_md.hit_objectives_mut())?; self.objective_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - state.solutions_mut().add(testcase)?; + .append_metadata(state, manager, observers, &mut tc_md)?; + state.solutions_mut().add(Rc::new(input.clone()), tc_md)?; } corpus } @@ -729,10 +727,9 @@ where let exit_kind = self.execute_input(state, executor, manager, &input)?; let observers = executor.observers(); // Always consider this to be "interesting" - let tc_md = TestcaseMetadata::builder() + let mut tc_md = TestcaseMetadata::builder() .executions(*state.executions()) .build(); - let mut testcase = Testcase::new(input.clone(), tc_md); // Maybe a solution #[cfg(not(feature = "introspection"))] @@ -752,11 +749,13 @@ where if is_solution { #[cfg(feature = "track_hit_feedbacks")] self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; + .append_hit_feedbacks(tc_md.hit_objectives_mut())?; self.objective_mut() - .append_metadata(state, manager, &*observers, &mut testcase)?; + .append_metadata(state, manager, &*observers, &mut tc_md)?; // we don't care about solution id - let _ = state.solutions_mut().add(testcase.clone())?; + let _ = state + .solutions_mut() + .add(Rc::new(input.clone()), tc_md.clone())?; manager.fire( state, @@ -791,8 +790,8 @@ where .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; // Add the input to the main corpus self.feedback_mut() - .append_metadata(state, manager, &*observers, &mut testcase)?; - let id = state.corpus_mut().add(testcase)?; + .append_metadata(state, manager, &*observers, &mut tc_md)?; + let id = state.corpus_mut().add(Rc::new(input.clone()), tc_md)?; self.scheduler_mut().on_add(state, id)?; let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { @@ -825,10 +824,10 @@ where .disabled(true) .build(); - let mut testcase = Testcase::new(input.clone(), tc_md); - // Add the disabled input to the main corpus - let id = state.corpus_mut().add_disabled(testcase)?; + let id = state + .corpus_mut() + .add_disabled(Rc::new(input.clone()), tc_md)?; Ok(id) } } @@ -981,10 +980,12 @@ where state.introspection_stats_mut().mark_manager_time(); { - if let Ok(mut testcase) = state.testcase_mut(id) { - let scheduled_count = testcase.scheduled_count(); + if let Ok(testcase) = state.testcase(id) { + let mut md = testcase.testcase_metadata_mut(); + + let scheduled_count = md.scheduled_count(); // increase scheduled count, this was fuzz_level in afl - testcase.set_scheduled_count(scheduled_count + 1); + md.set_scheduled_count(scheduled_count + 1); } } diff --git a/crates/libafl/src/inputs/generalized.rs b/crates/libafl/src/inputs/generalized.rs index e3b2d8f4445..57f5f1f7f40 100644 --- a/crates/libafl/src/inputs/generalized.rs +++ b/crates/libafl/src/inputs/generalized.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadata, - corpus::Testcase, + corpus::{Testcase, testcase::HasTestcaseMetadata}, inputs::BytesInput, stages::mutational::{MutatedTransform, MutatedTransformPost}, }; @@ -107,8 +107,12 @@ impl GeneralizedInputMetadata { impl MutatedTransform for GeneralizedInputMetadata { type Post = Self; - fn try_transform_from(base: &mut Testcase, _state: &S) -> Result { + fn try_transform_from( + base: &Testcase, + _state: &S, + ) -> Result { let meta = base + .testcase_metadata() .metadata_map() .get::() .ok_or_else(|| { diff --git a/crates/libafl/src/mutators/encoded_mutations.rs b/crates/libafl/src/mutators/encoded_mutations.rs index a27f7559643..1e0c94ce290 100644 --- a/crates/libafl/src/mutators/encoded_mutations.rs +++ b/crates/libafl/src/mutators/encoded_mutations.rs @@ -387,7 +387,7 @@ where let other_size = { // new scope to make the borrow checker happy - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; other_testcase.input().codes().len() }; @@ -416,7 +416,7 @@ where } } - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // no need to `load_input` again - we did that above already. let other = other_testcase.input(); @@ -474,7 +474,7 @@ where let other_size = { // new scope to make the borrow checker happy - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; other_testcase.input().codes().len() }; @@ -499,7 +499,7 @@ where .rand_mut() .below(unsafe { NonZero::new(size - len).unwrap_unchecked() }); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // no need to load the input again, it'll already be present at this point. let other = other_testcase.input(); diff --git a/crates/libafl/src/mutators/gramatron.rs b/crates/libafl/src/mutators/gramatron.rs index 5015fd2a434..97943b74fc7 100644 --- a/crates/libafl/src/mutators/gramatron.rs +++ b/crates/libafl/src/mutators/gramatron.rs @@ -12,8 +12,8 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, - corpus::Corpus, + Error, HasMetadata, HasMetadataMut, + corpus::{Corpus, HasTestcaseMetadata}, generators::GramatronGenerator, inputs::{GramatronInput, Terminal}, mutators::{MutationResult, Mutator}, @@ -119,7 +119,7 @@ pub struct GramatronSpliceMutator; impl Mutator for GramatronSpliceMutator where - S: HasRand + HasCorpus + HasMetadata, + S: HasRand + HasCorpus + HasMetadataMut, { fn mutate( &mut self, @@ -136,14 +136,14 @@ where let rand_num = state.rand_mut().next(); - let other_testcase_ref = state.corpus().get(id)?; - let mut other_testcase = other_testcase_ref.borrow_mut(); + let other_testcase = state.corpus().get(id)?; + let mut other_md = other_testcase.testcase_metadata_mut(); - if !other_testcase.has_metadata::() { - let meta = GramatronIdxMapMetadata::new(other_testcase.input()); - other_testcase.add_metadata(meta); + if !other_md.has_metadata::() { + let meta = GramatronIdxMapMetadata::new(other_testcase.input().as_ref()); + other_md.add_metadata(meta); } - let meta = other_testcase + let meta = other_md .metadata_map() .get::() .unwrap(); diff --git a/crates/libafl/src/mutators/grimoire.rs b/crates/libafl/src/mutators/grimoire.rs index 91143d7a5d0..455794d9021 100644 --- a/crates/libafl/src/mutators/grimoire.rs +++ b/crates/libafl/src/mutators/grimoire.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use crate::{ Error, HasMetadata, - corpus::Corpus, + corpus::{Corpus, HasTestcaseMetadata}, inputs::{GeneralizedInputMetadata, GeneralizedItem}, mutators::{MutationResult, Mutator, token_mutations::Tokens}, random_corpus_id, @@ -41,8 +41,9 @@ where let rand1 = state.rand_mut().next(); let rand2 = state.rand_mut().next(); - let other_testcase = state.corpus().get(id)?.borrow(); + let other_testcase = state.corpus().get(id)?; if let Some(other) = other_testcase + .testcase_metadata() .metadata_map() .get::() { @@ -92,8 +93,9 @@ where } } - let other_testcase = state.corpus().get(id)?.borrow(); + let other_testcase = state.corpus().get(id)?; match other_testcase + .testcase_metadata() .metadata_map() .get::() { diff --git a/crates/libafl/src/mutators/mopt_mutator.rs b/crates/libafl/src/mutators/mopt_mutator.rs index 897f1a23d89..febd164a3be 100644 --- a/crates/libafl/src/mutators/mopt_mutator.rs +++ b/crates/libafl/src/mutators/mopt_mutator.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use super::MutationId; use crate::{ - Error, HasMetadata, + Error, HasMetadataMut, corpus::{Corpus, CorpusId}, mutators::{ComposedByMutations, MutationResult, Mutator, MutatorsTuple, ScheduledMutator}, state::{HasCorpus, HasRand, HasSolutions}, @@ -371,7 +371,7 @@ pub struct StdMOptMutator { impl Mutator for StdMOptMutator where MT: MutatorsTuple, - S: HasRand + HasMetadata + HasCorpus + HasSolutions, + S: HasRand + HasMetadataMut + HasCorpus + HasSolutions, { #[inline] fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { @@ -501,7 +501,7 @@ impl StdMOptMutator { swarm_num: usize, ) -> Result where - S: HasMetadata + HasRand, + S: HasMetadataMut + HasRand, MT: NamedTuple, { if !state.has_metadata::() { @@ -519,7 +519,7 @@ impl StdMOptMutator { } fn core_mutate(&mut self, state: &mut S, input: &mut I) -> Result where - S: HasMetadata + HasRand + HasSolutions + HasCorpus, + S: HasMetadataMut + HasRand + HasSolutions + HasCorpus, MT: MutatorsTuple, { let mut r = MutationResult::Skipped; @@ -546,7 +546,7 @@ impl StdMOptMutator { fn pilot_mutate(&mut self, state: &mut S, input: &mut I) -> Result where - S: HasMetadata + HasRand + HasSolutions + HasCorpus, + S: HasMetadataMut + HasRand + HasSolutions + HasCorpus, MT: MutatorsTuple, { let mut r = MutationResult::Skipped; @@ -604,7 +604,7 @@ impl Named for StdMOptMutator { impl ScheduledMutator for StdMOptMutator where MT: MutatorsTuple, - S: HasRand + HasMetadata + HasCorpus + HasSolutions, + S: HasRand + HasMetadataMut + HasCorpus + HasSolutions, { /// Compute the number of iterations used to apply stacked mutations fn iterations(&self, state: &mut S, _: &I) -> u64 { diff --git a/crates/libafl/src/mutators/mutations.rs b/crates/libafl/src/mutators/mutations.rs index 39f38adb7e7..c4456b8a462 100644 --- a/crates/libafl/src/mutators/mutations.rs +++ b/crates/libafl/src/mutators/mutations.rs @@ -1302,8 +1302,8 @@ where } let other_size = { - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); - other_testcase.input_mut().mutator_bytes().len() + let other_testcase = state.corpus().get_from_all(id)?; + other_testcase.input().len() }; if other_size < 2 { @@ -1318,7 +1318,7 @@ where }); let target = state.rand_mut().below(nonzero_size); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // No need to load the input again, it'll still be cached. let other = other_testcase.input(); @@ -1401,10 +1401,7 @@ where } } - let other_size = { - let mut testcase = state.corpus().get_from_all(id)?.borrow_mut(); - testcase.input_mut().mutator_bytes().len() - }; + let other_size = state.corpus().get_from_all(id)?.input().len(); if other_size < 2 { return Ok(MutationResult::Skipped); @@ -1422,7 +1419,7 @@ where NonZero::new(min(other_size, size - target)).unwrap_unchecked() }); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // No need to load the input again, it'll still be cached. let other = other_testcase.input(); @@ -1515,9 +1512,9 @@ where } let other_size = { - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; let other_input = other_testcase.input(); - let input_mapped = (self.input_mapper)(other_input).map_to_option_bytes(); + let input_mapped = (self.input_mapper)(other_input.as_ref()).map_to_option_bytes(); input_mapped.map_or(0, >::len) }; @@ -1537,7 +1534,7 @@ where .rand_mut() .below(unsafe { NonZero::new(size).unwrap_unchecked() }); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // No need to load the input again, it'll still be cached. let other_input = &mut other_testcase.input(); let wrapped_mapped_other_input = (self.input_mapper)(other_input).map_to_option_bytes(); @@ -1610,9 +1607,9 @@ where } let other_size = { - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; let other_input = other_testcase.input(); - let input_mapped = (self.input_mapper)(other_input).map_to_option_bytes(); + let input_mapped = (self.input_mapper)(other_input.as_ref()).map_to_option_bytes(); input_mapped.map_or(0, >::len) }; @@ -1632,10 +1629,11 @@ where NonZero::new(min(other_size, size - target)).unwrap_unchecked() }); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // No need to load the input again, it'll still be cached. let other_input = &mut other_testcase.input(); - let wrapped_mapped_other_input = (self.input_mapper)(other_input).map_to_option_bytes(); + let wrapped_mapped_other_input = + (self.input_mapper)(other_input.as_ref()).map_to_option_bytes(); if wrapped_mapped_other_input.is_none() { return Ok(MutationResult::Skipped); } @@ -1701,7 +1699,7 @@ where } let (first_diff, last_diff) = { - let mut other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; let other = other_testcase.input(); let (f, l) = locate_diffs(input.mutator_bytes(), other.mutator_bytes()); @@ -1715,7 +1713,7 @@ where let split_at = state.rand_mut().between(first_diff, last_diff); - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; // Input will already be loaded. let other = other_testcase.input(); diff --git a/crates/libafl/src/mutators/numeric.rs b/crates/libafl/src/mutators/numeric.rs index 1b4e6b8bcc4..3d381dcbc0b 100644 --- a/crates/libafl/src/mutators/numeric.rs +++ b/crates/libafl/src/mutators/numeric.rs @@ -426,7 +426,7 @@ where return Ok(MutationResult::Skipped); } - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; *input = *other_testcase.input(); Ok(MutationResult::Mutated) } @@ -475,9 +475,9 @@ where return Ok(MutationResult::Skipped); } - let other_testcase = state.corpus().get_from_all(id)?.borrow_mut(); + let other_testcase = state.corpus().get_from_all(id)?; let other_input = other_testcase.input(); - let mapped_input = (self.input_mapper)(other_input).clone(); + let mapped_input = (self.input_mapper)(other_input.as_ref()).clone(); *input = mapped_input; Ok(MutationResult::Mutated) } diff --git a/crates/libafl/src/mutators/scheduled.rs b/crates/libafl/src/mutators/scheduled.rs index 677124cdda1..b16f47d0ef7 100644 --- a/crates/libafl/src/mutators/scheduled.rs +++ b/crates/libafl/src/mutators/scheduled.rs @@ -16,8 +16,8 @@ use serde::{Deserialize, Serialize}; use super::MutationId; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId}, + Error, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcaseMetadata}, mutators::{ MutationResult, Mutator, MutatorsTuple, token_mutations::{TokenInsert, TokenReplace}, @@ -306,15 +306,18 @@ where fn post_exec(&mut self, state: &mut S, corpus_id: Option) -> Result<(), Error> { if let Some(id) = corpus_id { - let mut testcase = (*state.corpus_mut().get(id)?).borrow_mut(); + let testcase = state.corpus().get(id)?; let mut log = Vec::>::new(); + while let Some(idx) = self.mutation_log.pop() { let name = self.scheduled.mutations().name(idx.0).unwrap().clone(); // TODO maybe return an Error on None log.push(name); } + let meta = LogMutationMetadata::new(log); - testcase.add_metadata(meta); + testcase.testcase_metadata_mut().add_metadata(meta); } + // Always reset the log for each run self.mutation_log.clear(); Ok(()) diff --git a/crates/libafl/src/mutators/token_mutations.rs b/crates/libafl/src/mutators/token_mutations.rs index 36aabcb1723..a9c47a69ea6 100644 --- a/crates/libafl/src/mutators/token_mutations.rs +++ b/crates/libafl/src/mutators/token_mutations.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use crate::mutators::str_decode; use crate::{ - Error, HasMetadata, + Error, HasMetadata, HasMetadataMut, corpus::{CorpusId, HasCurrentCorpusId}, inputs::{HasMutatorBytes, ResizableMutator}, mutators::{ @@ -1325,7 +1325,7 @@ impl AflppRedQueen { impl MultiMutator for AflppRedQueen where - S: HasMetadata + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusId, + S: HasMetadataMut + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusId, I: ResizableMutator + From> + HasMutatorBytes, { #[expect(clippy::needless_range_loop, clippy::too_many_lines)] diff --git a/crates/libafl/src/mutators/tuneable.rs b/crates/libafl/src/mutators/tuneable.rs index 10e66443092..4af018ce840 100644 --- a/crates/libafl/src/mutators/tuneable.rs +++ b/crates/libafl/src/mutators/tuneable.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; pub use crate::mutators::{mutations::*, token_mutations::*}; use crate::{ - Error, HasMetadata, + Error, HasMetadata, HasMetadataMut, mutators::{ ComposedByMutations, MutationId, MutationResult, Mutator, MutatorsTuple, ScheduledMutator, }, @@ -69,7 +69,7 @@ impl TuneableScheduledMutatorMetadata { } /// Gets the stored metadata, used to alter the [`TuneableScheduledMutator`] behavior, mut - pub fn get_mut(state: &mut S) -> Result<&mut Self, Error> { + pub fn get_mut(state: &mut S) -> Result<&mut Self, Error> { state .metadata_map_mut() .get_mut::() @@ -89,7 +89,7 @@ pub struct TuneableScheduledMutator { impl Mutator for TuneableScheduledMutator where MT: MutatorsTuple, - S: HasRand + HasMetadata, + S: HasRand + HasMetadataMut, { #[inline] fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { @@ -129,7 +129,7 @@ impl Named for TuneableScheduledMutator { impl ScheduledMutator for TuneableScheduledMutator where MT: MutatorsTuple, - S: HasRand + HasMetadata, + S: HasRand + HasMetadataMut, { /// Compute the number of iterations used to apply stacked mutations fn iterations(&self, state: &mut S, _: &I) -> u64 { @@ -212,7 +212,7 @@ impl TuneableScheduledMutator { pub fn new(state: &mut S, mutations: MT) -> Self where MT: NamedTuple, - S: HasRand + HasMetadata, + S: HasRand + HasMetadataMut, { if !state.has_metadata::() { state.add_metadata(TuneableScheduledMutatorMetadata::default()); @@ -233,7 +233,7 @@ impl TuneableScheduledMutator { /// as it internally only needs a single metadata lookup pub fn set_iters(&self, state: &mut S, iters: u64) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); @@ -256,7 +256,7 @@ impl TuneableScheduledMutator { mut iter_probabilities_pow: Vec, ) -> Result<(), Error> where - S: HasMetadata, + S: HasMetadataMut, { if iter_probabilities_pow.len() >= 32 { return Err(Error::illegal_argument( @@ -276,7 +276,7 @@ impl TuneableScheduledMutator { /// Gets the set amount of iterations pub fn get_iters(&self, state: &S) -> Option where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get(state).unwrap(); metadata.iters @@ -285,7 +285,7 @@ impl TuneableScheduledMutator { /// Sets the mutation ids pub fn set_mutation_ids(&self, state: &mut S, mutations: Vec) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids = mutations; @@ -302,7 +302,7 @@ impl TuneableScheduledMutator { mut mutation_probabilities: Vec, ) -> Result<(), Error> where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids.clear(); @@ -321,7 +321,7 @@ impl TuneableScheduledMutator { mutations: Vec, iters: u64, ) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids = mutations; @@ -332,7 +332,7 @@ impl TuneableScheduledMutator { /// Appends a mutation id to the end of the mutations pub fn push_mutation_id(state: &mut S, mutation_id: MutationId) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids.push(mutation_id); @@ -341,7 +341,7 @@ impl TuneableScheduledMutator { /// Resets this to a randomic mutational stage pub fn reset(self, state: &mut S) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = state .metadata_map_mut() diff --git a/crates/libafl/src/mutators/unicode/mod.rs b/crates/libafl/src/mutators/unicode/mod.rs index 618acab4094..ec27f3d8d19 100644 --- a/crates/libafl/src/mutators/unicode/mod.rs +++ b/crates/libafl/src/mutators/unicode/mod.rs @@ -35,7 +35,7 @@ where { type Post = UnicodeIdentificationMetadata; - fn try_transform_from(base: &mut Testcase, state: &S) -> Result { + fn try_transform_from(base: &Testcase, state: &S) -> Result { let input = base.load_input(state.corpus())?.clone(); let metadata = base.metadata::().cloned()?; Ok((input, metadata)) diff --git a/crates/libafl/src/observers/cmp.rs b/crates/libafl/src/observers/cmp.rs index f753f1b4a4a..d941c885e0e 100644 --- a/crates/libafl/src/observers/cmp.rs +++ b/crates/libafl/src/observers/cmp.rs @@ -11,7 +11,7 @@ use hashbrown::HashMap; use libafl_bolts::{AsSlice, HasLen, Named, ownedref::OwnedRefMut}; use serde::{Deserialize, Serialize}; -use crate::{Error, HasMetadata, executors::ExitKind, observers::Observer}; +use crate::{Error, HasMetadataMut, executors::ExitKind, observers::Observer}; /// A bytes string for cmplog with up to 32 elements. #[derive(Debug, Copy, Clone, Serialize, Deserialize, Eq, PartialEq)] @@ -253,7 +253,7 @@ where impl Observer for StdCmpObserver<'_, CM> where CM: Serialize + CmpMap + HasLen, - S: HasMetadata, + S: HasMetadataMut, { fn pre_exec(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { self.cmp_map.as_mut().reset()?; diff --git a/crates/libafl/src/schedulers/accounting.rs b/crates/libafl/src/schedulers/accounting.rs index 87ba1c0084d..bfbd705b742 100644 --- a/crates/libafl/src/schedulers/accounting.rs +++ b/crates/libafl/src/schedulers/accounting.rs @@ -12,8 +12,8 @@ use serde::{Deserialize, Serialize}; use super::IndexesLenTimeMinimizerScheduler; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId}, + Error, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcaseMetadata}, observers::CanTrack, schedulers::{ Scheduler, @@ -113,7 +113,7 @@ pub struct CoverageAccountingScheduler<'a, CS, I, O> { impl Scheduler for CoverageAccountingScheduler<'_, CS, I, O> where CS: Scheduler, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, I: HasLen, O: CanTrack, { @@ -140,13 +140,8 @@ where self.inner.cull(state)?; } let mut id = self.inner.base_mut().next(state)?; - while { - !state - .corpus() - .get(id)? - .borrow() - .has_metadata::() - } && state.rand_mut().coinflip(self.skip_non_favored_prob) + while { !state.corpus().get(id)?.has_metadata::() } + && state.rand_mut().coinflip(self.skip_non_favored_prob) { id = self.inner.base_mut().next(state)?; } @@ -175,7 +170,7 @@ where #[expect(clippy::cast_possible_wrap)] pub fn update_accounting_score(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, { let mut indexes = vec![]; let mut new_favoreds = vec![]; @@ -199,9 +194,10 @@ where equal_score = true; } - let mut old = state.corpus().get_from_all(*old_id)?.borrow_mut(); + let old = state.corpus().get_from_all(*old_id)?; + let mut md = old.testcase_metadata_mut(); let must_remove = { - let old_meta = old.metadata_map_mut().get_mut::().ok_or_else(|| { + let old_meta = md.metadata_map_mut().get_mut::().ok_or_else(|| { Error::key_not_found(format!( "AccountingIndexesMetadata, needed by CoverageAccountingScheduler, not found in testcase #{old_id}" )) @@ -211,7 +207,7 @@ where }; if must_remove { - drop(old.metadata_map_mut().remove::()); + drop(md.metadata_map_mut().remove::()); } } } @@ -239,7 +235,7 @@ where state .corpus() .get(id)? - .borrow_mut() + .testcase_metadata_mut() .metadata_map_mut() .insert(AccountingIndexesMetadata::with_tcref( indexes, @@ -262,19 +258,21 @@ where /// Cull the `Corpus` pub fn accounting_cull(&self, state: &S) -> Result<(), Error> where - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, { let Some(top_rated) = state.metadata_map().get::() else { return Ok(()); }; for (_key, id) in &top_rated.map { - let mut entry = state.corpus().get(*id)?.borrow_mut(); + let entry = state.corpus().get(*id)?; if entry.scheduled_count() > 0 { continue; } - entry.add_metadata(IsFavoredMetadata {}); + entry + .testcase_metadata_mut() + .add_metadata(IsFavoredMetadata {}); } Ok(()) @@ -286,7 +284,7 @@ where /// Provide the observer responsible for determining new indexes. pub fn new(observer: &O, state: &mut S, base: CS, accounting_map: &'a [u32]) -> Self where - S: HasMetadata, + S: HasMetadataMut, { match state.metadata_map().get::() { Some(meta) => { @@ -317,7 +315,7 @@ where accounting_map: &'a [u32], ) -> Self where - S: HasMetadata, + S: HasMetadataMut, { match state.metadata_map().get::() { Some(meta) => { diff --git a/crates/libafl/src/schedulers/minimizer.rs b/crates/libafl/src/schedulers/minimizer.rs index bb430cbee6e..dbc1718e446 100644 --- a/crates/libafl/src/schedulers/minimizer.rs +++ b/crates/libafl/src/schedulers/minimizer.rs @@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize}; use super::HasQueueCycles; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, Testcase}, + Error, HasMetadata, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcaseMetadata, Testcase}, feedbacks::MapIndexesMetadata, observers::CanTrack, require_index_tracking, @@ -84,25 +84,14 @@ where CS: RemovableScheduler + Scheduler, F: TestcaseScore, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, { - /// Replaces the [`Testcase`] at the given [`CorpusId`] - fn on_replace( - &mut self, - state: &mut S, - id: CorpusId, - testcase: &Testcase, - ) -> Result<(), Error> { - self.base.on_replace(state, id, testcase)?; - self.update_score(state, id) - } - /// Removes an entry from the corpus fn on_remove( &mut self, state: &mut S, id: CorpusId, - testcase: &Option>, + testcase: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { self.base.on_remove(state, id, testcase)?; let mut entries = @@ -116,10 +105,12 @@ where }; entries.sort_unstable(); // this should already be sorted, but just in case let mut map = HashMap::new(); - for current_id in state.corpus().ids() { - let mut old = state.corpus().get(current_id)?.borrow_mut(); - let factor = F::compute(state, &mut *old)?; - if let Some(old_map) = old.metadata_map_mut().get_mut::() { + let ids: Vec = state.corpus().ids().collect(); + for current_id in ids { + let factor = F::compute(state, current_id)?; + let old = state.corpus().get(current_id)?; + let mut old_md = old.testcase_metadata_mut(); + if let Some(old_map) = old_md.metadata_map_mut().get_mut::() { let mut e_iter = entries.iter(); let mut map_iter = old_map.as_iter(); // ASSERTION: guaranteed to be in order? @@ -169,8 +160,9 @@ where meta.map.reserve(reserve); for (entry, (_, new_id)) in map_iter { - let mut new = state.corpus().get(*new_id)?.borrow_mut(); - let new_meta = new.metadata_map_mut().get_mut::().ok_or_else(|| { + let new = state.corpus().get(*new_id)?; + let mut new_md = new.testcase_metadata_mut(); + let new_meta = new_md.metadata_map_mut().get_mut::().ok_or_else(|| { Error::key_not_found(format!( "{} needed for MinimizerScheduler not found in testcase #{new_id}", type_name::() @@ -185,6 +177,17 @@ where } Ok(()) } + + /// Replaces the [`Testcase`] at the given [`CorpusId`] + fn on_replace( + &mut self, + state: &mut S, + id: CorpusId, + prev: &Testcase>::TestcaseMetadataCell>, + ) -> Result<(), Error> { + self.base.on_replace(state, id, prev)?; + self.update_score(state, id) + } } impl Scheduler for MinimizerScheduler @@ -192,7 +195,7 @@ where CS: Scheduler, F: TestcaseScore, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { @@ -216,7 +219,7 @@ where !state .corpus() .get(id)? - .borrow() + .testcase_metadata() .has_metadata::() } && state.rand_mut().coinflip(self.skip_non_favored_prob) { @@ -245,7 +248,7 @@ where pub fn update_score(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where F: TestcaseScore, - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, { // Create a new top rated meta if not existing if state.metadata_map().get::().is_none() { @@ -254,9 +257,10 @@ where let mut new_favoreds = vec![]; { - let mut entry = state.corpus().get(id)?.borrow_mut(); - let factor = F::compute(state, &mut *entry)?; - let meta = entry.metadata_map_mut().get_mut::().ok_or_else(|| { + let factor = F::compute(state, id)?; + let entry = state.corpus().get(id)?; + let mut entry_md = entry.testcase_metadata_mut(); + let meta = entry_md.metadata_map_mut().get_mut::().ok_or_else(|| { Error::key_not_found(format!( "Metadata needed for MinimizerScheduler not found in testcase #{id}" )) @@ -268,13 +272,14 @@ where new_favoreds.push(*elem); // always retain current; we'll drop it later otherwise continue; } - let mut old = state.corpus().get(*old_id)?.borrow_mut(); - if factor > F::compute(state, &mut *old)? { + if factor > F::compute(state, *old_id)? { continue; } + let old = state.corpus().get(*old_id)?; + let mut old_md = old.testcase_metadata_mut(); let must_remove = { - let old_meta = old.metadata_map_mut().get_mut::().ok_or_else(|| { + let old_meta = old_md.metadata_map_mut().get_mut::().ok_or_else(|| { Error::key_not_found(format!( "{} needed for MinimizerScheduler not found in testcase #{old_id}", type_name::() @@ -285,7 +290,7 @@ where }; if must_remove && self.remove_metadata { - drop(old.metadata_map_mut().remove::()); + drop(old_md.metadata_map_mut().remove::()); } } @@ -300,7 +305,7 @@ where state .corpus() .get(id)? - .borrow_mut() + .testcase_metadata_mut() .metadata_map_mut() .remove::(), ); @@ -319,7 +324,7 @@ where } /// Cull the [`Corpus`] using the [`MinimizerScheduler`] - pub fn cull(&self, state: &S) -> Result<(), Error> + pub fn cull(&self, state: &mut S) -> Result<(), Error> where S: HasCorpus + HasMetadata, { @@ -331,8 +336,9 @@ where for (key, id) in &top_rated.map { if !acc.contains(key) { - let mut entry = state.corpus().get(*id)?.borrow_mut(); - let meta = entry.metadata_map().get::().ok_or_else(|| { + let entry = state.corpus().get(*id)?; + let mut md = entry.testcase_metadata_mut(); + let meta = md.metadata_map().get::().ok_or_else(|| { Error::key_not_found(format!( "{} needed for MinimizerScheduler not found in testcase #{id}", type_name::() @@ -342,7 +348,7 @@ where acc.insert(*elem); } - entry.add_metadata(IsFavoredMetadata {}); + md.add_metadata(IsFavoredMetadata {}); } } diff --git a/crates/libafl/src/schedulers/mod.rs b/crates/libafl/src/schedulers/mod.rs index 870204c98c0..d66d62f4662 100644 --- a/crates/libafl/src/schedulers/mod.rs +++ b/crates/libafl/src/schedulers/mod.rs @@ -3,6 +3,21 @@ use alloc::{borrow::ToOwned, string::ToString}; use core::{hash::Hash, marker::PhantomData}; +use libafl_bolts::{ + generic_hash_std, + rands::Rand, + tuples::{Handle, MatchName, MatchNameRef}, +}; + +use crate::{ + Error, HasMetadata, HasMetadataMut, + corpus::{ + Corpus, CorpusId, HasTestcase, HasTestcaseMetadata, SchedulerTestcaseMetadata, Testcase, + }, + random_corpus_id, + state::{HasCorpus, HasRand}, +}; + pub mod testcase_score; pub use testcase_score::{LenTimeMulTestcaseScore, TestcaseScore}; @@ -27,29 +42,20 @@ pub mod weighted; pub use weighted::{StdWeightedScheduler, WeightedScheduler}; pub mod tuneable; -use libafl_bolts::{ - generic_hash_std, - rands::Rand, - tuples::{Handle, MatchName, MatchNameRef}, -}; pub use tuneable::*; -use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, HasTestcase, SchedulerTestcaseMetadata, Testcase}, - random_corpus_id, - state::{HasCorpus, HasRand}, -}; - /// The scheduler also implements `on_remove` and `on_replace` if it implements this stage. -pub trait RemovableScheduler { +pub trait RemovableScheduler +where + S: HasCorpus, +{ /// Removed the given entry from the corpus at the given index - /// When you remove testcases, make sure that that testcase is not currently fuzzed one! + /// When you remove testcases, make sure that testcase is not currently fuzzed one! fn on_remove( &mut self, _state: &mut S, _id: CorpusId, - _testcase: &Option>, + _testcase: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { Ok(()) } @@ -59,7 +65,7 @@ pub trait RemovableScheduler { &mut self, _state: &mut S, _id: CorpusId, - _prev: &Testcase, + _prev: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { Ok(()) } @@ -73,13 +79,14 @@ pub fn on_add_metadata_default( ) -> Result<(), Error> where CS: AflScheduler, - S: HasTestcase + HasCorpus, + S: HasCorpus + HasTestcase + HasMetadataMut, { let current_id = *state.corpus().current(); let mut depth = match current_id { Some(parent_idx) => state .testcase(parent_idx)? + .testcase_metadata() .metadata::()? .depth(), None => 0, @@ -90,12 +97,13 @@ where // Attach a `SchedulerTestcaseMetadata` to the queue entry. depth += 1; - let mut testcase = state.testcase_mut(id)?; - testcase.add_metadata(SchedulerTestcaseMetadata::with_n_fuzz_entry( + let testcase = state.testcase(id)?; + let mut md = testcase.testcase_metadata_mut(); + md.add_metadata(SchedulerTestcaseMetadata::with_n_fuzz_entry( depth, scheduler.last_hash(), )); - testcase.set_parent_id_optional(current_id); + md.set_parent_id_optional(current_id); Ok(()) } @@ -108,7 +116,7 @@ pub fn on_evaluation_metadata_default( where CS: AflScheduler, CS::ObserverRef: AsRef, - S: HasMetadata, + S: HasMetadataMut, O: Hash, OT: MatchName, { @@ -138,8 +146,10 @@ where let current_id = *state.corpus().current(); if let Some(id) = current_id { - let mut testcase = state.testcase_mut(id)?; - let tcmeta = testcase.metadata_mut::()?; + let testcase = state.testcase(id)?; + let mut md = testcase.testcase_metadata_mut(); + + let tcmeta = md.metadata_mut::()?; if tcmeta.handicap() >= 4 { tcmeta.set_handicap(tcmeta.handicap() - 4); @@ -220,11 +230,7 @@ where fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { // Set parent id let current_id = *state.corpus().current(); - state - .corpus() - .get(id)? - .borrow_mut() - .set_parent_id_optional(current_id); + state.corpus().get(id)?.set_parent_id_optional(current_id); Ok(()) } diff --git a/crates/libafl/src/schedulers/powersched.rs b/crates/libafl/src/schedulers/powersched.rs index 8e13bed83a7..e61e2b87484 100644 --- a/crates/libafl/src/schedulers/powersched.rs +++ b/crates/libafl/src/schedulers/powersched.rs @@ -10,8 +10,8 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, HasTestcase, Testcase}, + Error, HasMetadata, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcase}, schedulers::{ AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler, on_add_metadata_default, on_evaluation_metadata_default, on_next_metadata_default, @@ -280,27 +280,7 @@ pub struct PowerQueueScheduler { phantom: PhantomData, } -impl RemovableScheduler for PowerQueueScheduler { - /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` - fn on_remove( - &mut self, - _state: &mut S, - _id: CorpusId, - _prev: &Option>, - ) -> Result<(), Error> { - Ok(()) - } - - /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` - fn on_replace( - &mut self, - _state: &mut S, - _id: CorpusId, - _prev: &Testcase, - ) -> Result<(), Error> { - Ok(()) - } -} +impl RemovableScheduler for PowerQueueScheduler where S: HasCorpus {} impl AflScheduler for PowerQueueScheduler { type ObserverRef = C; @@ -326,7 +306,7 @@ impl HasQueueCycles for PowerQueueScheduler { impl Scheduler for PowerQueueScheduler where - S: HasCorpus + HasMetadata + HasTestcase, + for<'a> S: HasCorpus + HasMetadataMut + HasTestcase, O: Hash, C: AsRef, { @@ -389,7 +369,7 @@ where #[must_use] pub fn new(state: &mut S, observer: &C, strat: PowerSchedule) -> Self where - S: HasMetadata, + S: HasMetadata + HasMetadataMut, { if !state.has_metadata::() { state.add_metadata::(SchedulerMetadata::new(Some(strat))); diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index 9906af74369..bffee26a0aa 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -9,8 +9,8 @@ use libafl_bolts::rands::Rand; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, Testcase}, + Error, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcaseMetadata, Testcase}, schedulers::{RemovableScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, HasRand}, }; @@ -66,9 +66,9 @@ impl ProbabilitySamplingScheduler { pub fn store_probability(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where F: TestcaseScore, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, { - let prob = F::compute(state, &mut *state.corpus().get(id)?.borrow_mut())?; + let prob = F::compute(state, id)?; debug_assert!( prob >= 0.0 && prob.is_finite(), "scheduler probability is {prob}; to work correctly it must be >= 0.0 and finite" @@ -86,13 +86,13 @@ impl ProbabilitySamplingScheduler { impl RemovableScheduler for ProbabilitySamplingScheduler where F: TestcaseScore, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, { fn on_remove( &mut self, state: &mut S, id: CorpusId, - _testcase: &Option>, + _testcase: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { let meta = state .metadata_map_mut() @@ -108,7 +108,7 @@ where &mut self, state: &mut S, id: CorpusId, - _prev: &Testcase, + _prev: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { let meta = state .metadata_map_mut() @@ -125,14 +125,15 @@ where impl Scheduler for ProbabilitySamplingScheduler where F: TestcaseScore, - S: HasCorpus + HasMetadata + HasRand, + S: HasCorpus + HasMetadataMut + HasRand, { fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { let current_id = *state.corpus().current(); + state .corpus() .get(id)? - .borrow_mut() + .testcase_metadata_mut() .set_parent_id_optional(current_id); if state.metadata_map().get::().is_none() { @@ -206,7 +207,10 @@ mod tests { where S: HasCorpus, { - fn compute(_state: &S, _: &mut Testcase) -> Result { + fn compute( + _state: &S, + _entry: &mut Testcase>::TestcaseMetadataRefMut<'_>>, + ) -> Result { Ok(FACTOR) } } diff --git a/crates/libafl/src/schedulers/queue.rs b/crates/libafl/src/schedulers/queue.rs index c9c8551479e..ef4b9dc24f4 100644 --- a/crates/libafl/src/schedulers/queue.rs +++ b/crates/libafl/src/schedulers/queue.rs @@ -3,8 +3,8 @@ use alloc::borrow::ToOwned; use crate::{ - Error, - corpus::{Corpus, CorpusId}, + Error, HasMetadataMut, + corpus::{Corpus, CorpusId, HasTestcaseMetadata}, schedulers::{HasQueueCycles, RemovableScheduler, Scheduler}, state::HasCorpus, }; @@ -16,20 +16,19 @@ pub struct QueueScheduler { runs_in_current_cycle: u64, } -impl RemovableScheduler for QueueScheduler {} +impl RemovableScheduler for QueueScheduler where S: HasCorpus {} impl Scheduler for QueueScheduler where - S: HasCorpus, + S: HasCorpus + HasMetadataMut, { fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { // Set parent id let current_id = *state.corpus().current(); - state - .corpus() - .get(id)? - .borrow_mut() - .set_parent_id_optional(current_id); + let current_tc = state.corpus().get(id)?; + let mut current_md = current_tc.testcase_metadata_mut(); + + current_md.set_parent_id_optional(current_id); Ok(()) } diff --git a/crates/libafl/src/schedulers/testcase_score.rs b/crates/libafl/src/schedulers/testcase_score.rs index 0aa4e5df201..b13396eda59 100644 --- a/crates/libafl/src/schedulers/testcase_score.rs +++ b/crates/libafl/src/schedulers/testcase_score.rs @@ -6,7 +6,7 @@ use num_traits::Zero; use crate::{ Error, HasMetadata, - corpus::{Corpus, SchedulerTestcaseMetadata, Testcase}, + corpus::{Corpus, CorpusId, HasTestcaseMetadata, SchedulerTestcaseMetadata}, feedbacks::MapIndexesMetadata, schedulers::{ minimizer::{IsFavoredMetadata, TopRatedsMetadata}, @@ -16,9 +16,12 @@ use crate::{ }; /// Compute the favor factor of a [`Testcase`]. Higher is better. -pub trait TestcaseScore { +pub trait TestcaseScore +where + S: HasCorpus, +{ /// Computes the favor factor of a [`Testcase`]. Higher is better. - fn compute(state: &S, entry: &mut Testcase) -> Result; + fn compute(state: &S, corpus_id: CorpusId) -> Result; } /// Multiply the testcase size with the execution time. @@ -32,10 +35,12 @@ where I: HasLen, { #[expect(clippy::cast_precision_loss)] - fn compute(state: &S, entry: &mut Testcase) -> Result { + fn compute(state: &S, corpus_id: CorpusId) -> Result { + let testcase = state.corpus().get(corpus_id)?; + let md = testcase.testcase_metadata(); + // TODO maybe enforce entry.exec_time().is_some() - Ok(entry.exec_time().map_or(1, |d| d.as_millis()) as f64 - * entry.load_len(state.corpus())? as f64) + Ok(md.exec_time().map_or(1, |d| d.as_millis()) as f64 * testcase.input_len() as f64) } } @@ -55,24 +60,24 @@ where { /// Compute the `power` we assign to each corpus entry #[expect(clippy::cast_precision_loss, clippy::too_many_lines)] - fn compute(state: &S, entry: &mut Testcase) -> Result { + fn compute(state: &S, corpus_id: CorpusId) -> Result { let psmeta = state.metadata::()?; + let corpus = state.corpus(); + let testcase = corpus.get(corpus_id)?; + let md = testcase.testcase_metadata(); let fuzz_mu = if let Some(strat) = psmeta.strat() { if *strat.base() == BaseSchedule::COE { - let corpus = state.corpus(); let mut n_paths = 0; let mut v = 0.0; - let cur_index = state.corpus().current().unwrap(); + let cur_index = corpus.current().unwrap(); for id in corpus.ids() { let n_fuzz_entry = if cur_index == id { - entry - .metadata::()? - .n_fuzz_entry() + md.metadata::()?.n_fuzz_entry() } else { corpus .get(id)? - .borrow() + .testcase_metadata() .metadata::()? .n_fuzz_entry() }; @@ -94,7 +99,7 @@ where }; let mut perf_score = 100.0; - let q_exec_us = entry + let q_exec_us = md .exec_time() .ok_or_else(|| Error::key_not_found("exec_time not set".to_string()))? .as_nanos() as f64; @@ -106,8 +111,8 @@ where psmeta.bitmap_size() / psmeta.bitmap_entries() }; - let favored = entry.has_metadata::(); - let tcmeta = entry.metadata::()?; + let favored = md.has_metadata::(); + let tcmeta = md.metadata::()?; if q_exec_us * 0.1 > avg_exec_us { perf_score = 10.0; @@ -179,7 +184,7 @@ where } } BaseSchedule::FAST => { - if entry.scheduled_count() != 0 { + if md.scheduled_count() != 0 { let lg = libm::log2(f64::from(psmeta.n_fuzz()[tcmeta.n_fuzz_entry()])); match lg { @@ -218,11 +223,11 @@ where } } BaseSchedule::LIN => { - factor = (entry.scheduled_count() as f64) + factor = (md.scheduled_count() as f64) / f64::from(psmeta.n_fuzz()[tcmeta.n_fuzz_entry()] + 1); } BaseSchedule::QUAD => { - factor = ((entry.scheduled_count() * entry.scheduled_count()) as f64) + factor = ((md.scheduled_count() * md.scheduled_count()) as f64) / f64::from(psmeta.n_fuzz()[tcmeta.n_fuzz_entry()] + 1); } } @@ -250,7 +255,7 @@ where perf_score = HAVOC_MAX_MULT * 100.0; } - if entry.objectives_found() > 0 && psmeta.strat().is_some_and(|s| s.avoid_crash()) { + if md.objectives_found() > 0 && psmeta.strat().is_some_and(|s| s.avoid_crash()) { perf_score *= 0.00; } @@ -269,23 +274,25 @@ where { /// Compute the `weight` used in weighted corpus entry selection algo #[expect(clippy::cast_precision_loss)] - fn compute(state: &S, entry: &mut Testcase) -> Result { + fn compute(state: &S, corpus_id: CorpusId) -> Result { let mut weight = 1.0; let psmeta = state.metadata::()?; + let testcase = state.corpus().get(corpus_id)?; + let md = testcase.testcase_metadata(); - let tcmeta = entry.metadata::()?; + let tcmeta = md.metadata::()?; // This means that this testcase has never gone through the calibration stage before1, // In this case we'll just return the default weight // This methoud is called in corpus's on_add() method. Fuzz_level is zero at that time. - if entry.scheduled_count() == 0 || psmeta.cycles() == 0 { + if md.scheduled_count() == 0 || psmeta.cycles() == 0 { return Ok(weight); } - let q_exec_us = entry + let q_exec_us = md .exec_time() .ok_or_else(|| Error::key_not_found("exec_time not set".to_string()))? .as_nanos() as f64; - let favored = entry.has_metadata::(); + let favored = md.has_metadata::(); let avg_exec_us = psmeta.exec_time().as_nanos() as f64 / psmeta.cycles() as f64; let avg_bitmap_size = psmeta.bitmap_size_log() / psmeta.bitmap_entries() as f64; @@ -312,9 +319,10 @@ where libm::log2(q_bitmap_size).max(1.0) / avg_bitmap_size }; - let tc_ref = match entry.metadata_map().get::() { - Some(meta) => meta.refcnt() as f64, - None => 0.0, + let tc_ref = if let Some(meta) = md.metadata_map().get::() { + meta.refcnt() as f64 + } else { + 0.0 }; let avg_top_size = match state.metadata::() { @@ -333,11 +341,11 @@ where } // was it fuzzed before? - if entry.scheduled_count() == 0 { + if md.scheduled_count() == 0 { weight *= 2.0; } - if entry.objectives_found() > 0 && psmeta.strat().is_some_and(|s| s.avoid_crash()) { + if md.objectives_found() > 0 && psmeta.strat().is_some_and(|s| s.avoid_crash()) { weight *= 0.00; } diff --git a/crates/libafl/src/schedulers/tuneable.rs b/crates/libafl/src/schedulers/tuneable.rs index 2728cef7146..0999f90a93f 100644 --- a/crates/libafl/src/schedulers/tuneable.rs +++ b/crates/libafl/src/schedulers/tuneable.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use super::RemovableScheduler; use crate::{ - Error, HasMetadata, + Error, HasMetadata, HasMetadataMut, corpus::{Corpus, CorpusId}, schedulers::Scheduler, state::HasCorpus, @@ -36,7 +36,7 @@ impl TuneableScheduler { #[must_use] pub fn new(state: &mut S) -> Self where - S: HasMetadata, + S: HasMetadataMut, { if !state.has_metadata::() { state.add_metadata(TuneableSchedulerMetadata::default()); @@ -46,7 +46,7 @@ impl TuneableScheduler { fn metadata_mut(state: &mut S) -> &mut TuneableSchedulerMetadata where - S: HasMetadata, + S: HasMetadataMut, { state .metadata_map_mut() @@ -67,7 +67,7 @@ impl TuneableScheduler { /// Sets the next corpus id to be used pub fn set_next(state: &mut S, next: CorpusId) where - S: HasMetadata, + S: HasMetadataMut, { Self::metadata_mut(state).next = Some(next); } @@ -83,7 +83,7 @@ impl TuneableScheduler { /// Resets this to a queue scheduler pub fn reset(state: &mut S) where - S: HasMetadata, + S: HasMetadataMut, { let metadata = Self::metadata_mut(state); metadata.next = None; @@ -101,7 +101,7 @@ impl TuneableScheduler { } } -impl RemovableScheduler for TuneableScheduler {} +impl RemovableScheduler for TuneableScheduler where S: HasCorpus {} impl Scheduler for TuneableScheduler where @@ -110,11 +110,7 @@ where fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { // Set parent id let current_id = *state.corpus().current(); - state - .corpus() - .get(id)? - .borrow_mut() - .set_parent_id_optional(current_id); + state.corpus().get(id)?.set_parent_id_optional(current_id); Ok(()) } diff --git a/crates/libafl/src/schedulers/weighted.rs b/crates/libafl/src/schedulers/weighted.rs index 33af2cadeb0..6539366f61d 100644 --- a/crates/libafl/src/schedulers/weighted.rs +++ b/crates/libafl/src/schedulers/weighted.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, + Error, HasMetadataMut, corpus::{Corpus, CorpusId, HasTestcase, Testcase}, random_corpus_id, schedulers::{ @@ -115,7 +115,7 @@ where #[must_use] pub fn new(state: &mut S, observer: &C) -> Self where - S: HasMetadata, + S: HasMetadataMut, { Self::with_schedule(state, observer, None) } @@ -124,7 +124,7 @@ where #[must_use] pub fn with_schedule(state: &mut S, observer: &C, strat: Option) -> Self where - S: HasMetadata, + S: HasMetadataMut, { let _ = state.metadata_or_insert_with(|| SchedulerMetadata::new(strat)); let _ = state.metadata_or_insert_with(WeightedScheduleMetadata::new); @@ -158,7 +158,7 @@ where pub fn create_alias_table(&self, state: &mut S) -> Result<(), Error> where F: TestcaseScore, - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, { let n = state.corpus().count(); @@ -173,8 +173,7 @@ where let mut sum: f64 = 0.0; for i in state.corpus().ids() { - let mut testcase = state.corpus().get(i)?.borrow_mut(); - let weight = F::compute(state, &mut *testcase)?; + let weight = F::compute(state, i)?; weights.insert(i, weight); sum += weight; } @@ -257,13 +256,16 @@ where } } -impl RemovableScheduler for WeightedScheduler { +impl RemovableScheduler for WeightedScheduler +where + S: HasCorpus, +{ /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` fn on_remove( &mut self, _state: &mut S, _id: CorpusId, - _prev: &Option>, + _testcase: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { self.table_invalidated = true; Ok(()) @@ -274,7 +276,7 @@ impl RemovableScheduler for WeightedScheduler { &mut self, _state: &mut S, _id: CorpusId, - _prev: &Testcase, + _prev: &Testcase>::TestcaseMetadataCell>, ) -> Result<(), Error> { self.table_invalidated = true; Ok(()) @@ -308,7 +310,7 @@ where C: AsRef + Named, F: TestcaseScore, O: Hash, - S: HasCorpus + HasMetadata + HasRand + HasTestcase, + S: HasCorpus + HasMetadataMut + HasRand + HasTestcase, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { diff --git a/crates/libafl/src/stages/afl_stats.rs b/crates/libafl/src/stages/afl_stats.rs index c42436b7be1..fe70f1db8ad 100644 --- a/crates/libafl/src/stages/afl_stats.rs +++ b/crates/libafl/src/stages/afl_stats.rs @@ -26,7 +26,10 @@ use serde::{Deserialize, Serialize}; use crate::feedbacks::{CRASH_FEEDBACK_NAME, TIMEOUT_FEEDBACK_NAME}; use crate::{ Error, HasMetadata, HasNamedMetadata, HasScheduler, - corpus::{Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata, Testcase}, + corpus::{ + Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata, Testcase, + testcase::HasTestcaseMetadata, + }, events::{Event, EventFirer, EventWithStats}, executors::HasObservers, feedbacks::{HasObserverHandle, MapFeedbackMetadata}, @@ -273,7 +276,7 @@ where "state is not currently processing a corpus index", )); }; - let testcase = state.corpus().get(corpus_idx)?.borrow(); + let testcase = state.corpus().get(corpus_idx)?; // NOTE: scheduled_count represents the amount of fuzz runs a // testcase has had. Since this stage is kept at the very end of stage list, // the entry would have been fuzzed already (and should contain IsFavoredMetadata) but would have a scheduled count of zero @@ -489,14 +492,14 @@ where Ok(()) } - fn maybe_update_is_favored_size(&mut self, testcase: &Testcase) { + fn maybe_update_is_favored_size(&mut self, testcase: &Testcase) { if testcase.has_metadata::() { self.is_favored_size += 1; } } - fn maybe_update_slowest_exec(&mut self, testcase: &Testcase) { - if let Some(exec_time) = testcase.exec_time() { + fn maybe_update_slowest_exec(&mut self, testcase: &Testcase) { + if let Some(exec_time) = testcase.testcase_metadata().exec_time() { if exec_time > &self.slowest_exec { self.slowest_exec = *exec_time; } @@ -507,8 +510,11 @@ where self.has_fuzzed_size += 1; } - fn maybe_update_max_depth(&mut self, testcase: &Testcase) { - if let Ok(metadata) = testcase.metadata::() { + fn maybe_update_max_depth(&mut self, testcase: &Testcase) { + if let Ok(metadata) = testcase + .testcase_metadata() + .metadata::() + { if metadata.depth() > self.max_depth { self.max_depth = metadata.depth(); } diff --git a/crates/libafl/src/stages/calibrate.rs b/crates/libafl/src/stages/calibrate.rs index 4ff593f1a15..666577f1a5c 100644 --- a/crates/libafl/src/stages/calibrate.rs +++ b/crates/libafl/src/stages/calibrate.rs @@ -13,8 +13,8 @@ use num_traits::Bounded; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasNamedMetadata, - corpus::{Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata}, + Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, + corpus::{Corpus, HasCurrentCorpusId, HasTestcaseMetadata, SchedulerTestcaseMetadata}, events::{Event, EventFirer, EventWithStats, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, feedbacks::{HasObserverHandle, map::MapFeedbackMetadata}, @@ -99,8 +99,8 @@ where Serialize + Deserialize<'de> + 'static + Default + Debug + Bounded, OT: ObserversTuple, S: HasCorpus - + HasMetadata - + HasNamedMetadata + + HasMetadataMut + + HasNamedMetadataMut + HasExecutions + HasCurrentTestcase + HasCurrentCorpusId, @@ -285,19 +285,21 @@ where psmeta.set_bitmap_size_log(psmeta.bitmap_size_log() + libm::log2(bitmap_size as f64)); psmeta.set_bitmap_entries(psmeta.bitmap_entries() + 1); - let mut testcase = state.current_testcase_mut()?; + let testcase = state.current_testcase()?; + let mut tc_md = testcase.testcase_metadata_mut(); - testcase.set_exec_time(total_time / (iter as u32)); - // log::trace!("time: {:#?}", testcase.exec_time()); + tc_md.set_exec_time(total_time / (iter as u32)); // If the testcase doesn't have its own `SchedulerTestcaseMetadata`, create it. - let data = if let Ok(metadata) = testcase.metadata_mut::() { + let data = if let Ok(metadata) = tc_md.metadata_mut::() { metadata } else { - let depth = match testcase.parent_id() { + let depth = match tc_md.parent_id() { Some(parent_id) => { - match (*state.corpus().get(parent_id)?) - .borrow() + match state + .corpus() + .get(parent_id)? + .testcase_metadata() .metadata_map() .get::() { @@ -307,10 +309,8 @@ where } _ => 0, }; - testcase.add_metadata(SchedulerTestcaseMetadata::new(depth)); - testcase - .metadata_mut::() - .unwrap() + tc_md.add_metadata(SchedulerTestcaseMetadata::new(depth)); + tc_md.metadata_mut::().unwrap() }; data.set_cycle_and_time((total_time, iter)); @@ -374,7 +374,7 @@ where impl Restartable for CalibrationStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // Calibration stage disallow restarts diff --git a/crates/libafl/src/stages/colorization.rs b/crates/libafl/src/stages/colorization.rs index f7ca8a24745..c1b5ff16475 100644 --- a/crates/libafl/src/stages/colorization.rs +++ b/crates/libafl/src/stages/colorization.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasNamedMetadata, + Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, corpus::HasCurrentCorpusId, events::EventFirer, executors::{Executor, HasObservers}, @@ -78,7 +78,7 @@ impl Stage for ColorizationStage, E: HasObservers + Executor, - S: HasCorpus + HasMetadata + HasRand + HasNamedMetadata + HasCurrentCorpusId, + S: HasCorpus + HasMetadataMut + HasRand + HasNamedMetadata + HasCurrentCorpusId, E::Observers: ObserversTuple, I: ResizableMutator + HasMutatorBytes + Clone, O: Hash, @@ -101,7 +101,7 @@ where impl Restartable for ColorizationStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // This is a deterministic stage @@ -161,7 +161,7 @@ where C: AsRef + Named, E: HasObservers + Executor, E::Observers: ObserversTuple, - S: HasCorpus + HasMetadata + HasRand + HasCurrentCorpusId + HasCurrentTestcase, + S: HasCorpus + HasMetadataMut + HasRand + HasCurrentCorpusId + HasCurrentTestcase, I: ResizableMutator + HasMutatorBytes + Clone, { #[inline] diff --git a/crates/libafl/src/stages/concolic.rs b/crates/libafl/src/stages/concolic.rs index c39cfd57913..a59c7c89777 100644 --- a/crates/libafl/src/stages/concolic.rs +++ b/crates/libafl/src/stages/concolic.rs @@ -14,8 +14,8 @@ use libafl_bolts::{ #[cfg(all(feature = "concolic_mutation", feature = "introspection"))] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, - corpus::HasCurrentCorpusId, + Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, + corpus::{HasCurrentCorpusId, HasTestcaseMetadata}, executors::{Executor, HasObservers}, observers::{ObserversTuple, concolic::ConcolicObserver}, stages::{Restartable, RetryCountRestartHelper, Stage, TracingStage}, @@ -53,7 +53,7 @@ where TE::Observers: ObserversTuple, S: HasExecutions + HasCorpus - + HasNamedMetadata + + HasMetadataMut + HasCurrentTestcase + HasCurrentCorpusId + MaybeHasClientPerfMonitor, @@ -70,7 +70,8 @@ where if let Some(observer) = self.inner.executor().observers().get(&self.observer_handle) { let metadata = observer.create_metadata_from_current_map(); state - .current_testcase_mut()? + .current_testcase()? + .testcase_metadata_mut() .metadata_map_mut() .insert(metadata); } @@ -80,7 +81,7 @@ where impl Restartable for ConcolicTracingStage<'_, EM, I, TE, S, Z> where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // This is a deterministic stage diff --git a/crates/libafl/src/stages/dump.rs b/crates/libafl/src/stages/dump.rs index ef10eee334e..179f9fab142 100644 --- a/crates/libafl/src/stages/dump.rs +++ b/crates/libafl/src/stages/dump.rs @@ -6,8 +6,7 @@ use alloc::{ }; use core::{clone::Clone, marker::PhantomData}; use std::{ - fs::{self, File}, - io::Write, + fs, path::{Path, PathBuf}, }; @@ -47,8 +46,8 @@ pub struct DumpToDiskStage { impl Stage for DumpToDiskStage where - CB1: FnMut(&Testcase, &S) -> Vec, - CB2: FnMut(&Testcase, &CorpusId) -> P, + CB1: FnMut(&Testcase>::TestcaseMetadataCell>, &S) -> Vec, + CB2: FnMut(&Testcase>::TestcaseMetadataCell>, &CorpusId) -> P, S: HasCorpus + HasSolutions + HasRand + HasMetadata, P: AsRef, { @@ -57,15 +56,25 @@ where &mut self, _fuzzer: &mut Z, _executor: &mut E, - state: &mut S, + _state: &mut S, _manager: &mut EM, ) -> Result<(), Error> { - self.dump_state_to_disk(state) + // self.dump_state_to_disk(state) + Ok(()) } } impl Restartable - for DumpToDiskStage, &CorpusId) -> String, EM, I, S, Z> + for DumpToDiskStage< + CB1, + fn(&Testcase>::TestcaseMetadataCell>, &CorpusId) -> String, + EM, + I, + S, + Z, + > +where + S: HasCorpus, { #[inline] fn should_restart(&mut self, _state: &mut S) -> Result { @@ -81,9 +90,17 @@ impl Restartable } /// Implementation for `DumpToDiskStage` with a default `generate_filename` function. -impl DumpToDiskStage, &CorpusId) -> String, EM, I, S, Z> +impl + DumpToDiskStage< + CB1, + fn(&Testcase>::TestcaseMetadataCell>, &CorpusId) -> String, + EM, + I, + S, + Z, + > where - S: HasSolutions + HasRand + HasMetadata, + S: HasCorpus + HasSolutions + HasRand + HasMetadata, I: Input, { /// Create a new [`DumpToDiskStage`] with a default `generate_filename` function. @@ -102,14 +119,18 @@ where /// Default `generate_filename` function. #[expect(clippy::trivially_copy_pass_by_ref)] - fn generate_filename(testcase: &Testcase, id: &CorpusId) -> String { + fn generate_filename( + testcase: &Testcase>::TestcaseMetadataCell>, + id: &CorpusId, + ) -> String { + // TODO: check that [ Some(id.0.to_string()), - testcase.filename().clone(), - testcase - .input() - .as_ref() - .map(|t| t.generate_name(Some(*id))), + Some(testcase.id().clone()), + // testcase + // .input() + // .as_ref() + // .map(|t| t.generate_name(Some(*id))), ] .iter() .flatten() @@ -161,56 +182,57 @@ where }) } - #[inline] - fn dump_state_to_disk>(&mut self, state: &mut S) -> Result<(), Error> - where - S: HasCorpus, - CB1: FnMut(&Testcase, &S) -> Vec, - CB2: FnMut(&Testcase, &CorpusId) -> P, - { - let (mut corpus_id, mut solutions_id) = - if let Some(meta) = state.metadata_map().get::() { - ( - meta.last_corpus.and_then(|x| state.corpus().next(x)), - meta.last_solution.and_then(|x| state.solutions().next(x)), - ) - } else { - (state.corpus().first(), state.solutions().first()) - }; - - while let Some(i) = corpus_id { - let mut testcase = state.corpus().get(i)?.borrow_mut(); - state.corpus().load_input_into(&mut testcase)?; - let bytes = (self.to_bytes)(&testcase, state); - - let fname = self - .corpus_dir - .join((self.generate_filename)(&testcase, &i)); - let mut f = File::create(fname)?; - drop(f.write_all(&bytes)); - - corpus_id = state.corpus().next(i); - } - - while let Some(i) = solutions_id { - let mut testcase = state.solutions().get(i)?.borrow_mut(); - state.solutions().load_input_into(&mut testcase)?; - let bytes = (self.to_bytes)(&testcase, state); - - let fname = self - .solutions_dir - .join((self.generate_filename)(&testcase, &i)); - let mut f = File::create(fname)?; - drop(f.write_all(&bytes)); - - solutions_id = state.solutions().next(i); - } - - state.add_metadata(DumpToDiskMetadata { - last_corpus: state.corpus().last(), - last_solution: state.solutions().last(), - }); - - Ok(()) - } + // #[inline] + // fn dump_state_to_disk, M>(&mut self, state: &mut S) -> Result<(), Error> + // where + // CB1: FnMut(&Testcase, &S) -> Vec, + // CB2: FnMut(&Testcase, &CorpusId) -> P, + // S: HasCorpus, + // M: HasTestcaseMetadata, + // { + // let (mut corpus_id, mut solutions_id) = + // if let Some(meta) = state.metadata_map().get::() { + // ( + // meta.last_corpus.and_then(|x| state.corpus().next(x)), + // meta.last_solution.and_then(|x| state.solutions().next(x)), + // ) + // } else { + // (state.corpus().first(), state.solutions().first()) + // }; + + // while let Some(i) = corpus_id { + // let testcase = state.corpus().get(i)?; + // // state.corpus().load_input_into(&mut testcase)?; + // let bytes = (self.to_bytes)(&testcase, state); + + // let fname = self + // .corpus_dir + // .join((self.generate_filename)(&testcase, &i)); + // let mut f = File::create(fname)?; + // drop(f.write_all(&bytes)); + + // corpus_id = state.corpus().next(i); + // } + + // while let Some(i) = solutions_id { + // let testcase = state.solutions().get(i)?; + // // state.solutions().load_input_into(&mut testcase)?; + // let bytes = (self.to_bytes)(&testcase, state); + + // let fname = self + // .solutions_dir + // .join((self.generate_filename)(&testcase, &i)); + // let mut f = File::create(fname)?; + // drop(f.write_all(&bytes)); + + // solutions_id = state.solutions().next(i); + // } + + // state.add_metadata(DumpToDiskMetadata { + // last_corpus: state.corpus().last(), + // last_solution: state.solutions().last(), + // }); + + // Ok(()) + // } } diff --git a/crates/libafl/src/stages/generalization.rs b/crates/libafl/src/stages/generalization.rs index 712eb1ca0cd..106e08a9914 100644 --- a/crates/libafl/src/stages/generalization.rs +++ b/crates/libafl/src/stages/generalization.rs @@ -14,13 +14,11 @@ use libafl_bolts::{ #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, - corpus::{Corpus, HasCurrentCorpusId}, + Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, + corpus::{Corpus, HasCurrentCorpusId, HasTestcaseMetadata}, executors::{Executor, HasObservers}, feedbacks::map::MapNoveltiesMetadata, - inputs::{ - BytesInput, GeneralizedInputMetadata, GeneralizedItem, HasMutatorBytes, ResizableMutator, - }, + inputs::{BytesInput, GeneralizedInputMetadata, GeneralizedItem, ResizableMutator}, mark_feature_time, observers::{CanTrack, MapObserver, ObserversTuple}, require_novelties_tracking, @@ -64,7 +62,7 @@ impl Named for GeneralizationStage impl Restartable for GeneralizationStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { @@ -112,25 +110,29 @@ where start_timer!(state); { let corpus = state.corpus(); - let mut testcase = corpus.get(corpus_id)?.borrow_mut(); + let testcase = corpus.get(corpus_id)?; if testcase.scheduled_count() > 0 { return Ok(()); } - - corpus.load_input_into(&mut testcase)?; } mark_feature_time!(state, PerfFeature::GetInputFromCorpus); - let mut entry = state.corpus().get(corpus_id)?.borrow_mut(); - let input = entry.input_mut().as_mut().unwrap(); - - let payload: Vec<_> = input.mutator_bytes().iter().map(|&x| Some(x)).collect(); + let entry = state.corpus().get(corpus_id)?; + let md = entry.testcase_metadata(); + let input = entry.input(); + + let payload: Vec<_> = input + .as_ref() + .clone() + .into_inner() + .into_iter() + .map(|x| Some(x)) + .collect(); if payload.len() > MAX_GENERALIZED_LEN { return Ok(()); } - let original = input.clone(); - let meta = entry.metadata_map().get::().ok_or_else(|| { + let meta = md.metadata_map().get::().ok_or_else(|| { Error::key_not_found(format!( "MapNoveltiesMetadata needed for GeneralizationStage not found in testcase #{corpus_id} (check the arguments of MapFeedback::new(...))" )) @@ -138,7 +140,7 @@ where if meta.as_slice().is_empty() { return Ok(()); // don't generalise inputs which don't have novelties } - (payload, original, meta.as_slice().to_vec()) + (payload, input.clone(), meta.as_slice().to_vec()) }; // Do not generalized unstable inputs @@ -336,8 +338,11 @@ where assert!(meta.generalized().first() == Some(&GeneralizedItem::Gap)); assert!(meta.generalized().last() == Some(&GeneralizedItem::Gap)); - let mut entry = state.corpus().get(corpus_id)?.borrow_mut(); - entry.metadata_map_mut().insert(meta); + let entry = state.corpus().get(corpus_id)?; + entry + .testcase_metadata_mut() + .metadata_map_mut() + .insert(meta); } Ok(()) diff --git a/crates/libafl/src/stages/mod.rs b/crates/libafl/src/stages/mod.rs index 56b243efa61..880607355d0 100644 --- a/crates/libafl/src/stages/mod.rs +++ b/crates/libafl/src/stages/mod.rs @@ -46,7 +46,7 @@ pub use unicode::*; pub use verify_timeouts::{TimeoutsToVerify, VerifyTimeoutsStage}; use crate::{ - Error, HasNamedMetadata, + Error, HasNamedMetadata, HasNamedMetadataMut, corpus::{CorpusId, HasCurrentCorpusId}, events::SendExiting, state::{HasCurrentStageId, HasExecutions, MaybeHasClientPerfMonitor, Stoppable}, @@ -337,7 +337,7 @@ where impl Restartable for ClosureStage where - S: HasNamedMetadata + HasCurrentCorpusId, + S: HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { @@ -385,7 +385,7 @@ impl RetryCountRestartHelper { /// Don't allow restart pub fn no_retry(state: &mut S, name: &str) -> Result where - S: HasNamedMetadata + HasCurrentCorpusId, + S: HasNamedMetadataMut + HasCurrentCorpusId, { Self::should_restart(state, name, 1) } @@ -395,7 +395,7 @@ impl RetryCountRestartHelper { /// Returns `true` if the stage should run pub fn should_restart(state: &mut S, name: &str, max_retries: usize) -> Result where - S: HasNamedMetadata + HasCurrentCorpusId, + S: HasNamedMetadataMut + HasCurrentCorpusId, { let corpus_id = state.current_corpus_id()?.ok_or_else(|| { Error::illegal_state( @@ -434,7 +434,7 @@ impl RetryCountRestartHelper { /// Clears the progress pub fn clear_progress(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { state.named_metadata_mut::(name)?.tries_remaining = None; Ok(()) @@ -508,7 +508,7 @@ impl ExecutionCountRestartHelper { /// Initialize progress for the stage this wrapper wraps. pub fn should_restart(&mut self, state: &mut S, name: &str) -> Result where - S: HasNamedMetadata + HasExecutions, + S: HasNamedMetadataMut + HasExecutions, { let executions = *state.executions(); let metadata = @@ -522,7 +522,7 @@ impl ExecutionCountRestartHelper { /// Clear progress for the stage this wrapper wraps. pub fn clear_progress(&mut self, state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { self.started_at_execs = None; let _metadata = state.remove_named_metadata::(name); diff --git a/crates/libafl/src/stages/mutational.rs b/crates/libafl/src/stages/mutational.rs index 454aa7ba0ad..7c591a63cdc 100644 --- a/crates/libafl/src/stages/mutational.rs +++ b/crates/libafl/src/stages/mutational.rs @@ -12,8 +12,8 @@ use libafl_bolts::{Named, rands::Rand}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, - corpus::{Corpus, CorpusId, HasCurrentCorpusId, Testcase}, + Error, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, + corpus::{CorpusId, HasCurrentCorpusId, Testcase, testcase::HasTestcaseMetadata}, fuzzer::Evaluator, inputs::Input, mark_feature_time, @@ -23,7 +23,6 @@ use crate::{ start_timer, state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasRand, MaybeHasClientPerfMonitor}, }; - // TODO multi mutators stage /// Action performed after the un-transformed input is executed (e.g., updating metadata) @@ -47,7 +46,10 @@ pub trait MutatedTransform: Sized { type Post: MutatedTransformPost; /// Transform the provided testcase into this type - fn try_transform_from(base: &mut Testcase, state: &S) -> Result; + fn try_transform_from( + base: &Testcase, + state: &S, + ) -> Result; /// Transform this instance back into the original input type fn try_transform_into(self, state: &S) -> Result<(I, Self::Post), Error>; @@ -62,9 +64,11 @@ where type Post = (); #[inline] - fn try_transform_from(base: &mut Testcase, state: &S) -> Result { - state.corpus().load_input_into(base)?; - Ok(base.input().as_ref().unwrap().clone()) + fn try_transform_from( + base: &Testcase, + _state: &S, + ) -> Result { + Ok(base.input().as_ref().clone()) } #[inline] @@ -169,7 +173,7 @@ where impl Restartable for StdMutationalStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::should_restart(state, &self.name, 3) @@ -258,9 +262,9 @@ where .saturating_sub(self.execs_since_progress_start(state)?); */ let num = self.iterations(state)?; - let mut testcase = state.current_testcase_mut()?; + let testcase = state.current_testcase()?; - let Ok(input) = I1::try_transform_from(&mut testcase, state) else { + let Ok(input) = I1::try_transform_from(&testcase, state) else { return Ok(()); }; drop(testcase); @@ -324,8 +328,8 @@ where state: &mut S, manager: &mut EM, ) -> Result<(), Error> { - let mut testcase = state.current_testcase_mut()?; - let Ok(input) = I::try_transform_from(&mut testcase, state) else { + let testcase = state.current_testcase()?; + let Ok(input) = I::try_transform_from(&testcase, state) else { return Ok(()); }; drop(testcase); @@ -345,7 +349,7 @@ where impl Restartable for MultiMutationalStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/power.rs b/crates/libafl/src/stages/power.rs index 10a86c029bb..2fe2baca203 100644 --- a/crates/libafl/src/stages/power.rs +++ b/crates/libafl/src/stages/power.rs @@ -11,7 +11,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, + Error, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers}, fuzzer::Evaluator, @@ -23,7 +23,7 @@ use crate::{ mutational::{MutatedTransform, MutatedTransformPost}, }, start_timer, - state::{HasCurrentTestcase, HasExecutions, HasRand, MaybeHasClientPerfMonitor}, + state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasRand, MaybeHasClientPerfMonitor}, }; /// The unique id for this stage @@ -47,7 +47,7 @@ impl Named for PowerMutationalStage impl MutationalStage for PowerMutationalStage where - S: HasCurrentTestcase, + S: HasCorpus + HasCurrentCorpusId, F: TestcaseScore, { type Mutator = M; @@ -67,8 +67,8 @@ where #[expect(clippy::cast_sign_loss)] fn iterations(&self, state: &mut S) -> Result { // Update handicap - let mut testcase = state.current_testcase_mut()?; - let score = F::compute(state, &mut testcase)? as usize; + let current_corpus_id = state.current_corpus_id()?.unwrap(); + let score = F::compute(state, current_corpus_id)? as usize; Ok(score) } @@ -105,7 +105,7 @@ where impl Restartable for PowerMutationalStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // Make sure we don't get stuck crashing on a single testcase @@ -122,7 +122,11 @@ where E: Executor + HasObservers, F: TestcaseScore, M: Mutator, - S: HasMetadata + HasRand + HasCurrentTestcase + MaybeHasClientPerfMonitor, + S: HasMetadata + + HasRand + + HasCurrentTestcase + + MaybeHasClientPerfMonitor + + HasCurrentCorpusId, I: MutatedTransform + Clone, Z: Evaluator, { @@ -160,9 +164,9 @@ where .saturating_sub(self.execs_since_progress_start(state)?); */ let num = self.iterations(state)?; - let mut testcase = state.current_testcase_mut()?; + let testcase = state.current_testcase()?; - let Ok(input) = I::try_transform_from(&mut testcase, state) else { + let Ok(input) = I::try_transform_from(&testcase, state) else { return Ok(()); }; drop(testcase); diff --git a/crates/libafl/src/stages/push/mod.rs b/crates/libafl/src/stages/push/mod.rs index c5c926483ee..c5ff4786feb 100644 --- a/crates/libafl/src/stages/push/mod.rs +++ b/crates/libafl/src/stages/push/mod.rs @@ -21,7 +21,8 @@ use libafl_bolts::Named; pub use mutational::StdMutationalPushStage; use crate::{ - Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasMetadata, HasScheduler, + Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasMetadata, HasNamedMetadataMut, + HasScheduler, common::HasNamedMetadata, corpus::{CorpusId, HasCurrentCorpusId}, events::{EventFirer, EventRestarter, HasEventManagerId, ProgressReporter}, @@ -244,7 +245,7 @@ impl Named for PushStageAdapter { impl Restartable for PushStageAdapter where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/push/mutational.rs b/crates/libafl/src/stages/push/mutational.rs index fa1e1ff249c..561d61b587d 100644 --- a/crates/libafl/src/stages/push/mutational.rs +++ b/crates/libafl/src/stages/push/mutational.rs @@ -130,8 +130,9 @@ where start_timer!(state); let input = state - .corpus_mut() - .cloned_input_for_id(self.current_corpus_id.unwrap()); + .corpus() + .get(self.current_corpus_id.unwrap()) + .map(|tc| tc.input().as_ref().clone()); let mut input = match input { Err(e) => return Some(Err(e)), Ok(input) => input, diff --git a/crates/libafl/src/stages/replay.rs b/crates/libafl/src/stages/replay.rs index 8addf866882..8204f154e7f 100644 --- a/crates/libafl/src/stages/replay.rs +++ b/crates/libafl/src/stages/replay.rs @@ -12,7 +12,7 @@ use libafl_bolts::{Named, impl_serdeany}; use serde::{Deserialize, Serialize}; use crate::{ - Error, Evaluator, HasMetadata, + Error, Evaluator, HasMetadataMut, corpus::{Corpus, CorpusId}, stages::{Restartable, Stage}, state::{HasCorpus, HasSolutions}, @@ -108,7 +108,7 @@ impl ReplayStage { impl Stage for ReplayStage where - S: HasCorpus + HasSolutions + HasMetadata, + S: HasCorpus + HasSolutions + HasMetadataMut, Z: Evaluator, I: Clone, { @@ -132,9 +132,8 @@ where log::info!("Replaying corpus: {id}"); let input = { - let mut tc = state.corpus().get(id)?.borrow_mut(); - let input = tc.load_input(state.corpus())?; - input.clone() + let tc = state.corpus().get(id)?; + tc.input().as_ref().clone() }; fuzzer.evaluate_input(state, executor, manager, &input)?; @@ -151,9 +150,8 @@ where } log::info!("Replaying solution: {id}"); let input = { - let mut tc = state.solutions().get(id)?.borrow_mut(); - let input = tc.load_input(state.corpus())?; - input.clone() + let tc = state.solutions().get(id)?; + tc.input().as_ref().clone() }; fuzzer.evaluate_input(state, executor, manager, &input)?; @@ -165,7 +163,7 @@ where impl Restartable for ReplayStage where - S: HasMetadata, + S: HasMetadataMut, { fn should_restart(&mut self, state: &mut S) -> Result { state.metadata_or_insert_with(ReplayRestarterMetadata::default); diff --git a/crates/libafl/src/stages/shadow.rs b/crates/libafl/src/stages/shadow.rs index 7bfdd68b4c9..3ceec60e099 100644 --- a/crates/libafl/src/stages/shadow.rs +++ b/crates/libafl/src/stages/shadow.rs @@ -12,7 +12,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasNamedMetadata, + Error, HasNamedMetadata, HasNamedMetadataMut, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers, ShadowExecutor}, mark_feature_time, @@ -104,7 +104,7 @@ where impl Restartable for ShadowTracingStage where - S: HasNamedMetadata + HasCurrentCorpusId, + S: HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::no_retry(state, &self.name) diff --git a/crates/libafl/src/stages/sync.rs b/crates/libafl/src/stages/sync.rs index ea94295b62d..a7ac2654095 100644 --- a/crates/libafl/src/stages/sync.rs +++ b/crates/libafl/src/stages/sync.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasNamedMetadata, + Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, corpus::{Corpus, CorpusId, HasCurrentCorpusId}, events::{Event, EventConfig, EventFirer, EventWithStats, llmp::LlmpEventConverter}, executors::{Executor, ExitKind, HasObservers}, @@ -80,7 +80,7 @@ where Z: Evaluator, S: HasCorpus + HasRand - + HasMetadata + + HasMetadataMut + HasNamedMetadata + HasCurrentCorpusId + MaybeHasClientPerfMonitor, @@ -155,7 +155,7 @@ where impl Restartable for SyncFromDiskStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { @@ -253,7 +253,7 @@ where ICB: InputConverter, S: HasExecutions + HasRand - + HasMetadata + + HasMetadataMut + HasSolutions + HasCurrentTestcase + Stoppable @@ -280,7 +280,7 @@ where last_id.map_or_else(|| state.corpus().first(), |id| state.corpus().next(id)); while let Some(id) = cur_id { - let input = state.corpus().cloned_input_for_id(id)?; + let input = state.corpus().get(id)?.cloned_input(); self.client.fire( state, diff --git a/crates/libafl/src/stages/time_tracker.rs b/crates/libafl/src/stages/time_tracker.rs index ec79de33c09..b01447322f4 100644 --- a/crates/libafl/src/stages/time_tracker.rs +++ b/crates/libafl/src/stages/time_tracker.rs @@ -1,12 +1,14 @@ //! Stage that wraps another stage and tracks it's execution time in `State` + use core::{marker::PhantomData, time::Duration}; use libafl_bolts::{Error, current_time}; use crate::{ - HasMetadata, + HasMetadataMut, stages::{Restartable, Stage}, }; + /// Track an inner Stage's execution time #[derive(Debug)] pub struct TimeTrackingStageWrapper { @@ -28,7 +30,7 @@ impl TimeTrackingStageWrapper { impl Stage for TimeTrackingStageWrapper where - S: HasMetadata, + S: HasMetadataMut, ST: Stage, T: libafl_bolts::serdeany::SerdeAny + From, { diff --git a/crates/libafl/src/stages/tmin.rs b/crates/libafl/src/stages/tmin.rs index cd66d1d51a3..897cdfbd493 100644 --- a/crates/libafl/src/stages/tmin.rs +++ b/crates/libafl/src/stages/tmin.rs @@ -2,9 +2,10 @@ use alloc::{ borrow::{Cow, ToOwned}, + rc::Rc, string::ToString, }; -use core::{borrow::BorrowMut, fmt::Debug, hash::Hash, marker::PhantomData}; +use core::{fmt::Debug, hash::Hash, marker::PhantomData}; use ahash::RandomState; use libafl_bolts::{ @@ -19,8 +20,8 @@ use crate::feedbacks::premature_last_result_err; use crate::monitors::stats::PerfFeature; use crate::{ Error, ExecutesInput, ExecutionProcessor, HasFeedback, HasMetadata, HasNamedMetadata, - HasScheduler, - corpus::{Corpus, HasCurrentCorpusId, Testcase}, + HasNamedMetadataMut, HasScheduler, + corpus::{Corpus, HasCurrentCorpusId, testcase::TestcaseMetadata}, events::EventFirer, executors::{ExitKind, HasObservers}, feedbacks::{Feedback, FeedbackFactory, HasObserverHandle, StateInitializer}, @@ -96,7 +97,7 @@ where impl Restartable for StdTMinMutationalStage where - S: HasNamedMetadata + HasExecutions, + S: HasNamedMetadataMut + HasExecutions, { fn should_restart(&mut self, state: &mut S) -> Result { self.restart_helper.should_restart(state, &self.name) @@ -191,7 +192,7 @@ where } start_timer!(state); - let transformed = I::try_transform_from(state.current_testcase_mut()?.borrow_mut(), state)?; + let transformed = I::try_transform_from(&mut state.current_testcase()?, state)?; let mut base = state.current_input_cloned()?; // potential post operation if base is replaced by a shorter input let mut base_post = None; @@ -283,17 +284,24 @@ where fuzzer .feedback_mut() .is_interesting(state, manager, &base, &*observers, &exit_kind)?; - let mut testcase = Testcase::from(base); - testcase.set_executions(*state.executions()); - testcase.set_parent_id(base_corpus_id); + + let mut tc_md = TestcaseMetadata::builder() + .executions(*state.executions()) + .parent_id(Some(base_corpus_id)) + .build(); fuzzer .feedback_mut() - .append_metadata(state, manager, &*observers, &mut testcase)?; - let prev = state.corpus_mut().replace(base_corpus_id, testcase)?; + .append_metadata(state, manager, &*observers, &mut tc_md)?; + + let prev = state + .corpus_mut() + .replace(base_corpus_id, Rc::new(base), tc_md)?; + fuzzer .scheduler_mut() .on_replace(state, base_corpus_id, &prev)?; + // perform the post operation for the new testcase, e.g. to update metadata. // base_post should be updated along with the base (and is no longer None) base_post @@ -366,6 +374,7 @@ where M: Hash, C: AsRef, OT: MatchName, + S: HasCorpus, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/stages/tracing.rs b/crates/libafl/src/stages/tracing.rs index 6df82fdf0ca..02239972984 100644 --- a/crates/libafl/src/stages/tracing.rs +++ b/crates/libafl/src/stages/tracing.rs @@ -11,7 +11,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasNamedMetadata, + Error, HasNamedMetadata, HasNamedMetadataMut, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers}, inputs::Input, @@ -35,11 +35,7 @@ impl TracingStage where TE: Executor + HasObservers, TE::Observers: ObserversTuple, - S: HasExecutions - + HasCorpus - + HasNamedMetadata - + HasCurrentTestcase - + MaybeHasClientPerfMonitor, + S: HasExecutions + HasCorpus + HasCurrentTestcase + MaybeHasClientPerfMonitor, { /// Perform tracing on the given `CorpusId`. Useful for if wrapping [`TracingStage`] with your /// own stage and you need to manage [`super::NestedStageRetryCountRestartHelper`] differently @@ -98,7 +94,7 @@ where impl Restartable for TracingStage where - S: HasNamedMetadata + HasCurrentCorpusId, + S: HasNamedMetadataMut + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::no_retry(state, &self.name) diff --git a/crates/libafl/src/stages/tuneable.rs b/crates/libafl/src/stages/tuneable.rs index ac47097b10a..6404b6e7bb2 100644 --- a/crates/libafl/src/stages/tuneable.rs +++ b/crates/libafl/src/stages/tuneable.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, Evaluator, HasMetadata, HasNamedMetadata, mark_feature_time, + Error, Evaluator, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, mark_feature_time, mutators::{MutationResult, Mutator}, nonzero, stages::{ @@ -38,7 +38,7 @@ pub const STD_TUNEABLE_MUTATIONAL_STAGE_NAME: &str = "TuneableMutationalStage"; /// Set the number of iterations to be used by this mutational stage by name pub fn set_iters_by_name(state: &mut S, iters: u64, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { let metadata = state .named_metadata_map_mut() @@ -52,7 +52,7 @@ where /// Set the number of iterations to be used by this mutational stage with a default name pub fn set_iters_std(state: &mut S, iters: u64) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -84,7 +84,7 @@ pub fn set_seed_fuzz_time_by_name( name: &str, ) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { let metadata = state .named_metadata_map_mut() @@ -98,7 +98,7 @@ where /// Set the time for a single seed to be used by this mutational stage with a default name pub fn set_seed_fuzz_time_std(state: &mut S, fuzz_time: Duration) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -126,7 +126,7 @@ where /// Reset this to a normal, randomized, stage by name pub fn reset_by_name(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { state .named_metadata_map_mut() @@ -141,7 +141,7 @@ where /// Reset this to a normal, randomized, stage with a default name pub fn reset_std(state: &mut S) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -215,7 +215,7 @@ where impl Restartable for TuneableMutationalStage where - S: HasNamedMetadata + HasExecutions, + S: HasNamedMetadataMut + HasExecutions, { fn should_restart(&mut self, state: &mut S) -> Result { self.restart_helper.should_restart(state, &self.name) @@ -251,8 +251,8 @@ where let iters = self.fixed_iters(state)?; start_timer!(state); - let mut testcase = state.current_testcase_mut()?; - let Ok(input) = I::try_transform_from(&mut testcase, state) else { + let testcase = state.current_testcase()?; + let Ok(input) = I::try_transform_from(&testcase, state) else { return Ok(()); }; drop(testcase); @@ -307,32 +307,41 @@ where /// Creates a new default tuneable mutational stage #[must_use] - pub fn new(state: &mut S, mutator: M) -> Self { + pub fn new(state: &mut S, mutator: M) -> Self + where + S: HasNamedMetadataMut, + { Self::transforming(state, mutator, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } /// Crates a new tuneable mutational stage with the given name - pub fn with_name(state: &mut S, mutator: M, name: &str) -> Self { + pub fn with_name(state: &mut S, mutator: M, name: &str) -> Self + where + S: HasNamedMetadataMut, + { Self::transforming(state, mutator, name) } /// Set the number of iterations to be used by this [`TuneableMutationalStage`] pub fn set_iters(&self, state: &mut S, iters: u64) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_iters_by_name(state, iters, &self.name) } /// Set the number of iterations to be used by the std [`TuneableMutationalStage`] - pub fn set_iters_std(state: &mut S, iters: u64) -> Result<(), Error> { + pub fn set_iters_std(state: &mut S, iters: u64) -> Result<(), Error> + where + S: HasNamedMetadataMut, + { set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } /// Set the number of iterations to be used by the [`TuneableMutationalStage`] with the given name pub fn set_iters_by_name(state: &mut S, iters: u64, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_iters_by_name(state, iters, name) } @@ -361,13 +370,16 @@ where /// Set the time to mutate a single input in this [`TuneableMutationalStage`] pub fn set_seed_fuzz_time(&self, state: &mut S, fuzz_time: Duration) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_seed_fuzz_time_by_name(state, fuzz_time, &self.name) } /// Set the time to mutate a single input in the std [`TuneableMutationalStage`] - pub fn set_seed_fuzz_time_std(state: &mut S, fuzz_time: Duration) -> Result<(), Error> { + pub fn set_seed_fuzz_time_std(state: &mut S, fuzz_time: Duration) -> Result<(), Error> + where + S: HasNamedMetadataMut, + { set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -378,7 +390,7 @@ where name: &str, ) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { set_seed_fuzz_time_by_name(state, fuzz_time, name) } @@ -407,20 +419,23 @@ where /// Reset this to a normal, randomized, stage with pub fn reset(&self, state: &mut S) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { reset_by_name(state, &self.name) } /// Reset the std stage to a normal, randomized, stage - pub fn reset_std(state: &mut S) -> Result<(), Error> { + pub fn reset_std(state: &mut S) -> Result<(), Error> + where + S: HasNamedMetadataMut, + { reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } /// Reset this to a normal, randomized, stage by name pub fn reset_by_name(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadata, + S: HasNamedMetadataMut, { reset_by_name(state, name) } @@ -459,7 +474,7 @@ impl TuneableMutationalStage where M: Mutator, Z: Evaluator, - S: HasRand + HasNamedMetadata, + S: HasRand + HasNamedMetadataMut, { /// Creates a new transforming mutational stage #[must_use] diff --git a/crates/libafl/src/stages/verify_timeouts.rs b/crates/libafl/src/stages/verify_timeouts.rs index d28e2f678d6..8385cc1d793 100644 --- a/crates/libafl/src/stages/verify_timeouts.rs +++ b/crates/libafl/src/stages/verify_timeouts.rs @@ -10,7 +10,7 @@ use libafl_bolts::Error; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ - Evaluator, HasMetadata, + Evaluator, HasMetadataMut, executors::{Executor, HasObservers, HasTimeout}, inputs::BytesInput, observers::ObserversTuple, @@ -84,7 +84,7 @@ where E::Observers: ObserversTuple, E: Executor + HasObservers + HasTimeout, Z: Evaluator, - S: HasMetadata, + S: HasMetadataMut, I: Debug + Serialize + DeserializeOwned + Default + 'static + Clone, { fn perform( diff --git a/crates/libafl/src/state/mod.rs b/crates/libafl/src/state/mod.rs index fbbaa00c794..10a1c493b7e 100644 --- a/crates/libafl/src/state/mod.rs +++ b/crates/libafl/src/state/mod.rs @@ -2,13 +2,7 @@ #[cfg(feature = "std")] use alloc::vec::Vec; -use core::{ - borrow::BorrowMut, - cell::{Ref, RefMut}, - fmt::Debug, - marker::PhantomData, - time::Duration, -}; +use core::{fmt::Debug, marker::PhantomData, time::Duration}; #[cfg(feature = "std")] use std::{ fs, @@ -24,15 +18,12 @@ use libafl_bolts::{ }; use serde::{Deserialize, Serialize, de::DeserializeOwned}; -mod stack; -pub use stack::StageStack; - #[cfg(feature = "std")] use crate::fuzzer::ExecuteInputResult; #[cfg(feature = "introspection")] use crate::monitors::stats::ClientPerfStats; use crate::{ - Error, HasMetadata, HasNamedMetadata, + Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, corpus::{Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, InMemoryCorpus, Testcase}, events::{Event, EventFirer, EventWithStats, LogSeverity}, feedbacks::StateInitializer, @@ -41,6 +32,10 @@ use crate::{ inputs::{Input, NopInput}, stages::StageId, }; + +mod stack; +pub use stack::StageStack; + /// The maximum size of a testcase pub const DEFAULT_MAX_SIZE: usize = 1_048_576; @@ -51,6 +46,7 @@ pub trait HasCorpus { /// The testcase corpus fn corpus(&self) -> &Self::Corpus; + /// The testcase corpus (mutable) fn corpus_mut(&mut self) -> &mut Self::Corpus; } @@ -160,6 +156,7 @@ pub trait HasStartTime { /// The starting time (mutable) fn start_time_mut(&mut self) -> &mut Duration; + /// Get the time elapsed since start. fn time_since_start(&self) -> Duration { let start_time = self.start_time(); let current_time = current_time(); @@ -302,15 +299,12 @@ where C: Corpus, { /// To get the testcase - fn testcase(&self, id: CorpusId) -> Result>, Error> { - let tc = self.corpus().get(id)?; - Ok(tc.borrow()) - } - - /// To get mutable testcase - fn testcase_mut(&self, id: CorpusId) -> Result>, Error> { + fn testcase( + &self, + id: CorpusId, + ) -> Result>::TestcaseMetadataCell>, Error> { let tc = self.corpus().get(id)?; - Ok(tc.borrow_mut()) + Ok(tc) } } @@ -341,7 +335,9 @@ impl HasMetadata for StdState { fn metadata_map(&self) -> &SerdeAnyMap { &self.metadata } +} +impl HasMetadataMut for StdState { /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) #[inline] fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { @@ -355,7 +351,9 @@ impl HasNamedMetadata for StdState { fn named_metadata_map(&self) -> &NamedSerdeAnyMap { &self.named_metadata } +} +impl HasNamedMetadataMut for StdState { /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) #[inline] fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap { @@ -464,14 +462,9 @@ pub trait HasCurrentTestcase: HasCorpus { /// Gets the current [`Testcase`] we are fuzzing /// /// Will return [`Error::key_not_found`] if no `corpus_id` is currently set. - fn current_testcase(&self) -> Result>, Error>; - //fn current_testcase(&self) -> Result<&Testcase, Error>; - - /// Gets the current [`Testcase`] we are fuzzing (mut) - /// - /// Will return [`Error::key_not_found`] if no `corpus_id` is currently set. - fn current_testcase_mut(&self) -> Result>, Error>; - //fn current_testcase_mut(&self) -> Result<&mut Testcase, Error>; + fn current_testcase( + &self, + ) -> Result>::TestcaseMetadataCell>, Error>; /// Gets a cloned representation of the current [`Testcase`]. /// @@ -488,29 +481,21 @@ where T: HasCorpus + HasCurrentCorpusId, I: Clone, { - fn current_testcase(&self) -> Result>, Error> { + fn current_testcase( + &self, + ) -> Result>::TestcaseMetadataCell>, Error> { let Some(corpus_id) = self.current_corpus_id()? else { return Err(Error::key_not_found( "We are not currently processing a testcase", )); }; - Ok(self.corpus().get(corpus_id)?.borrow()) - } - - fn current_testcase_mut(&self) -> Result>, Error> { - let Some(corpus_id) = self.current_corpus_id()? else { - return Err(Error::illegal_state( - "We are not currently processing a testcase", - )); - }; - - Ok(self.corpus().get(corpus_id)?.borrow_mut()) + self.corpus().get(corpus_id) } fn current_input_cloned(&self) -> Result { - let mut testcase = self.current_testcase_mut()?; - Ok(testcase.borrow_mut().load_input(self.corpus())?.clone()) + let testcase = self.current_testcase()?; + Ok(testcase.cloned_input()) } } @@ -1243,7 +1228,10 @@ impl HasMaxSize for NopState { } } -impl HasCorpus for NopState { +impl HasCorpus for NopState +where + I: Input, +{ type Corpus = InMemoryCorpus; fn corpus(&self) -> &Self::Corpus { @@ -1293,7 +1281,9 @@ impl HasMetadata for NopState { fn metadata_map(&self) -> &SerdeAnyMap { &self.metadata } +} +impl HasMetadataMut for NopState { fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { &mut self.metadata } @@ -1303,7 +1293,9 @@ impl HasNamedMetadata for NopState { fn named_metadata_map(&self) -> &NamedSerdeAnyMap { &self.named_metadata } +} +impl HasNamedMetadataMut for NopState { fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap { &mut self.named_metadata } diff --git a/crates/libafl_libfuzzer/runtime/src/merge.rs b/crates/libafl_libfuzzer/runtime/src/merge.rs index a042d4d6333..aacf7ac7048 100644 --- a/crates/libafl_libfuzzer/runtime/src/merge.rs +++ b/crates/libafl_libfuzzer/runtime/src/merge.rs @@ -211,7 +211,7 @@ pub fn merge( } for id in fuzzer.scheduler().current().clone() { - let mut testcase = state.corpus_mut().get(id)?.borrow_mut(); + let mut testcase = state.corpus().get(id)?; let file_path = testcase .file_path_mut() .as_mut() diff --git a/fuzzers/baby/tutorial/src/metadata.rs b/fuzzers/baby/tutorial/src/metadata.rs index 1a53a65b305..2bb2ce61fe6 100644 --- a/fuzzers/baby/tutorial/src/metadata.rs +++ b/fuzzers/baby/tutorial/src/metadata.rs @@ -21,9 +21,12 @@ pub struct PacketLenTestcaseScore {} impl TestcaseScore for PacketLenTestcaseScore where - S: HasMetadata, + S: HasCorpus + HasMetadata, { - fn compute(_state: &S, entry: &mut Testcase) -> Result { + fn compute( + _state: &S, + entry: &mut Testcase>::TestcaseMetadataRefMut<'_>>, + ) -> Result { Ok(entry .metadata_map() .get::() diff --git a/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs b/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs index 13d6ff34128..4746d03eb2d 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs @@ -78,7 +78,7 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _md: &mut TestcaseMetadata, ) -> Result<(), Error> { if self.should_run() { let file_path = testcase diff --git a/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs b/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs index 89aec0b462e..04b43273d88 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs @@ -93,7 +93,7 @@ where state: &mut S, manager: &mut EM, observers: &OT, - testcase: &mut Testcase, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.inner .append_metadata(state, manager, observers, testcase)?; diff --git a/utils/libafl_repo_tools/Cargo.lock b/utils/libafl_repo_tools/Cargo.lock index d9716abf7d4..615745c5298 100644 --- a/utils/libafl_repo_tools/Cargo.lock +++ b/utils/libafl_repo_tools/Cargo.lock @@ -212,7 +212,7 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "libafl_repo_tools" -version = "0.15.3" +version = "0.15.4" dependencies = [ "clap", "colored", From 2918caff9fa086be3346acda5a63dc5c0efd0ff8 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Wed, 17 Sep 2025 15:01:41 +0200 Subject: [PATCH 03/12] merge fix --- Cargo.lock | 4 +- crates/libafl/src/corpus/dynamic.rs | 48 ++++++++++--------- crates/libafl/src/corpus/mod.rs | 4 +- crates/libafl/src/fuzzer/mod.rs | 31 +++++++----- .../libafl/src/schedulers/testcase_score.rs | 2 +- crates/libafl/src/stages/calibrate.rs | 2 +- crates/libafl/src/state/mod.rs | 21 ++++---- 7 files changed, 58 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbbc08f88f0..fe28bb07a30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -649,9 +649,9 @@ dependencies = [ [[package]] name = "bitbybit" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb157f9753a7cddfcf4a4f5fed928fbf4ce1b7b64b6bcc121d7a9f95d698997b" +checksum = "ec187a89ab07e209270175faf9e07ceb2755d984954e58a2296e325ddece2762" dependencies = [ "arbitrary-int", "proc-macro2", diff --git a/crates/libafl/src/corpus/dynamic.rs b/crates/libafl/src/corpus/dynamic.rs index c20fa863296..bf732608be6 100644 --- a/crates/libafl/src/corpus/dynamic.rs +++ b/crates/libafl/src/corpus/dynamic.rs @@ -6,7 +6,7 @@ use core::marker::PhantomData; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; -use crate::corpus::{Corpus, CorpusId, Testcase}; +use crate::corpus::{Corpus, CorpusId, Testcase, TestcaseMetadata}; /// An dynamic corpus type accepting two types of corpus at runtime. This helps rustc better /// reason about the bounds compared to dyn objects. @@ -39,17 +39,17 @@ where C1: Corpus, C2: Corpus, { - fn add(&mut self, testcase: Testcase) -> Result { + fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result { match self { - Self::Corpus1(c1, _) => c1.add(testcase), - Self::Corpus2(c2, _) => c2.add(testcase), + Self::Corpus1(c1, _) => c1.add(input, md), + Self::Corpus2(c2, _) => c2.add(input, md), } } - fn add_disabled(&mut self, testcase: Testcase) -> Result { + fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result { match self { - Self::Corpus1(c1, _) => c1.add_disabled(testcase), - Self::Corpus2(c2, _) => c2.add_disabled(testcase), + Self::Corpus1(c1, _) => c1.add_disabled(input, md), + Self::Corpus2(c2, _) => c2.add_disabled(input, md), } } @@ -95,17 +95,14 @@ where } } - fn get(&self, id: CorpusId) -> Result>, Error> { + /// Get testcase by id + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { match self { - Self::Corpus1(c1, _) => c1.get(id), - Self::Corpus2(c2, _) => c2.get(id), - } - } - - fn get_from_all(&self, id: CorpusId) -> Result>, Error> { - match self { - Self::Corpus1(c1, _) => c1.get_from_all(id), - Self::Corpus2(c2, _) => c2.get_from_all(id), + Self::Corpus1(c1, _) => c1.get_from(id), + Self::Corpus2(c2, _) => c2.get_from(id), } } @@ -151,10 +148,15 @@ where } } - // fn replace(&mut self, idx: CorpusId, testcase: Testcase) -> Result, Error> { - // match self { - // Self::Corpus1(c1, _) => c1.replace(idx, testcase), - // Self::Corpus2(c2, _) => c2.replace(idx, testcase), - // } - // } + fn replace( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result, Error> { + match self { + Self::Corpus1(c1, _) => c1.replace(id, input, md), + Self::Corpus2(c2, _) => c2.replace(id, input, md), + } + } } diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index 84b74bf4b3d..5de8b6d9516 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -19,8 +19,8 @@ pub use cache::{Cache, FifoCache, IdentityCache}; pub mod single; pub use single::SingleCorpus; -pub mod dynamic; -pub use dynamic::DynamicCorpus; +// pub mod dynamic; +// pub use dynamic::DynamicCorpus; pub mod combined; pub use combined::CombinedCorpus; diff --git a/crates/libafl/src/fuzzer/mod.rs b/crates/libafl/src/fuzzer/mod.rs index a0aeb929620..e4c814059e9 100644 --- a/crates/libafl/src/fuzzer/mod.rs +++ b/crates/libafl/src/fuzzer/mod.rs @@ -13,10 +13,9 @@ use serde::{Serialize, de::DeserializeOwned}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{ - Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, - testcase::{HasTestcaseMetadata, TestcaseMetadata}, + Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, HasTestcaseMetadata, TestcaseMetadata, }, events::{ Event, EventConfig, EventFirer, EventReceiver, EventWithStats, ProgressReporter, @@ -424,30 +423,34 @@ where ExecuteInputResult::Corpus => { // Not a solution // Add the input to the main corpus - let mut testcase = Testcase::from(input.clone()); + let mut md = TestcaseMetadata::default(); + #[cfg(feature = "track_hit_feedbacks")] self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + .append_hit_feedbacks(md.hit_feedbacks_mut())?; self.feedback_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - let id = state.corpus_mut().add(testcase)?; + .append_metadata(state, manager, observers, &mut md)?; + + let id = state.corpus_mut().add(Rc::new(input.clone()), md)?; self.scheduler_mut().on_add(state, id)?; Ok(Some(id)) } ExecuteInputResult::Solution => { // The input is a solution, add it to the respective corpus - let mut testcase = Testcase::from(input.clone()); - testcase.set_parent_id_optional(*state.corpus().current()); - if let Ok(mut tc) = state.current_testcase_mut() { + let mut md = TestcaseMetadata::default(); + + md.set_parent_id_optional(*state.corpus().current()); + + if let Ok(mut tc) = state.current_testcase() { tc.found_objective(); } #[cfg(feature = "track_hit_feedbacks")] self.objective_mut() .append_hit_feedbacks(testcase.hit_objectives_mut())?; self.objective_mut() - .append_metadata(state, manager, observers, &mut testcase)?; - state.solutions_mut().add(testcase)?; + .append_metadata(state, manager, observers, &mut md)?; + state.solutions_mut().add(Rc::new(input.clone()), md)?; Ok(None) } @@ -726,7 +729,9 @@ where self.objective_mut() .append_metadata(state, manager, &*observers, &mut tc_md)?; // we don't care about solution id - let id = state.solutions_mut().add(Rc::new(input.clone()), tc_md.clone())?; + let id = state + .solutions_mut() + .add(Rc::new(input.clone()), tc_md.clone())?; manager.fire( state, diff --git a/crates/libafl/src/schedulers/testcase_score.rs b/crates/libafl/src/schedulers/testcase_score.rs index c6967099c52..f52fa649762 100644 --- a/crates/libafl/src/schedulers/testcase_score.rs +++ b/crates/libafl/src/schedulers/testcase_score.rs @@ -27,7 +27,7 @@ where /// Compute the favor factor of a [`Testcase`]. Lower is better. pub trait TestcasePenalty { /// Computes the favor factor of a [`Testcase`]. Higher is better. - fn compute(state: &S, entry: &mut Testcase) -> Result; + fn compute(state: &S, corpus_id: CorpusId) -> Result; } /// Multiply the testcase size with the execution time. diff --git a/crates/libafl/src/stages/calibrate.rs b/crates/libafl/src/stages/calibrate.rs index 209ce2ff217..ff0d3fd4caf 100644 --- a/crates/libafl/src/stages/calibrate.rs +++ b/crates/libafl/src/stages/calibrate.rs @@ -13,7 +13,7 @@ use num_traits::Bounded; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, + Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, HasScheduler, corpus::{Corpus, HasCurrentCorpusId, HasTestcaseMetadata, SchedulerTestcaseMetadata}, events::{Event, EventFirer, EventWithStats, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, diff --git a/crates/libafl/src/state/mod.rs b/crates/libafl/src/state/mod.rs index b4e2af1cc60..76d39065da3 100644 --- a/crates/libafl/src/state/mod.rs +++ b/crates/libafl/src/state/mod.rs @@ -9,6 +9,15 @@ use std::{ path::{Path, PathBuf}, }; +#[cfg(feature = "std")] +use libafl_bolts::core_affinity::{CoreId, Cores}; +use libafl_bolts::{ + current_time, + rands::{Rand, StdRand}, + serdeany::{NamedSerdeAnyMap, SerdeAnyMap}, +}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; + #[cfg(feature = "introspection")] use crate::monitors::stats::ClientPerfStats; use crate::{ @@ -22,18 +31,6 @@ use crate::{ stages::StageId, }; -#[cfg(feature = "std")] -use libafl_bolts::core_affinity::{CoreId, Cores}; -use libafl_bolts::{ - current_time, - rands::{Rand, StdRand}, - serdeany::{NamedSerdeAnyMap, SerdeAnyMap}, -}; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; - -mod stack; -pub use stack::StageStack; - mod stack; pub use stack::StageStack; From e0899f4e8ddbc573e2e336cad87550689d12727c Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 11:36:26 +0200 Subject: [PATCH 04/12] create a cache cell. rename testcase metadata cell trait. cache dirty cell writeback implementation. implemented cache policies. --- crates/libafl/src/corpus/cache.rs | 305 ++++++++++++++---- crates/libafl/src/corpus/combined.rs | 18 +- crates/libafl/src/corpus/mod.rs | 23 +- crates/libafl/src/corpus/nop.rs | 5 +- crates/libafl/src/corpus/single.rs | 7 +- crates/libafl/src/corpus/store/inmemory.rs | 14 +- crates/libafl/src/corpus/store/maps.rs | 50 +-- crates/libafl/src/corpus/store/mod.rs | 9 +- crates/libafl/src/corpus/store/ondisk.rs | 94 +++--- crates/libafl/src/corpus/testcase.rs | 69 ++-- crates/libafl/src/executors/inprocess/mod.rs | 2 +- crates/libafl/src/fuzzer/mod.rs | 2 +- crates/libafl/src/inputs/generalized.rs | 4 +- crates/libafl/src/mutators/gramatron.rs | 2 +- crates/libafl/src/mutators/grimoire.rs | 2 +- crates/libafl/src/mutators/scheduled.rs | 2 +- crates/libafl/src/schedulers/accounting.rs | 2 +- crates/libafl/src/schedulers/minimizer.rs | 4 +- crates/libafl/src/schedulers/mod.rs | 4 +- .../src/schedulers/probabilistic_sampling.rs | 4 +- crates/libafl/src/schedulers/queue.rs | 2 +- .../libafl/src/schedulers/testcase_score.rs | 2 +- crates/libafl/src/schedulers/weighted.rs | 2 +- crates/libafl/src/stages/afl_stats.rs | 11 +- crates/libafl/src/stages/calibrate.rs | 2 +- crates/libafl/src/stages/concolic.rs | 2 +- crates/libafl/src/stages/generalization.rs | 2 +- crates/libafl/src/stages/mutational.rs | 6 +- crates/libafl/src/stages/tmin.rs | 5 +- 29 files changed, 437 insertions(+), 219 deletions(-) diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index d1f2a62e444..d3db46db754 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -10,28 +10,52 @@ use std::{cell::RefCell, collections::VecDeque, marker::PhantomData}; use libafl_bolts::Error; -use crate::corpus::{ - CorpusId, Testcase, - store::{RemovableStore, Store}, - testcase::{HasTestcaseMetadata, TestcaseMetadata}, +use crate::{ + corpus::{ + CorpusId, Testcase, + maps::InMemoryCorpusMap, + store::{RemovableStore, Store}, + testcase::{IsTestcaseMetadataCell, TestcaseMetadata}, + }, + inputs::Input, }; -/// A First In -> First Out cache policy. +/// Describes how a change to metadata should be propagated to the fallback store #[derive(Debug)] -pub struct FifoCache { - cached_ids: VecDeque, - cache_max_len: usize, - phantom: PhantomData<(I, CS, FS)>, +pub enum CachePolicy { + /// Propagate the changes when the cell gets dropped. + /// Expect more writes to the fallback store, with + WritebackOnDrop, + /// Propagate the changes when the cache is flushed explicitly. + /// Less writes to the fallback stores will be issued, but the used is responsible for + /// flushing the cache regularly, to avoid data loss. + WritebackOnFlush, } -/// An identity cache, storing everything both in the cache and the backing store. +/// Describe a cache policy +pub trait HasCachePolicy { + /// The cache policy + const CACHE_POLICY: CachePolicy; +} + +/// An implementor for [`CachePolicy::WritebackOnDrop`] #[derive(Debug)] -pub struct IdentityCache; +pub struct WritebackOnDropPolicy; +impl HasCachePolicy for WritebackOnDropPolicy { + const CACHE_POLICY: CachePolicy = CachePolicy::WritebackOnDrop; +} + +/// An implementor for [`CachePolicy::WritebackOnFlush`] +#[derive(Debug)] +pub struct WritebackOnFlushPolicy; +impl HasCachePolicy for WritebackOnFlushPolicy { + const CACHE_POLICY: CachePolicy = CachePolicy::WritebackOnFlush; +} /// A cache, managing a cache store and a fallback store. -pub trait Cache { +pub trait Cache { /// A [`TestcaseMetadata`] cell. - type TestcaseMetadataCell: HasTestcaseMetadata; + type TestcaseMetadataCell: IsTestcaseMetadataCell; /// Add a testcase to the cache fn add( @@ -62,14 +86,13 @@ pub trait Cache { ) -> Result, Error>; /// Replace a testcase in the cache - fn replace( + fn replace_metadata( &mut self, id: CorpusId, - input: Rc, md: TestcaseMetadata, cache_store: &mut CS, fallback_store: &mut FS, - ) -> Result, Error>; + ) -> Result; /// Flush the cache, committing the cached testcase to the fallback store. fn flush( @@ -78,58 +101,128 @@ pub trait Cache { cache_store: &mut CS, fallback_store: &mut FS, ) -> Result<(), Error>; + + /// Mark a corpus entry as written explicitly, for subsequent flushes. + /// + /// Thus, a cache [`Self::flush`] should propagate to entries marked as [`Self::written`]. + fn written(&mut self, id: CorpusId); } -impl FifoCache +/// A composed testcase metadata cell, linking the cached cell with the fallback cell. +#[derive(Debug, Clone)] +pub struct CacheTestcaseMetadataCell where - CS: RemovableStore, - FS: Store, - I: Clone, + CC: IsTestcaseMetadataCell, + FC: IsTestcaseMetadataCell, { - fn get_inner( - &mut self, - id: CorpusId, - cache_store: &mut CS, - fallback_store: &FS, - cache_add_fn: CAF, - cache_get_fn: CGF, - cache_rm_fn: CRF, - fallback_get_fn: FGF, - ) -> Result, Error> + write_access: RefCell, + cache_cell: CC, + fallback_cell: FC, +} + +/// An identity cache, storing everything both in the cache and the backing store. +#[derive(Debug)] +pub struct IdentityCache { + cell_map: M, +} + +/// A `First In / First Out` cache policy. +#[derive(Debug)] +pub struct FifoCache { + cached_ids: VecDeque, + cache_max_len: usize, + phantom: PhantomData<(I, CS, FS)>, +} + +impl CacheTestcaseMetadataCell +where + CC: IsTestcaseMetadataCell, + FC: IsTestcaseMetadataCell, +{ + /// Create a new [`CacheTestcaseMetadataCell`] + pub fn new(cache_cell: CC, fallback_cell: FC) -> Self { + Self { + write_access: RefCell::new(false), + cache_cell, + fallback_cell, + } + } +} + +impl IsTestcaseMetadataCell for CacheTestcaseMetadataCell +where + CC: IsTestcaseMetadataCell, + FC: IsTestcaseMetadataCell, +{ + type TestcaseMetadataRef<'a> + = CC::TestcaseMetadataRef<'a> where - CAF: FnOnce(&mut CS, CorpusId, Testcase>) -> Result<(), Error>, - CGF: FnOnce(&CS, CorpusId) -> Result, Error>, - CRF: FnOnce(&mut CS, CorpusId) -> Result, Error>, - FGF: FnOnce(&FS, CorpusId) -> Result, Error>, - { - if self.cached_ids.contains(&id) { - cache_get_fn(cache_store, id) - } else { - if self.cached_ids.len() == self.cache_max_len { - let to_evict = self.cached_ids.pop_back().unwrap(); - cache_rm_fn(cache_store, to_evict)?; - } + Self: 'a; + type TestcaseMetadataRefMut<'a> + = CC::TestcaseMetadataRefMut<'a> + where + Self: 'a; - debug_assert!(self.cached_ids.len() < self.cache_max_len); + fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + self.cache_cell.testcase_metadata() + } - // tescase is not cached, fetch it from fallback - let fb_tc = fallback_get_fn(&fallback_store, id)?.cloned(); - cache_add_fn(cache_store, id, fb_tc)?; + fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + *self.write_access.borrow_mut() = true; + self.cache_cell.testcase_metadata_mut() + } - self.cached_ids.push_front(id); + fn into_testcase_metadata(self) -> TestcaseMetadata { + self.cache_cell.testcase_metadata().clone() + } - cache_get_fn(cache_store, id) + fn replace_testcase_metadata(&self, testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + self.fallback_cell + .replace_testcase_metadata(testcase_metadata.clone()); + self.cache_cell.replace_testcase_metadata(testcase_metadata) + } + + fn flush(&self) -> Result<(), Error> { + let write_access = self.write_access.borrow_mut(); + + if *write_access { + *self.fallback_cell.testcase_metadata_mut() = + self.cache_cell.testcase_metadata().clone(); + self.fallback_cell.flush()?; + *self.write_access.borrow_mut() = false; } + + Ok(()) } } -impl Cache for IdentityCache +impl Drop for CacheTestcaseMetadataCell +where + CC: IsTestcaseMetadataCell, + FC: IsTestcaseMetadataCell, +{ + fn drop(&mut self) { + self.flush().unwrap(); + } +} + +impl Cache for IdentityCache where CS: RemovableStore, FS: Store, - I: Clone, + I: Input, + M: InMemoryCorpusMap< + Testcase< + I, + Rc>, + >, + >, + P: HasCachePolicy, + >::TestcaseMetadataCell: Clone, + >::TestcaseMetadataCell: Clone, { - type TestcaseMetadataCell = CS::TestcaseMetadataCell; + type TestcaseMetadataCell = + Rc>; fn add( &mut self, @@ -159,23 +252,45 @@ where &mut self, id: CorpusId, cache_store: &mut CS, - _fallback_store: &FS, + fallback_store: &FS, ) -> Result, Error> { - cache_store.get_from::(id) + if let Some(tc) = self.cell_map.get(id) { + Ok(tc.clone()) + } else { + let (input, cc) = cache_store.get_from::(id)?.into_inner(); + let (_, fc) = fallback_store.get_from::(id)?.into_inner(); + + let cache_cell = Rc::new(CacheTestcaseMetadataCell::new(cc, fc)); + let testcase = Testcase::new(input, cache_cell.clone()); + + self.cell_map.add(id, testcase.clone()); + + Ok(testcase) + } } - fn replace( - &mut self, - id: CorpusId, - input: Rc, - md: TestcaseMetadata, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result, Error> { - let old_tc = cache_store.replace(id, input.clone(), md.clone())?; - fallback_store.replace(id, input, md)?; + // fn replace( + // &mut self, + // id: CorpusId, + // input: Rc, + // md: TestcaseMetadata, + // cache_store: &mut CS, + // fallback_store: &mut FS, + // ) -> Result, Error> { + // let old_tc = cache_store.replace(id, input.clone(), md.clone())?; + // fallback_store.replace(id, input, md)?; - Ok(old_tc) + // Ok(old_tc) + // } + + fn replace_metadata( + &mut self, + _id: CorpusId, + _md: TestcaseMetadata, + _cache_store: &mut CS, + _fallback_store: &mut FS, + ) -> Result { + todo!() } fn flush( @@ -186,9 +301,56 @@ where ) -> Result<(), Error> { todo!() } + + fn written(&mut self, _id: CorpusId) { + todo!() + } } -impl Cache for FifoCache +impl FifoCache +where + CS: RemovableStore, + FS: Store, + I: Clone, +{ + fn get_inner( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &FS, + cache_add_fn: CAF, + cache_get_fn: CGF, + cache_rm_fn: CRF, + fallback_get_fn: FGF, + ) -> Result, Error> + where + CAF: FnOnce(&mut CS, CorpusId, Testcase>) -> Result<(), Error>, + CGF: FnOnce(&CS, CorpusId) -> Result, Error>, + CRF: FnOnce(&mut CS, CorpusId) -> Result, Error>, + FGF: FnOnce(&FS, CorpusId) -> Result, Error>, + { + if self.cached_ids.contains(&id) { + cache_get_fn(cache_store, id) + } else { + if self.cached_ids.len() == self.cache_max_len { + let to_evict = self.cached_ids.pop_back().unwrap(); + cache_rm_fn(cache_store, to_evict)?; + } + + debug_assert!(self.cached_ids.len() < self.cache_max_len); + + // tescase is not cached, fetch it from fallback + let fb_tc = fallback_get_fn(&fallback_store, id)?.cloned(); + cache_add_fn(cache_store, id, fb_tc)?; + + self.cached_ids.push_front(id); + + cache_get_fn(cache_store, id) + } + } +} + +impl Cache for FifoCache where CS: RemovableStore, FS: Store, @@ -230,7 +392,7 @@ where fallback_store, |cache_store, corpus_id, testcase| { let (input, md) = testcase.into_inner(); - cache_store.add(corpus_id, input, md) + cache_store.add(corpus_id, input, md.into_testcase_metadata()) }, |cache_store, corpus_id| cache_store.get(corpus_id), |cache_store, corpus_id| cache_store.remove(corpus_id), @@ -238,14 +400,13 @@ where ) } - fn replace( + fn replace_metadata( &mut self, _id: CorpusId, - _input: Rc, - _metadata: TestcaseMetadata, + _md: TestcaseMetadata, _cache_store: &mut CS, _fallback_store: &mut FS, - ) -> Result, Error> { + ) -> Result { todo!() } @@ -257,4 +418,8 @@ where ) -> Result<(), Error> { todo!() } + + fn written(&mut self, _id: CorpusId) { + todo!() + } } diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index 85e2574f36b..852c7c02e99 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -13,7 +13,7 @@ use crate::corpus::testcase::TestcaseMetadata; /// A [`CombinedCorpus`] tries first to use the main store according to some policy. /// If it fails, it falls back to the secondary store. #[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct CombinedCorpus { +pub struct CombinedCorpus { /// The cache store cache_store: RefCell, /// The fallback store @@ -26,17 +26,17 @@ pub struct CombinedCorpus { keys: Vec, /// The current ID current: Option, - phantom: PhantomData, + phantom: PhantomData<(I, P)>, } -impl Corpus for CombinedCorpus +impl Corpus for CombinedCorpus where - C: Cache, + C: Cache, CS: Store, FS: Store, I: Clone, { - type TestcaseMetadataCell = CS::TestcaseMetadataCell; + type TestcaseMetadataCell = C::TestcaseMetadataCell; fn count(&self) -> usize { self.fallback_store.count() @@ -88,15 +88,13 @@ where cache.get_from::(id, cache_store, &self.fallback_store) } - fn replace( + fn replace_metadata( &mut self, id: CorpusId, - input: Rc, md: TestcaseMetadata, - ) -> Result, Error> { - self.cache.borrow_mut().replace( + ) -> Result { + self.cache.borrow_mut().replace_metadata( id, - input, md, &mut *self.cache_store.borrow_mut(), &mut self.fallback_store, diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index 5de8b6d9516..c1e67a2782e 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -10,7 +10,7 @@ use crate::Error; pub mod testcase; pub use testcase::{ - HasTestcase, HasTestcaseMetadata, SchedulerTestcaseMetadata, Testcase, TestcaseMetadata, + HasTestcase, IsTestcaseMetadataCell, SchedulerTestcaseMetadata, Testcase, TestcaseMetadata, }; pub mod cache; @@ -36,6 +36,8 @@ pub use nop::NopCorpus; pub mod store; pub use store::{InMemoryStore, OnDiskStore, maps}; +use crate::corpus::cache::WritebackOnFlushPolicy; + /// The standard fully in-memory corpus map. #[cfg(not(feature = "corpus_btreemap"))] pub type StdInMemoryCorpusMap = maps::HashCorpusMap>>>; @@ -59,8 +61,13 @@ pub type InMemoryCorpus = SingleCorpus>; pub type OnDiskCorpus = SingleCorpus>>; /// The standard corpus for storing on disk and in-memory. -pub type InMemoryOnDiskCorpus = - CombinedCorpus, StdOnDiskStore, I>; +pub type InMemoryOnDiskCorpus = CombinedCorpus< + IdentityCache>, + StdInMemoryStore, + StdOnDiskStore, + I, + WritebackOnFlushPolicy, +>; /// The standard corpus for storing on disk and in-memory with a cache. /// Useful for very large corpuses. @@ -69,6 +76,7 @@ pub type CachedOnDiskCorpus = CombinedCorpus< StdInMemoryStore, StdOnDiskStore, I, + WritebackOnFlushPolicy, >; /// An abstraction for the index that identify a testcase in the corpus @@ -134,7 +142,7 @@ macro_rules! random_corpus_id_with_disabled { /// Corpus with all current [`Testcase`]s, or solutions pub trait Corpus: Sized { /// A [`TestcaseMetadata`] cell. - type TestcaseMetadataCell: HasTestcaseMetadata; + type TestcaseMetadataCell: IsTestcaseMetadataCell; /// Returns the number of all enabled entries fn count(&self) -> usize; @@ -172,13 +180,12 @@ pub trait Corpus: Sized { id: CorpusId, ) -> Result, Error>; - /// Replace a [`Testcase`] by another one. - fn replace( + /// Replace a [`TestcaseMetadata`] by another one. + fn replace_metadata( &mut self, id: CorpusId, - input: Rc, md: TestcaseMetadata, - ) -> Result, Error>; + ) -> Result; /// Current testcase scheduled fn current(&self) -> &Option; diff --git a/crates/libafl/src/corpus/nop.rs b/crates/libafl/src/corpus/nop.rs index 5af696f9bb8..cb227f466e7 100644 --- a/crates/libafl/src/corpus/nop.rs +++ b/crates/libafl/src/corpus/nop.rs @@ -103,12 +103,11 @@ impl Corpus for NopCorpus { CorpusId::from(0_usize) } - fn replace( + fn replace_metadata( &mut self, _id: CorpusId, - _input: Rc, _md: TestcaseMetadata, - ) -> Result, Error> { + ) -> Result { Err(Error::unsupported("Unsupported by NopCorpus")) } } diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 86c3537d2f8..7fa4db9cc5f 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -90,13 +90,12 @@ where self.store.get_from::(id) } - fn replace( + fn replace_metadata( &mut self, id: CorpusId, - input: Rc, md: TestcaseMetadata, - ) -> Result, Error> { - self.store.replace(id, input, md) + ) -> Result { + self.store.replace_metadata(id, md) } fn current(&self) -> &Option { diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index 92bff8c1d4b..ddef0fbad78 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -95,16 +95,12 @@ where } } - fn replace( + fn replace_metadata( &mut self, - id: CorpusId, - input: Rc, - metadata: TestcaseMetadata, - ) -> Result, Error> { - Ok(self - .enabled_map - .replace(id, Testcase::new(input, TMC::instantiate(metadata))) - .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?) + _id: CorpusId, + _metadata: TestcaseMetadata, + ) -> Result { + todo!() } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/store/maps.rs b/crates/libafl/src/corpus/store/maps.rs index bd0d6941aa6..8f606c0c3a8 100644 --- a/crates/libafl/src/corpus/store/maps.rs +++ b/crates/libafl/src/corpus/store/maps.rs @@ -5,7 +5,7 @@ use std::{collections::BTreeMap, vec::Vec}; use num_traits::Zero; use serde::{Deserialize, Serialize}; -use crate::corpus::CorpusId; +use crate::corpus::{CorpusId, IsTestcaseMetadataCell, TestcaseMetadata}; /// A trait implemented by in-memory corpus maps pub trait InMemoryCorpusMap { @@ -26,8 +26,12 @@ pub trait InMemoryCorpusMap { /// Get by id; considers only enabled testcases fn get_mut(&mut self, id: CorpusId) -> Option<&mut T>; - /// Replace a testcase by another one - fn replace(&mut self, id: CorpusId, testcase: T) -> Option; + /// Replace the metadata of a given testcase + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option; /// Get the prev corpus id in chronological order fn prev(&self, id: CorpusId) -> Option; @@ -125,7 +129,10 @@ impl CorpusIdHistory { } } -impl InMemoryCorpusMap for HashCorpusMap { +impl InMemoryCorpusMap for HashCorpusMap +where + T: IsTestcaseMetadataCell, +{ fn count(&self) -> usize { self.map.len() } @@ -168,16 +175,13 @@ impl InMemoryCorpusMap for HashCorpusMap { self.map.get_mut(&id).map(|storage| &mut storage.testcase) } - fn replace(&mut self, id: CorpusId, testcase: T) -> Option { + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option { let old_tc = self.map.get_mut(&id)?; - - let new_tc = TestcaseStorageItem { - testcase, - prev: old_tc.prev, - next: old_tc.next, - }; - - self.map.insert(id, new_tc).map(|storage| storage.testcase) + Some(old_tc.testcase.replace_testcase_metadata(testcase_metadata)) } fn prev(&self, id: CorpusId) -> Option { @@ -207,7 +211,10 @@ impl InMemoryCorpusMap for HashCorpusMap { } } -impl InMemoryCorpusMap for BtreeCorpusMap { +impl InMemoryCorpusMap for BtreeCorpusMap +where + T: IsTestcaseMetadataCell, +{ fn count(&self) -> usize { self.map.len() } @@ -222,12 +229,6 @@ impl InMemoryCorpusMap for BtreeCorpusMap { self.history.add(id); } - // fn replace(&mut self, id: CorpusId, new_testcase: T) -> Option { - // self.map - // .get_mut(&id) - // .map(|entry| entry.replace(new_testcase)) - // } - fn get(&self, id: CorpusId) -> Option<&T> { self.map.get(&id) } @@ -236,8 +237,13 @@ impl InMemoryCorpusMap for BtreeCorpusMap { self.map.get_mut(&id) } - fn replace(&mut self, id: CorpusId, testcase: T) -> Option { - self.map.insert(id, testcase) + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option { + let tc = self.get_mut(id)?; + Some(tc.replace_testcase_metadata(testcase_metadata)) } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index 0ca252b0dc2..81b4ed85c92 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -5,7 +5,7 @@ use alloc::rc::Rc; use libafl_bolts::Error; use super::{CorpusId, Testcase}; -use crate::corpus::testcase::{HasTestcaseMetadata, TestcaseMetadata}; +use crate::corpus::testcase::{IsTestcaseMetadataCell, TestcaseMetadata}; pub mod maps; pub use maps::{BtreeCorpusMap, HashCorpusMap, InMemoryCorpusMap}; @@ -19,7 +19,7 @@ pub use ondisk::{OnDiskMetadataFormat, OnDiskStore}; /// A store is responsible for storing and retrieving [`Testcase`]s, ordered by add time. pub trait Store { /// A [`TestcaseMetadata`] cell. - type TestcaseMetadataCell: HasTestcaseMetadata; + type TestcaseMetadataCell: IsTestcaseMetadataCell; /// Returns the number of all enabled entries fn count(&self) -> usize; @@ -65,12 +65,11 @@ pub trait Store { ) -> Result, Error>; /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. - fn replace( + fn replace_metadata( &mut self, id: CorpusId, - input: Rc, metadata: TestcaseMetadata, - ) -> Result, Error>; + ) -> Result; /// Get the prev corpus id in chronological order fn prev(&self, id: CorpusId) -> Option; diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index 52d163d4216..9a0638f1ba8 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -4,7 +4,8 @@ use alloc::rc::Rc; use core::marker::PhantomData; use std::{ cell::{Ref, RefCell, RefMut}, - fs::{self, File}, + fs, + fs::OpenOptions, io::Write, path::{Path, PathBuf}, string::String, @@ -18,12 +19,14 @@ use super::{InMemoryCorpusMap, Store}; use crate::{ corpus::{ CorpusId, Testcase, - testcase::{HasTestcaseMetadata, TestcaseMetadata}, + testcase::{IsTestcaseMetadataCell, TestcaseMetadata}, }, inputs::Input, }; /// An on-disk store +/// +/// The maps only store the unique ID associated to the added [`Testcase`]s. #[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct OnDiskStore { disk_mgr: Rc>, @@ -99,22 +102,24 @@ impl OnDiskMetadataFormat { #[derive(Debug)] pub struct OnDiskTestcaseCell { mgr: Rc>, + id: String, testcase_md: RefCell, - modified: bool, + modified: RefCell, } impl OnDiskTestcaseCell { /// Get a new [`OnDiskTestcaseCell`]. - pub fn new(mgr: Rc>, testcase_md: TestcaseMetadata) -> Self { + pub fn new(mgr: Rc>, id: String, testcase_md: TestcaseMetadata) -> Self { Self { mgr, + id, testcase_md: RefCell::new(testcase_md), - modified: false, + modified: RefCell::new(false), } } } -impl HasTestcaseMetadata for OnDiskTestcaseCell { +impl IsTestcaseMetadataCell for OnDiskTestcaseCell { type TestcaseMetadataRef<'a> = Ref<'a, TestcaseMetadata> where @@ -129,24 +134,31 @@ impl HasTestcaseMetadata for OnDiskTestcaseCell { } fn testcase_metadata_mut<'a>(&'a self) -> RefMut<'a, TestcaseMetadata> { + *self.modified.borrow_mut() = true; self.testcase_md.borrow_mut() } fn into_testcase_metadata(self) -> TestcaseMetadata { self.testcase_md.clone().into_inner() } + + fn replace_testcase_metadata(&self, _testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + todo!() + } + + fn flush(&self) -> Result<(), Error> { + self.mgr + .save_metadata(&self.id, &*self.testcase_md.borrow()) + } } impl Drop for OnDiskTestcaseCell { fn drop(&mut self) { - todo!() + self.flush().unwrap(); } } -impl DiskMgr -where - I: Input, -{ +impl DiskMgr { fn testcase_path(&self, testcase_id: &String) -> PathBuf { self.root_dir.join(&testcase_id) } @@ -155,28 +167,40 @@ where self.root_dir.join(format!(".{}.metadata", testcase_id)) } - fn save_testcase(&self, input: Rc, md: TestcaseMetadata) -> Result { - let testcase_id = Testcase::>::compute_id(input.as_ref()); - let testcase_path = self.testcase_path(&testcase_id); - // let mut lockfile = TestcaseLockfile::new(self, testcase_id)?; - - // if lockfile.inc_used() { - // save md to file - let ser_fmt = self.md_format.clone(); - let testcase_md_path = self.testcase_md_path(&testcase_id); + /// The file is created if it does not exist, or reused if it's already there + pub fn save_metadata(&self, id: &String, md: &TestcaseMetadata) -> Result<(), Error> { + let testcase_md_path = self.testcase_md_path(id); - let mut testcase_md_f = File::create_new(testcase_md_path.as_path()).unwrap(); - let testcase_md_ser = ser_fmt.to_vec(&md)?; + let mut testcase_md_f = OpenOptions::new() + .write(true) + .create(true) + .open(&testcase_md_path)?; + let testcase_md_ser = self.md_format.to_vec(&md)?; testcase_md_f.write_all(&testcase_md_ser)?; - // testcase_f.write_all(testcase.input().target_bytes().as_ref())?; + Ok(()) + } +} + +impl DiskMgr +where + I: Input, +{ + fn save_input(&self, input: Rc) -> Result { + let testcase_id = Testcase::>::compute_id(input.as_ref()); + let testcase_path = self.testcase_path(&testcase_id); input.as_ref().to_file(testcase_path.as_path())?; - // } Ok(testcase_id) } + fn save_testcase(&self, input: Rc, md: &TestcaseMetadata) -> Result { + let id = self.save_input(input)?; + self.save_metadata(&id, md)?; + Ok(id) + } + /// prerequisite: the testcase should not have been "removed" before. /// also, it should only happen if it has been saved before. fn load_testcase( @@ -194,7 +218,7 @@ where Ok(Testcase::new( Rc::new(input), - OnDiskTestcaseCell::new(self.clone(), md), + OnDiskTestcaseCell::new(self.clone(), testcase_id.clone(), md), )) } } @@ -223,7 +247,7 @@ where } fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(input, md)?; + let testcase_id = self.disk_mgr.save_testcase(input, &md)?; self.enabled_map.add(id, testcase_id); Ok(()) } @@ -234,7 +258,7 @@ where input: Rc, md: TestcaseMetadata, ) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(input, md)?; + let testcase_id = self.disk_mgr.save_testcase(input, &md)?; self.disabled_map.add(id, testcase_id); Ok(()) } @@ -257,18 +281,12 @@ where self.disk_mgr.load_testcase(&tc_id) } - fn replace( + fn replace_metadata( &mut self, - id: CorpusId, - input: Rc, - md: TestcaseMetadata, - ) -> Result, Error> { - let new_testcase_id = self.disk_mgr.save_testcase(input, md)?; - let old_testcase_id = self - .enabled_map - .replace(id, new_testcase_id) - .ok_or_else(|| Error::key_not_found(format!("Index not found: {id}")))?; - self.disk_mgr.load_testcase(&old_testcase_id) + _id: CorpusId, + _metadata: TestcaseMetadata, + ) -> Result { + todo!() } fn prev(&self, id: CorpusId) -> Option { diff --git a/crates/libafl/src/corpus/testcase.rs b/crates/libafl/src/corpus/testcase.rs index c9f9d5f234e..265ac63648b 100644 --- a/crates/libafl/src/corpus/testcase.rs +++ b/crates/libafl/src/corpus/testcase.rs @@ -30,13 +30,13 @@ use crate::{ }; /// A testcase metadata cell that can be instantiated only from a [`TestcaseMetadata`]. -pub trait HasInstantiableTestcaseMetadata: HasTestcaseMetadata { +pub trait HasInstantiableTestcaseMetadata: IsTestcaseMetadataCell { /// Instantiate a testcase metadata cell from a [`TestcaseMetadata`]. fn instantiate(metadata: TestcaseMetadata) -> Self; } /// Trait implemented by possible [`TestcaseMetadata`] cells. -pub trait HasTestcaseMetadata { +pub trait IsTestcaseMetadataCell { /// A reference to a testcase metadata. type TestcaseMetadataRef<'a>: Deref where @@ -49,10 +49,25 @@ pub trait HasTestcaseMetadata { /// Get a reference to the testcase metadata. fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a>; + /// Get a mutable reference to the testcase metadata. fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a>; + /// Consume the cell, and get the inner testcase metadata. fn into_testcase_metadata(self) -> TestcaseMetadata; + + /// Replace the inner testcase metadata with new metadata, returning the old metadata + fn replace_testcase_metadata(&self, testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + let mut tc_ref = self.testcase_metadata_mut(); + let old_tc = tc_ref.clone(); + *tc_ref = testcase_metadata; + old_tc + } + + /// Propagate metadata cell changes + fn flush(&self) -> Result<(), Error> { + Ok(()) + } } /// A dummy (empty) [`TestcaseMetadata`] reference. @@ -77,7 +92,7 @@ impl<'a> DerefMut for NopTestcaseMetadataRef<'a> { } } -impl HasTestcaseMetadata for NopTestcaseMetadataCell { +impl IsTestcaseMetadataCell for NopTestcaseMetadataCell { type TestcaseMetadataRef<'a> = NopTestcaseMetadataRef<'a>; type TestcaseMetadataRefMut<'a> = NopTestcaseMetadataRef<'a>; @@ -92,9 +107,13 @@ impl HasTestcaseMetadata for NopTestcaseMetadataCell { fn into_testcase_metadata(self) -> TestcaseMetadata { panic!("Invalid testcase metadata") } + + fn replace_testcase_metadata(&self, _testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + panic!("Invalid testcase metadata") + } } -impl HasTestcaseMetadata for RefCell { +impl IsTestcaseMetadataCell for RefCell { type TestcaseMetadataRef<'a> = Ref<'a, TestcaseMetadata>; type TestcaseMetadataRefMut<'a> = RefMut<'a, TestcaseMetadata>; @@ -113,6 +132,10 @@ impl HasTestcaseMetadata for RefCell { fn into_testcase_metadata(self) -> TestcaseMetadata { self.into_inner() } + + fn replace_testcase_metadata(&self, testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + RefCell::replace(self, testcase_metadata) + } } impl HasInstantiableTestcaseMetadata for RefCell { @@ -121,9 +144,9 @@ impl HasInstantiableTestcaseMetadata for RefCell { } } -impl HasTestcaseMetadata for Rc +impl IsTestcaseMetadataCell for Rc where - T: HasTestcaseMetadata + Clone, + T: IsTestcaseMetadataCell + Clone, { type TestcaseMetadataRef<'a> = T::TestcaseMetadataRef<'a> @@ -149,6 +172,10 @@ where fn into_testcase_metadata(self) -> TestcaseMetadata { self.deref().clone().into_testcase_metadata() } + + fn replace_testcase_metadata(&self, testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + T::replace_testcase_metadata(self, testcase_metadata) + } } impl HasInstantiableTestcaseMetadata for Rc @@ -160,9 +187,9 @@ where } } -impl HasTestcaseMetadata for Testcase +impl IsTestcaseMetadataCell for Testcase where - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { type TestcaseMetadataRef<'a> = M::TestcaseMetadataRef<'a> @@ -184,6 +211,10 @@ where fn into_testcase_metadata(self) -> TestcaseMetadata { self.metadata.into_testcase_metadata() } + + fn replace_testcase_metadata(&self, testcase_metadata: TestcaseMetadata) -> TestcaseMetadata { + self.metadata.replace_testcase_metadata(testcase_metadata) + } } /// Shorthand to receive a [`Ref`] or [`RefMut`] to a stored [`Testcase`], by [`CorpusId`]. @@ -257,7 +288,7 @@ where impl Debug for Testcase where I: Debug, - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { f.debug_struct("Testcase") @@ -270,7 +301,7 @@ where impl Serialize for Testcase where - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { fn serialize(&self, _serializer: S) -> Result where @@ -282,7 +313,7 @@ where impl<'de, I, M> Deserialize<'de> for Testcase where - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { fn deserialize(_deserializer: D) -> Result where @@ -315,6 +346,11 @@ impl Testcase { pub fn id(&self) -> &String { &self.id } + + /// Decompose a [`Testcase`] into its inner input and metadata. + pub fn into_inner(self) -> (Rc, M) { + (self.input, self.metadata) + } } impl Testcase @@ -330,7 +366,7 @@ where impl Testcase where I: Input, - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { /// Create a new Testcase instance given an input pub fn new(input: Rc, metadata: M) -> Self { @@ -354,7 +390,7 @@ where impl Testcase where - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, I: Clone, { /// Clone the input embedded in the [`Testcase`]. @@ -366,7 +402,7 @@ where /// Impl of a testcase impl Testcase where - M: HasTestcaseMetadata, + M: IsTestcaseMetadataCell, { /// Get the same testcase, with an owned [`TestcaseMetadata`]. pub fn cloned(self) -> Testcase> { @@ -377,11 +413,6 @@ where } } - /// Decompose a [`Testcase`] into its inner input and metadata. - pub fn into_inner(self) -> (Rc, TestcaseMetadata) { - (self.input, self.metadata.into_testcase_metadata()) - } - /// Test whether the metadata map contains a metadata #[inline] pub fn has_metadata(&self) -> bool diff --git a/crates/libafl/src/executors/inprocess/mod.rs b/crates/libafl/src/executors/inprocess/mod.rs index 89ad794470f..450df770443 100644 --- a/crates/libafl/src/executors/inprocess/mod.rs +++ b/crates/libafl/src/executors/inprocess/mod.rs @@ -17,7 +17,7 @@ use libafl_bolts::tuples::{RefIndexable, tuple_list}; use crate::{ Error, common::HasMetadataMut, - corpus::{Corpus, HasTestcaseMetadata, testcase::TestcaseMetadata}, + corpus::{Corpus, IsTestcaseMetadataCell, testcase::TestcaseMetadata}, events::{Event, EventFirer, EventRestarter, EventWithStats}, executors::{ Executor, ExitKind, HasObservers, diff --git a/crates/libafl/src/fuzzer/mod.rs b/crates/libafl/src/fuzzer/mod.rs index e4c814059e9..b11208d9db9 100644 --- a/crates/libafl/src/fuzzer/mod.rs +++ b/crates/libafl/src/fuzzer/mod.rs @@ -15,7 +15,7 @@ use crate::monitors::stats::PerfFeature; use crate::{ Error, HasMetadata, corpus::{ - Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, HasTestcaseMetadata, TestcaseMetadata, + Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, IsTestcaseMetadataCell, TestcaseMetadata, }, events::{ Event, EventConfig, EventFirer, EventReceiver, EventWithStats, ProgressReporter, diff --git a/crates/libafl/src/inputs/generalized.rs b/crates/libafl/src/inputs/generalized.rs index 57f5f1f7f40..d578740e383 100644 --- a/crates/libafl/src/inputs/generalized.rs +++ b/crates/libafl/src/inputs/generalized.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadata, - corpus::{Testcase, testcase::HasTestcaseMetadata}, + corpus::{Testcase, testcase::IsTestcaseMetadataCell}, inputs::BytesInput, stages::mutational::{MutatedTransform, MutatedTransformPost}, }; @@ -107,7 +107,7 @@ impl GeneralizedInputMetadata { impl MutatedTransform for GeneralizedInputMetadata { type Post = Self; - fn try_transform_from( + fn try_transform_from( base: &Testcase, _state: &S, ) -> Result { diff --git a/crates/libafl/src/mutators/gramatron.rs b/crates/libafl/src/mutators/gramatron.rs index 97943b74fc7..ce28c544aaa 100644 --- a/crates/libafl/src/mutators/gramatron.rs +++ b/crates/libafl/src/mutators/gramatron.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadata, HasMetadataMut, - corpus::{Corpus, HasTestcaseMetadata}, + corpus::{Corpus, IsTestcaseMetadataCell}, generators::GramatronGenerator, inputs::{GramatronInput, Terminal}, mutators::{MutationResult, Mutator}, diff --git a/crates/libafl/src/mutators/grimoire.rs b/crates/libafl/src/mutators/grimoire.rs index 455794d9021..5de61e0131c 100644 --- a/crates/libafl/src/mutators/grimoire.rs +++ b/crates/libafl/src/mutators/grimoire.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use crate::{ Error, HasMetadata, - corpus::{Corpus, HasTestcaseMetadata}, + corpus::{Corpus, IsTestcaseMetadataCell}, inputs::{GeneralizedInputMetadata, GeneralizedItem}, mutators::{MutationResult, Mutator, token_mutations::Tokens}, random_corpus_id, diff --git a/crates/libafl/src/mutators/scheduled.rs b/crates/libafl/src/mutators/scheduled.rs index b16f47d0ef7..e33fc9ca79d 100644 --- a/crates/libafl/src/mutators/scheduled.rs +++ b/crates/libafl/src/mutators/scheduled.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; use super::MutationId; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcaseMetadata}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, mutators::{ MutationResult, Mutator, MutatorsTuple, token_mutations::{TokenInsert, TokenReplace}, diff --git a/crates/libafl/src/schedulers/accounting.rs b/crates/libafl/src/schedulers/accounting.rs index bfbd705b742..e1cda6c7091 100644 --- a/crates/libafl/src/schedulers/accounting.rs +++ b/crates/libafl/src/schedulers/accounting.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use super::IndexesLenTimeMinimizerScheduler; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcaseMetadata}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, observers::CanTrack, schedulers::{ Scheduler, diff --git a/crates/libafl/src/schedulers/minimizer.rs b/crates/libafl/src/schedulers/minimizer.rs index 5fc32a7651d..ba8ffc36097 100644 --- a/crates/libafl/src/schedulers/minimizer.rs +++ b/crates/libafl/src/schedulers/minimizer.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use super::HasQueueCycles; use crate::{ Error, HasMetadata, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcaseMetadata, Testcase}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase}, feedbacks::MapIndexesMetadata, observers::CanTrack, require_index_tracking, @@ -183,7 +183,7 @@ where &mut self, state: &mut S, id: CorpusId, - prev: &Testcase>::TestcaseMetadataCell>, + prev: &>::TestcaseMetadataCell, ) -> Result<(), Error> { self.base.on_replace(state, id, prev)?; self.update_score(state, id) diff --git a/crates/libafl/src/schedulers/mod.rs b/crates/libafl/src/schedulers/mod.rs index bb5a90384f2..1409654d472 100644 --- a/crates/libafl/src/schedulers/mod.rs +++ b/crates/libafl/src/schedulers/mod.rs @@ -12,7 +12,7 @@ use libafl_bolts::{ use crate::{ Error, HasMetadata, HasMetadataMut, corpus::{ - Corpus, CorpusId, HasTestcase, HasTestcaseMetadata, SchedulerTestcaseMetadata, Testcase, + Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, SchedulerTestcaseMetadata, Testcase, }, random_corpus_id, state::{HasCorpus, HasRand}, @@ -65,7 +65,7 @@ where &mut self, _state: &mut S, _id: CorpusId, - _prev: &Testcase>::TestcaseMetadataCell>, + _prev: &>::TestcaseMetadataCell, ) -> Result<(), Error> { Ok(()) } diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index bffee26a0aa..9b725a98e70 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcaseMetadata, Testcase}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase}, schedulers::{RemovableScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, HasRand}, }; @@ -108,7 +108,7 @@ where &mut self, state: &mut S, id: CorpusId, - _prev: &Testcase>::TestcaseMetadataCell>, + _prev: &>::TestcaseMetadataCell, ) -> Result<(), Error> { let meta = state .metadata_map_mut() diff --git a/crates/libafl/src/schedulers/queue.rs b/crates/libafl/src/schedulers/queue.rs index ef4b9dc24f4..641555ef0ee 100644 --- a/crates/libafl/src/schedulers/queue.rs +++ b/crates/libafl/src/schedulers/queue.rs @@ -4,7 +4,7 @@ use alloc::borrow::ToOwned; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcaseMetadata}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, schedulers::{HasQueueCycles, RemovableScheduler, Scheduler}, state::HasCorpus, }; diff --git a/crates/libafl/src/schedulers/testcase_score.rs b/crates/libafl/src/schedulers/testcase_score.rs index f52fa649762..87befc29d71 100644 --- a/crates/libafl/src/schedulers/testcase_score.rs +++ b/crates/libafl/src/schedulers/testcase_score.rs @@ -6,7 +6,7 @@ use num_traits::Zero; use crate::{ Error, HasMetadata, - corpus::{Corpus, CorpusId, HasTestcaseMetadata, SchedulerTestcaseMetadata}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, SchedulerTestcaseMetadata}, feedbacks::MapIndexesMetadata, schedulers::{ minimizer::{IsFavoredMetadata, TopRatedsMetadata}, diff --git a/crates/libafl/src/schedulers/weighted.rs b/crates/libafl/src/schedulers/weighted.rs index 6539366f61d..e920acca6f2 100644 --- a/crates/libafl/src/schedulers/weighted.rs +++ b/crates/libafl/src/schedulers/weighted.rs @@ -276,7 +276,7 @@ where &mut self, _state: &mut S, _id: CorpusId, - _prev: &Testcase>::TestcaseMetadataCell>, + _prev: &>::TestcaseMetadataCell, ) -> Result<(), Error> { self.table_invalidated = true; Ok(()) diff --git a/crates/libafl/src/stages/afl_stats.rs b/crates/libafl/src/stages/afl_stats.rs index fe70f1db8ad..b9de076ed6d 100644 --- a/crates/libafl/src/stages/afl_stats.rs +++ b/crates/libafl/src/stages/afl_stats.rs @@ -28,7 +28,7 @@ use crate::{ Error, HasMetadata, HasNamedMetadata, HasScheduler, corpus::{ Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata, Testcase, - testcase::HasTestcaseMetadata, + testcase::IsTestcaseMetadataCell, }, events::{Event, EventFirer, EventWithStats}, executors::HasObservers, @@ -492,13 +492,16 @@ where Ok(()) } - fn maybe_update_is_favored_size(&mut self, testcase: &Testcase) { + fn maybe_update_is_favored_size( + &mut self, + testcase: &Testcase, + ) { if testcase.has_metadata::() { self.is_favored_size += 1; } } - fn maybe_update_slowest_exec(&mut self, testcase: &Testcase) { + fn maybe_update_slowest_exec(&mut self, testcase: &Testcase) { if let Some(exec_time) = testcase.testcase_metadata().exec_time() { if exec_time > &self.slowest_exec { self.slowest_exec = *exec_time; @@ -510,7 +513,7 @@ where self.has_fuzzed_size += 1; } - fn maybe_update_max_depth(&mut self, testcase: &Testcase) { + fn maybe_update_max_depth(&mut self, testcase: &Testcase) { if let Ok(metadata) = testcase .testcase_metadata() .metadata::() diff --git a/crates/libafl/src/stages/calibrate.rs b/crates/libafl/src/stages/calibrate.rs index ff0d3fd4caf..fb32f763d61 100644 --- a/crates/libafl/src/stages/calibrate.rs +++ b/crates/libafl/src/stages/calibrate.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, HasScheduler, - corpus::{Corpus, HasCurrentCorpusId, HasTestcaseMetadata, SchedulerTestcaseMetadata}, + corpus::{Corpus, HasCurrentCorpusId, IsTestcaseMetadataCell, SchedulerTestcaseMetadata}, events::{Event, EventFirer, EventWithStats, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, feedbacks::{HasObserverHandle, map::MapFeedbackMetadata}, diff --git a/crates/libafl/src/stages/concolic.rs b/crates/libafl/src/stages/concolic.rs index 15e70626f28..776c8d0d0ab 100644 --- a/crates/libafl/src/stages/concolic.rs +++ b/crates/libafl/src/stages/concolic.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use crate::monitors::stats::PerfFeature; use crate::{ Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, - corpus::{HasCurrentCorpusId, HasTestcaseMetadata}, + corpus::{HasCurrentCorpusId, IsTestcaseMetadataCell}, executors::{Executor, HasObservers}, observers::{ObserversTuple, concolic::ConcolicObserver}, stages::{Restartable, RetryCountRestartHelper, Stage, TracingStage}, diff --git a/crates/libafl/src/stages/generalization.rs b/crates/libafl/src/stages/generalization.rs index 106e08a9914..6a00036604c 100644 --- a/crates/libafl/src/stages/generalization.rs +++ b/crates/libafl/src/stages/generalization.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use crate::monitors::stats::PerfFeature; use crate::{ Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, - corpus::{Corpus, HasCurrentCorpusId, HasTestcaseMetadata}, + corpus::{Corpus, HasCurrentCorpusId, IsTestcaseMetadataCell}, executors::{Executor, HasObservers}, feedbacks::map::MapNoveltiesMetadata, inputs::{BytesInput, GeneralizedInputMetadata, GeneralizedItem, ResizableMutator}, diff --git a/crates/libafl/src/stages/mutational.rs b/crates/libafl/src/stages/mutational.rs index 7c591a63cdc..d6d62a58291 100644 --- a/crates/libafl/src/stages/mutational.rs +++ b/crates/libafl/src/stages/mutational.rs @@ -13,7 +13,7 @@ use libafl_bolts::{Named, rands::Rand}; use crate::monitors::stats::PerfFeature; use crate::{ Error, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, - corpus::{CorpusId, HasCurrentCorpusId, Testcase, testcase::HasTestcaseMetadata}, + corpus::{CorpusId, HasCurrentCorpusId, Testcase, testcase::IsTestcaseMetadataCell}, fuzzer::Evaluator, inputs::Input, mark_feature_time, @@ -46,7 +46,7 @@ pub trait MutatedTransform: Sized { type Post: MutatedTransformPost; /// Transform the provided testcase into this type - fn try_transform_from( + fn try_transform_from( base: &Testcase, state: &S, ) -> Result; @@ -64,7 +64,7 @@ where type Post = (); #[inline] - fn try_transform_from( + fn try_transform_from( base: &Testcase, _state: &S, ) -> Result { diff --git a/crates/libafl/src/stages/tmin.rs b/crates/libafl/src/stages/tmin.rs index 897cdfbd493..f961fded791 100644 --- a/crates/libafl/src/stages/tmin.rs +++ b/crates/libafl/src/stages/tmin.rs @@ -2,7 +2,6 @@ use alloc::{ borrow::{Cow, ToOwned}, - rc::Rc, string::ToString, }; use core::{fmt::Debug, hash::Hash, marker::PhantomData}; @@ -294,9 +293,7 @@ where .feedback_mut() .append_metadata(state, manager, &*observers, &mut tc_md)?; - let prev = state - .corpus_mut() - .replace(base_corpus_id, Rc::new(base), tc_md)?; + let prev = state.corpus_mut().replace_metadata(base_corpus_id, tc_md)?; fuzzer .scheduler_mut() From 64deb685a459170338bb7796e7628b6cbe02e1e2 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 13:50:39 +0200 Subject: [PATCH 05/12] better cache policy --- crates/libafl/src/corpus/cache.rs | 123 ++++++++++++++++++--------- crates/libafl/src/corpus/combined.rs | 12 +-- crates/libafl/src/corpus/mod.rs | 4 - 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index d3db46db754..d74bd064d20 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -5,7 +5,7 @@ //! - a **cache store** holding on the testcases with quick access. //! - a **backing store** with more expensive access, used when the testcase cannot be found in the cache store. -use alloc::rc::Rc; +use alloc::{rc::Rc, vec::Vec}; use std::{cell::RefCell, collections::VecDeque, marker::PhantomData}; use libafl_bolts::Error; @@ -21,39 +21,40 @@ use crate::{ }; /// Describes how a change to metadata should be propagated to the fallback store -#[derive(Debug)] -pub enum CachePolicy { - /// Propagate the changes when the cell gets dropped. - /// Expect more writes to the fallback store, with - WritebackOnDrop, - /// Propagate the changes when the cache is flushed explicitly. - /// Less writes to the fallback stores will be issued, but the used is responsible for - /// flushing the cache regularly, to avoid data loss. - WritebackOnFlush, -} - -/// Describe a cache policy pub trait HasCachePolicy { - /// The cache policy - const CACHE_POLICY: CachePolicy; + /// Mark a corpus id as dirty + fn dirty(&self, corpus_id: CorpusId); } -/// An implementor for [`CachePolicy::WritebackOnDrop`] +/// Propagate the changes when the cell gets dropped. +/// Expect more writes to the fallback store. #[derive(Debug)] pub struct WritebackOnDropPolicy; impl HasCachePolicy for WritebackOnDropPolicy { - const CACHE_POLICY: CachePolicy = CachePolicy::WritebackOnDrop; + fn dirty(&self, _corpus_id: CorpusId) { + // do nothing + } } -/// An implementor for [`CachePolicy::WritebackOnFlush`] +/// Propagate the changes when the cache is flushed explicitly. +/// +/// Less writes to the fallback stores will be issued, but the used is responsible for +/// flushing the cache regularly. +/// If the cache is not flushed, no data will be written to the fallback store, resulting in +/// data loss. #[derive(Debug)] -pub struct WritebackOnFlushPolicy; +pub struct WritebackOnFlushPolicy { + dirty_entries: RefCell>, +} + impl HasCachePolicy for WritebackOnFlushPolicy { - const CACHE_POLICY: CachePolicy = CachePolicy::WritebackOnFlush; + fn dirty(&self, corpus_id: CorpusId) { + self.dirty_entries.borrow_mut().push(corpus_id); + } } /// A cache, managing a cache store and a fallback store. -pub trait Cache { +pub trait Cache { /// A [`TestcaseMetadata`] cell. type TestcaseMetadataCell: IsTestcaseMetadataCell; @@ -105,25 +106,47 @@ pub trait Cache { /// Mark a corpus entry as written explicitly, for subsequent flushes. /// /// Thus, a cache [`Self::flush`] should propagate to entries marked as [`Self::written`]. - fn written(&mut self, id: CorpusId); + fn written(&self, id: CorpusId); } /// A composed testcase metadata cell, linking the cached cell with the fallback cell. -#[derive(Debug, Clone)] -pub struct CacheTestcaseMetadataCell +#[derive(Debug)] +pub struct CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell, + CP: HasCachePolicy, FC: IsTestcaseMetadataCell, { write_access: RefCell, + cache_policy: Rc, cache_cell: CC, fallback_cell: FC, } +impl Clone for CacheTestcaseMetadataCell +where + CC: IsTestcaseMetadataCell + Clone, + CP: HasCachePolicy, + FC: IsTestcaseMetadataCell + Clone, +{ + fn clone(&self) -> Self { + Self { + write_access: self.write_access.clone(), + cache_policy: self.cache_policy.clone(), + cache_cell: self.cache_cell.clone(), + fallback_cell: self.fallback_cell.clone(), + } + } +} + /// An identity cache, storing everything both in the cache and the backing store. +/// +/// It only supports [`WritebackOnFlushPolicy`] since all the testcases are stored in memory on load +/// forever. #[derive(Debug)] pub struct IdentityCache { - cell_map: M, + cell_map: RefCell, + cache_policy: Rc, } /// A `First In / First Out` cache policy. @@ -134,14 +157,16 @@ pub struct FifoCache { phantom: PhantomData<(I, CS, FS)>, } -impl CacheTestcaseMetadataCell +impl CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell, + CP: HasCachePolicy, FC: IsTestcaseMetadataCell, { /// Create a new [`CacheTestcaseMetadataCell`] - pub fn new(cache_cell: CC, fallback_cell: FC) -> Self { + pub fn new(cache_policy: Rc, cache_cell: CC, fallback_cell: FC) -> Self { Self { + cache_policy, write_access: RefCell::new(false), cache_cell, fallback_cell, @@ -149,9 +174,10 @@ where } } -impl IsTestcaseMetadataCell for CacheTestcaseMetadataCell +impl IsTestcaseMetadataCell for CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell, + CP: HasCachePolicy, FC: IsTestcaseMetadataCell, { type TestcaseMetadataRef<'a> @@ -196,9 +222,10 @@ where } } -impl Drop for CacheTestcaseMetadataCell +impl Drop for CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell, + CP: HasCachePolicy, FC: IsTestcaseMetadataCell, { fn drop(&mut self) { @@ -206,7 +233,7 @@ where } } -impl Cache for IdentityCache +impl Cache for IdentityCache where CS: RemovableStore, FS: Store, @@ -214,15 +241,25 @@ where M: InMemoryCorpusMap< Testcase< I, - Rc>, + Rc< + CacheTestcaseMetadataCell< + CS::TestcaseMetadataCell, + WritebackOnFlushPolicy, + FS::TestcaseMetadataCell, + >, + >, >, >, - P: HasCachePolicy, >::TestcaseMetadataCell: Clone, >::TestcaseMetadataCell: Clone, { - type TestcaseMetadataCell = - Rc>; + type TestcaseMetadataCell = Rc< + CacheTestcaseMetadataCell< + CS::TestcaseMetadataCell, + WritebackOnFlushPolicy, + FS::TestcaseMetadataCell, + >, + >; fn add( &mut self, @@ -254,16 +291,20 @@ where cache_store: &mut CS, fallback_store: &FS, ) -> Result, Error> { - if let Some(tc) = self.cell_map.get(id) { + if let Some(tc) = self.cell_map.borrow().get(id) { Ok(tc.clone()) } else { let (input, cc) = cache_store.get_from::(id)?.into_inner(); let (_, fc) = fallback_store.get_from::(id)?.into_inner(); - let cache_cell = Rc::new(CacheTestcaseMetadataCell::new(cc, fc)); + let cache_cell = Rc::new(CacheTestcaseMetadataCell::new( + self.cache_policy.clone(), + cc, + fc, + )); let testcase = Testcase::new(input, cache_cell.clone()); - self.cell_map.add(id, testcase.clone()); + self.cell_map.borrow_mut().add(id, testcase.clone()); Ok(testcase) } @@ -302,8 +343,8 @@ where todo!() } - fn written(&mut self, _id: CorpusId) { - todo!() + fn written(&self, id: CorpusId) { + self.cache_policy.dirty(id) } } @@ -350,7 +391,7 @@ where } } -impl Cache for FifoCache +impl Cache for FifoCache where CS: RemovableStore, FS: Store, @@ -419,7 +460,7 @@ where todo!() } - fn written(&mut self, _id: CorpusId) { + fn written(&self, _id: CorpusId) { todo!() } } diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index 852c7c02e99..b76545e42ac 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -13,25 +13,25 @@ use crate::corpus::testcase::TestcaseMetadata; /// A [`CombinedCorpus`] tries first to use the main store according to some policy. /// If it fails, it falls back to the secondary store. #[derive(Default, Serialize, Deserialize, Clone, Debug)] -pub struct CombinedCorpus { +pub struct CombinedCorpus { /// The cache store cache_store: RefCell, /// The fallback store fallback_store: FS, - /// The policty taking decisions - cache: RefCell, + /// The policy taking decisions + cache: Rc>, /// The corpus ID counter counter: CorpusCounter, /// The keys in order (use `Vec::binary_search`) keys: Vec, /// The current ID current: Option, - phantom: PhantomData<(I, P)>, + phantom: PhantomData, } -impl Corpus for CombinedCorpus +impl Corpus for CombinedCorpus where - C: Cache, + C: Cache, CS: Store, FS: Store, I: Clone, diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index c1e67a2782e..a57bf1cfd7d 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -36,8 +36,6 @@ pub use nop::NopCorpus; pub mod store; pub use store::{InMemoryStore, OnDiskStore, maps}; -use crate::corpus::cache::WritebackOnFlushPolicy; - /// The standard fully in-memory corpus map. #[cfg(not(feature = "corpus_btreemap"))] pub type StdInMemoryCorpusMap = maps::HashCorpusMap>>>; @@ -66,7 +64,6 @@ pub type InMemoryOnDiskCorpus = CombinedCorpus< StdInMemoryStore, StdOnDiskStore, I, - WritebackOnFlushPolicy, >; /// The standard corpus for storing on disk and in-memory with a cache. @@ -76,7 +73,6 @@ pub type CachedOnDiskCorpus = CombinedCorpus< StdInMemoryStore, StdOnDiskStore, I, - WritebackOnFlushPolicy, >; /// An abstraction for the index that identify a testcase in the corpus From 2799aaa2255e1ccc4bf94b30d42e42161b4c051e Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 15:55:05 +0200 Subject: [PATCH 06/12] add input to `append_metadata`. continue adapting new corpus stuff for more code. add generic add function to corpus / store. --- crates/libafl/src/corpus/cache.rs | 47 +------ crates/libafl/src/corpus/combined.rs | 22 +-- crates/libafl/src/corpus/mod.rs | 18 ++- crates/libafl/src/corpus/nop.rs | 12 +- crates/libafl/src/corpus/single.rs | 14 +- crates/libafl/src/corpus/store/inmemory.rs | 20 +-- crates/libafl/src/corpus/store/mod.rs | 7 +- crates/libafl/src/corpus/store/ondisk.rs | 16 +-- crates/libafl/src/executors/inprocess/mod.rs | 6 +- crates/libafl/src/feedbacks/bool.rs | 1 + .../libafl/src/feedbacks/capture_feedback.rs | 1 + crates/libafl/src/feedbacks/concolic.rs | 1 + .../libafl/src/feedbacks/custom_filename.rs | 3 +- crates/libafl/src/feedbacks/list.rs | 1 + crates/libafl/src/feedbacks/map.rs | 1 + crates/libafl/src/feedbacks/mod.rs | 13 +- crates/libafl/src/feedbacks/nautilus.rs | 12 +- crates/libafl/src/feedbacks/simd.rs | 4 +- crates/libafl/src/feedbacks/stdio.rs | 2 + crates/libafl/src/fuzzer/mod.rs | 35 ++--- crates/libafl/src/mutators/mutations.rs | 15 +- crates/libafl/src/mutators/unicode/mod.rs | 7 +- .../src/schedulers/probabilistic_sampling.rs | 2 +- crates/libafl/src/stages/afl_stats.rs | 6 +- crates/libafl/src/stages/concolic.rs | 4 +- crates/libafl/src/stages/dump.rs | 133 +++++++++--------- crates/libafl/src/stages/tmin.rs | 10 +- crates/libafl_frida/src/asan/errors.rs | 7 +- .../libafl_libfuzzer/runtime/src/feedbacks.rs | 3 +- fuzzers/baby/tutorial/src/metadata.rs | 7 +- .../libafl-fuzz/src/feedback/filepath.rs | 3 +- .../src/feedback/persistent_record.rs | 1 + .../libafl-fuzz/src/feedback/seed.rs | 1 + 33 files changed, 222 insertions(+), 213 deletions(-) diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index d74bd064d20..bebc4104062 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -59,17 +59,7 @@ pub trait Cache { type TestcaseMetadataCell: IsTestcaseMetadataCell; /// Add a testcase to the cache - fn add( - &mut self, - id: CorpusId, - input: Rc, - md: TestcaseMetadata, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error>; - - /// Add a disabled testcase to the cache - fn add_disabled( + fn add_shared( &mut self, id: CorpusId, input: Rc, @@ -261,19 +251,7 @@ where >, >; - fn add( - &mut self, - id: CorpusId, - input: Rc, - md: TestcaseMetadata, - cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - cache_store.add(id, input.clone(), md.clone())?; - fallback_store.add(id, input, md) - } - - fn add_disabled( + fn add_shared( &mut self, id: CorpusId, input: Rc, @@ -281,8 +259,8 @@ where cache_store: &mut CS, fallback_store: &mut FS, ) -> Result<(), Error> { - cache_store.add_disabled(id, input.clone(), md.clone())?; - fallback_store.add_disabled(id, input, md) + cache_store.add_shared::(id, input.clone(), md.clone())?; + fallback_store.add_shared::(id, input, md) } fn get_from( @@ -399,18 +377,7 @@ where { type TestcaseMetadataCell = CS::TestcaseMetadataCell; - fn add( - &mut self, - id: CorpusId, - input: Rc, - metadata: TestcaseMetadata, - _cache_store: &mut CS, - fallback_store: &mut FS, - ) -> Result<(), Error> { - fallback_store.add(id, input, metadata) - } - - fn add_disabled( + fn add_shared( &mut self, id: CorpusId, input: Rc, @@ -418,7 +385,7 @@ where _cache_store: &mut CS, fallback_store: &mut FS, ) -> Result<(), Error> { - fallback_store.add_disabled(id, input, metadata) + fallback_store.add_shared::(id, input, metadata) } fn get_from( @@ -433,7 +400,7 @@ where fallback_store, |cache_store, corpus_id, testcase| { let (input, md) = testcase.into_inner(); - cache_store.add(corpus_id, input, md.into_testcase_metadata()) + cache_store.add_shared::(corpus_id, input, md.into_testcase_metadata()) }, |cache_store, corpus_id| cache_store.get(corpus_id), |cache_store, corpus_id| cache_store.remove(corpus_id), diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index b76545e42ac..e3d61a129a1 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -50,24 +50,14 @@ where self.fallback_store.count_all() } - fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result { - let new_id = self.counter.new_id(); - - self.cache.borrow_mut().add( - new_id, - input, - md, - &mut *self.cache_store.borrow_mut(), - &mut self.fallback_store, - )?; - - Ok(new_id) - } - - fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result { + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { let new_id = self.counter.new_id(); - self.cache.borrow_mut().add_disabled( + self.cache.borrow_mut().add_shared::( new_id, input, md, diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index a57bf1cfd7d..8a5acbf8ac2 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -155,10 +155,24 @@ pub trait Corpus: Sized { } /// Add an enabled testcase to the corpus and return its index - fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result; + fn add(&mut self, input: I, md: TestcaseMetadata) -> Result { + self.add_shared::(Rc::new(input), md) + } /// Add a disabled testcase to the corpus and return its index - fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result; + fn add_disabled(&mut self, input: I, md: TestcaseMetadata) -> Result { + self.add_shared::(Rc::new(input), md) + } + + /// Add a testcase to the corpus, and returns its index. + /// The associated type tells whether the input should be added to the enabled or the disabled corpus. + /// + /// The input can be shared through [`Rc`]. + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result; /// Get testcase by id; considers only enabled testcases fn get(&self, id: CorpusId) -> Result, Error> { diff --git a/crates/libafl/src/corpus/nop.rs b/crates/libafl/src/corpus/nop.rs index cb227f466e7..cbb08f07b62 100644 --- a/crates/libafl/src/corpus/nop.rs +++ b/crates/libafl/src/corpus/nop.rs @@ -42,13 +42,11 @@ impl Corpus for NopCorpus { /// Add an enabled testcase to the corpus and return its index #[inline] - fn add(&mut self, _input: Rc, _md: TestcaseMetadata) -> Result { - Err(Error::unsupported("Unsupported by NopCorpus")) - } - - /// Add a disabled testcase to the corpus and return its index - #[inline] - fn add_disabled(&mut self, _input: Rc, _md: TestcaseMetadata) -> Result { + fn add_shared( + &mut self, + _input: Rc, + _md: TestcaseMetadata, + ) -> Result { Err(Error::unsupported("Unsupported by NopCorpus")) } diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 7fa4db9cc5f..584eee55e10 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -70,15 +70,13 @@ where self.store.count_all() } - fn add(&mut self, input: Rc, md: TestcaseMetadata) -> Result { - let new_id = self.counter.new_id(); - self.store.add(new_id, input, md)?; - Ok(new_id) - } - - fn add_disabled(&mut self, input: Rc, md: TestcaseMetadata) -> Result { + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { let new_id = self.counter.new_id(); - self.store.add_disabled(new_id, input, md)?; + self.store.add_shared::(new_id, input, md)?; Ok(new_id) } diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index ddef0fbad78..6c05e4b0d25 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -56,21 +56,21 @@ where self.enabled_map.is_empty() } - fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error> { - Ok(self - .enabled_map - .add(id, Testcase::new(input, TMC::instantiate(md)))) - } - - fn add_disabled( + fn add_shared( &mut self, id: CorpusId, input: Rc, md: TestcaseMetadata, ) -> Result<(), Error> { - Ok(self - .disabled_map - .add(id, Testcase::new(input, TMC::instantiate(md)))) + if ENABLED { + Ok(self + .enabled_map + .add(id, Testcase::new(input, TMC::instantiate(md)))) + } else { + Ok(self + .disabled_map + .add(id, Testcase::new(input, TMC::instantiate(md)))) + } } fn get_from( diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index 81b4ed85c92..1f2e163fcb5 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -37,11 +37,8 @@ pub trait Store { self.count() == 0 } - /// Store the testcase associated to corpus_id to the enabled set. - fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error>; - - /// Store the testcase associated to corpus_id to the disabled set. - fn add_disabled( + /// Store the testcase associated to corpus_id to the set. + fn add_shared( &mut self, id: CorpusId, input: Rc, diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index 9a0638f1ba8..a0a5b4af11a 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -246,20 +246,20 @@ where self.disabled_map.count() } - fn add(&mut self, id: CorpusId, input: Rc, md: TestcaseMetadata) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(input, &md)?; - self.enabled_map.add(id, testcase_id); - Ok(()) - } - - fn add_disabled( + fn add_shared( &mut self, id: CorpusId, input: Rc, md: TestcaseMetadata, ) -> Result<(), Error> { let testcase_id = self.disk_mgr.save_testcase(input, &md)?; - self.disabled_map.add(id, testcase_id); + + if ENABLED { + self.enabled_map.add(id, testcase_id); + } else { + self.disabled_map.add(id, testcase_id); + } + Ok(()) } diff --git a/crates/libafl/src/executors/inprocess/mod.rs b/crates/libafl/src/executors/inprocess/mod.rs index 450df770443..572850b3ace 100644 --- a/crates/libafl/src/executors/inprocess/mod.rs +++ b/crates/libafl/src/executors/inprocess/mod.rs @@ -2,7 +2,7 @@ //! It should usually be paired with extra error-handling, such as a restarting event manager, to be effective. //! //! Needs the `fork` feature flag. -use alloc::{boxed::Box, rc::Rc}; +use alloc::boxed::Box; use core::{ borrow::BorrowMut, ffi::c_void, @@ -358,12 +358,12 @@ pub fn run_observers_and_save_state( fuzzer .objective_mut() - .append_metadata(state, event_mgr, &*observers, &mut testcase_md) + .append_metadata(state, event_mgr, &*observers, input, &mut testcase_md) .expect("Failed adding metadata"); state .solutions_mut() - .add(Rc::new(input.clone()), testcase_md) + .add(input.clone(), testcase_md) .expect("In run_observers_and_save_state solutions failure."); let event = Event::Objective { diff --git a/crates/libafl/src/feedbacks/bool.rs b/crates/libafl/src/feedbacks/bool.rs index e91d8966022..df5935929a3 100644 --- a/crates/libafl/src/feedbacks/bool.rs +++ b/crates/libafl/src/feedbacks/bool.rs @@ -87,6 +87,7 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, + _input: &I, _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) diff --git a/crates/libafl/src/feedbacks/capture_feedback.rs b/crates/libafl/src/feedbacks/capture_feedback.rs index 133d1389253..c706cfe7391 100644 --- a/crates/libafl/src/feedbacks/capture_feedback.rs +++ b/crates/libafl/src/feedbacks/capture_feedback.rs @@ -64,6 +64,7 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, + _input: &I, _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) diff --git a/crates/libafl/src/feedbacks/concolic.rs b/crates/libafl/src/feedbacks/concolic.rs index 29be79df890..ce3ad31e427 100644 --- a/crates/libafl/src/feedbacks/concolic.rs +++ b/crates/libafl/src/feedbacks/concolic.rs @@ -73,6 +73,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.add_concolic_feedback_to_metadata::(observers, md); diff --git a/crates/libafl/src/feedbacks/custom_filename.rs b/crates/libafl/src/feedbacks/custom_filename.rs index 96cd72e4e04..eda78485980 100644 --- a/crates/libafl/src/feedbacks/custom_filename.rs +++ b/crates/libafl/src/feedbacks/custom_filename.rs @@ -89,7 +89,8 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _input: &I, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { *testcase.filename_mut() = Some(self.generator.set_name(state, testcase)?); Ok(()) diff --git a/crates/libafl/src/feedbacks/list.rs b/crates/libafl/src/feedbacks/list.rs index 32d58c79ca5..4f710dafea8 100644 --- a/crates/libafl/src/feedbacks/list.rs +++ b/crates/libafl/src/feedbacks/list.rs @@ -177,6 +177,7 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, + _input: &I, _md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.append_list_observer_metadata(state); diff --git a/crates/libafl/src/feedbacks/map.rs b/crates/libafl/src/feedbacks/map.rs index b3c51bd8dad..c073622e126 100644 --- a/crates/libafl/src/feedbacks/map.rs +++ b/crates/libafl/src/feedbacks/map.rs @@ -368,6 +368,7 @@ where state: &mut S, manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { if let Some(novelties) = self.novelties.as_mut().map(core::mem::take) { diff --git a/crates/libafl/src/feedbacks/mod.rs b/crates/libafl/src/feedbacks/mod.rs index dbad27e8b32..574efef1054 100644 --- a/crates/libafl/src/feedbacks/mod.rs +++ b/crates/libafl/src/feedbacks/mod.rs @@ -149,6 +149,7 @@ pub trait Feedback: StateInitializer + Named { _state: &mut S, _manager: &mut EM, _observers: &OT, + _input: &I, _md: &mut TestcaseMetadata, ) -> Result<(), Error> { Ok(()) @@ -298,10 +299,13 @@ where state: &mut S, manager: &mut EM, observers: &OT, + input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.first.append_metadata(state, manager, observers, md)?; - self.second.append_metadata(state, manager, observers, md) + self.first + .append_metadata(state, manager, observers, input, md)?; + self.second + .append_metadata(state, manager, observers, input, md) } } @@ -656,9 +660,11 @@ where state: &mut S, manager: &mut EM, observers: &OT, + input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.inner.append_metadata(state, manager, observers, md) + self.inner + .append_metadata(state, manager, observers, input, md) } } @@ -925,6 +931,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { let Some(observer) = observers.get(&self.observer_handle) else { diff --git a/crates/libafl/src/feedbacks/nautilus.rs b/crates/libafl/src/feedbacks/nautilus.rs index f58c018db03..8a0f36f57be 100644 --- a/crates/libafl/src/feedbacks/nautilus.rs +++ b/crates/libafl/src/feedbacks/nautilus.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadata, common::nautilus::grammartec::{chunkstore::ChunkStore, context::Context}, - corpus::{Corpus, Testcase}, + corpus::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, generators::NautilusContext, @@ -64,13 +64,12 @@ impl<'a> NautilusFeedback<'a> { fn append_nautilus_metadata_to_state( &mut self, state: &mut S, - testcase: &mut Testcase, + input: &NautilusInput, + md: &mut TestcaseMetadata, ) -> Result<(), Error> where S: HasCorpus + HasMetadata, { - state.corpus().load_input_into(testcase)?; - let input = testcase.input().as_ref().unwrap().clone(); let meta = state .metadata_map_mut() .get_mut::() @@ -110,9 +109,10 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + input: &NautilusInput, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.append_nautilus_metadata_to_state(state, testcase) + self.append_nautilus_metadata_to_state(state, input, md) } #[cfg(feature = "track_hit_feedbacks")] diff --git a/crates/libafl/src/feedbacks/simd.rs b/crates/libafl/src/feedbacks/simd.rs index ef31517b0ee..7876e8d3497 100644 --- a/crates/libafl/src/feedbacks/simd.rs +++ b/crates/libafl/src/feedbacks/simd.rs @@ -253,8 +253,10 @@ where state: &mut S, manager: &mut EM, observers: &OT, + input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.map.append_metadata(state, manager, observers, md) + self.map + .append_metadata(state, manager, observers, input, md) } } diff --git a/crates/libafl/src/feedbacks/stdio.rs b/crates/libafl/src/feedbacks/stdio.rs index 41735391058..4781d15e837 100644 --- a/crates/libafl/src/feedbacks/stdio.rs +++ b/crates/libafl/src/feedbacks/stdio.rs @@ -74,6 +74,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.append_stdout_observation_to_testcase(observers, md) @@ -130,6 +131,7 @@ where _state: &mut S, _manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { let observer = observers diff --git a/crates/libafl/src/fuzzer/mod.rs b/crates/libafl/src/fuzzer/mod.rs index b11208d9db9..4e37ae558af 100644 --- a/crates/libafl/src/fuzzer/mod.rs +++ b/crates/libafl/src/fuzzer/mod.rs @@ -1,6 +1,6 @@ //! The `Fuzzer` is the main struct for a fuzz campaign. -use alloc::{rc::Rc, string::ToString, vec::Vec}; +use alloc::{string::ToString, vec::Vec}; #[cfg(feature = "std")] use core::hash::Hash; use core::{fmt::Debug, time::Duration}; @@ -429,9 +429,9 @@ where self.feedback_mut() .append_hit_feedbacks(md.hit_feedbacks_mut())?; self.feedback_mut() - .append_metadata(state, manager, observers, &mut md)?; + .append_metadata(state, manager, observers, input, &mut md)?; - let id = state.corpus_mut().add(Rc::new(input.clone()), md)?; + let id = state.corpus_mut().add(input.clone(), md)?; self.scheduler_mut().on_add(state, id)?; Ok(Some(id)) @@ -447,10 +447,10 @@ where } #[cfg(feature = "track_hit_feedbacks")] self.objective_mut() - .append_hit_feedbacks(testcase.hit_objectives_mut())?; + .append_hit_feedbacks(md.hit_objectives_mut())?; self.objective_mut() - .append_metadata(state, manager, observers, &mut md)?; - state.solutions_mut().add(Rc::new(input.clone()), md)?; + .append_metadata(state, manager, observers, input, &mut md)?; + state.solutions_mut().add(input.clone(), md)?; Ok(None) } @@ -726,12 +726,15 @@ where #[cfg(feature = "track_hit_feedbacks")] self.objective_mut() .append_hit_feedbacks(tc_md.hit_objectives_mut())?; - self.objective_mut() - .append_metadata(state, manager, &*observers, &mut tc_md)?; + self.objective_mut().append_metadata( + state, + manager, + &*observers, + &input, + &mut tc_md, + )?; // we don't care about solution id - let id = state - .solutions_mut() - .add(Rc::new(input.clone()), tc_md.clone())?; + let id = state.solutions_mut().add(input.clone(), tc_md.clone())?; manager.fire( state, @@ -765,11 +768,11 @@ where #[cfg(feature = "track_hit_feedbacks")] self.feedback_mut() - .append_hit_feedbacks(testcase.hit_feedbacks_mut())?; + .append_hit_feedbacks(tc_md.hit_feedbacks_mut())?; // Add the input to the main corpus self.feedback_mut() - .append_metadata(state, manager, &*observers, &mut tc_md)?; - let id = state.corpus_mut().add(Rc::new(input.clone()), tc_md)?; + .append_metadata(state, manager, &*observers, &input, &mut tc_md)?; + let id = state.corpus_mut().add(input.clone(), tc_md)?; self.scheduler_mut().on_add(state, id)?; let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { @@ -803,9 +806,7 @@ where .build(); // Add the disabled input to the main corpus - let id = state - .corpus_mut() - .add_disabled(Rc::new(input.clone()), tc_md)?; + let id = state.corpus_mut().add_disabled(input.clone(), tc_md)?; Ok(id) } } diff --git a/crates/libafl/src/mutators/mutations.rs b/crates/libafl/src/mutators/mutations.rs index c4456b8a462..c85d51a2a7e 100644 --- a/crates/libafl/src/mutators/mutations.rs +++ b/crates/libafl/src/mutators/mutations.rs @@ -1793,6 +1793,8 @@ pub fn str_decode(item: &str) -> Result, Error> { #[cfg(test)] mod tests { + use alloc::rc::Rc; + use libafl_bolts::{ rands::StdRand, tuples::{HasConstLen, tuple_list, tuple_list_type}, @@ -1800,8 +1802,12 @@ mod tests { use super::*; use crate::{ - HasMetadata, corpus::InMemoryCorpus, feedbacks::ConstFeedback, inputs::BytesInput, - mutators::MutatorsTuple, state::StdState, + HasMetadata, + corpus::{InMemoryCorpus, TestcaseMetadata}, + feedbacks::ConstFeedback, + inputs::BytesInput, + mutators::MutatorsTuple, + state::StdState, }; type TestMutatorsTupleType = tuple_list_type!( @@ -1868,7 +1874,10 @@ mod tests { let mut objective = ConstFeedback::new(false); corpus - .add(BytesInput::new(vec![0x42; 0x1337]).into()) + .add( + Rc::new(BytesInput::new(vec![0x42; 0x1337])), + TestcaseMetadata::default(), + ) .unwrap(); StdState::new( diff --git a/crates/libafl/src/mutators/unicode/mod.rs b/crates/libafl/src/mutators/unicode/mod.rs index ec27f3d8d19..8c113aabb35 100644 --- a/crates/libafl/src/mutators/unicode/mod.rs +++ b/crates/libafl/src/mutators/unicode/mod.rs @@ -11,7 +11,7 @@ use libafl_bolts::{Error, HasLen, Named, rands::Rand}; use crate::{ HasMetadata, - corpus::{CorpusId, HasTestcase, Testcase}, + corpus::{Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, Testcase}, inputs::{BytesInput, HasMutatorBytes, ResizableMutator}, mutators::{MutationResult, Mutator, Tokens, rand_range}, nonzero, @@ -35,7 +35,10 @@ where { type Post = UnicodeIdentificationMetadata; - fn try_transform_from(base: &Testcase, state: &S) -> Result { + fn try_transform_from( + base: &Testcase, + state: &S, + ) -> Result { let input = base.load_input(state.corpus())?.clone(); let metadata = base.metadata::().cloned()?; Ok((input, metadata)) diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index 9b725a98e70..e37bf46b242 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -209,7 +209,7 @@ mod tests { { fn compute( _state: &S, - _entry: &mut Testcase>::TestcaseMetadataRefMut<'_>>, + _entry: &mut Testcase>::TestcaseMetadataCell>, ) -> Result { Ok(FACTOR) } diff --git a/crates/libafl/src/stages/afl_stats.rs b/crates/libafl/src/stages/afl_stats.rs index b9de076ed6d..9d27b52b41c 100644 --- a/crates/libafl/src/stages/afl_stats.rs +++ b/crates/libafl/src/stages/afl_stats.rs @@ -529,8 +529,9 @@ where } #[cfg(feature = "track_hit_feedbacks")] - fn maybe_update_last_crash(&mut self, testcase: &Testcase, state: &S) + fn maybe_update_last_crash(&mut self, testcase: &Testcase, state: &S) where + M: IsTestcaseMetadataCell, S: HasExecutions, { #[cfg(feature = "track_hit_feedbacks")] @@ -544,8 +545,9 @@ where } #[cfg(feature = "track_hit_feedbacks")] - fn maybe_update_last_hang(&mut self, testcase: &Testcase, state: &S) + fn maybe_update_last_hang(&mut self, testcase: &Testcase, state: &S) where + M: IsTestcaseMetadataCell, S: HasExecutions, { if testcase diff --git a/crates/libafl/src/stages/concolic.rs b/crates/libafl/src/stages/concolic.rs index 776c8d0d0ab..1805233acbf 100644 --- a/crates/libafl/src/stages/concolic.rs +++ b/crates/libafl/src/stages/concolic.rs @@ -377,7 +377,7 @@ where S: HasExecutions + HasCorpus + HasMetadata - + HasNamedMetadata + + HasNamedMetadataMut + HasCurrentTestcase + MaybeHasClientPerfMonitor + HasCurrentCorpusId, @@ -419,7 +419,7 @@ where #[cfg(feature = "concolic_mutation")] impl Restartable for SimpleConcolicMutationalStage where - S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/dump.rs b/crates/libafl/src/stages/dump.rs index 179f9fab142..1b4ede11dbd 100644 --- a/crates/libafl/src/stages/dump.rs +++ b/crates/libafl/src/stages/dump.rs @@ -6,7 +6,8 @@ use alloc::{ }; use core::{clone::Clone, marker::PhantomData}; use std::{ - fs, + fs::{self, File}, + io::Write, path::{Path, PathBuf}, }; @@ -14,8 +15,8 @@ use libafl_bolts::impl_serdeany; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, - corpus::{Corpus, CorpusId, Testcase}, + Error, HasMetadataMut, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase, TestcaseMetadata}, inputs::Input, stages::{Restartable, Stage}, state::{HasCorpus, HasRand, HasSolutions}, @@ -46,9 +47,9 @@ pub struct DumpToDiskStage { impl Stage for DumpToDiskStage where - CB1: FnMut(&Testcase>::TestcaseMetadataCell>, &S) -> Vec, - CB2: FnMut(&Testcase>::TestcaseMetadataCell>, &CorpusId) -> P, - S: HasCorpus + HasSolutions + HasRand + HasMetadata, + CB1: FnMut(&I, &TestcaseMetadata, &S) -> Vec, + CB2: FnMut(&I, &TestcaseMetadata, &CorpusId) -> P, + S: HasCorpus + HasSolutions + HasRand + HasMetadataMut, P: AsRef, { #[inline] @@ -56,11 +57,10 @@ where &mut self, _fuzzer: &mut Z, _executor: &mut E, - _state: &mut S, + state: &mut S, _manager: &mut EM, ) -> Result<(), Error> { - // self.dump_state_to_disk(state) - Ok(()) + self.dump_state_to_disk(state) } } @@ -100,7 +100,7 @@ impl Z, > where - S: HasCorpus + HasSolutions + HasRand + HasMetadata, + S: HasCorpus + HasSolutions + HasRand + HasMetadataMut, I: Input, { /// Create a new [`DumpToDiskStage`] with a default `generate_filename` function. @@ -142,7 +142,7 @@ where impl DumpToDiskStage where - S: HasMetadata + HasSolutions, + S: HasMetadataMut + HasSolutions, { /// Create a new [`DumpToDiskStage`] with a custom `generate_filename` function. pub fn new_with_custom_filenames( @@ -182,57 +182,62 @@ where }) } - // #[inline] - // fn dump_state_to_disk, M>(&mut self, state: &mut S) -> Result<(), Error> - // where - // CB1: FnMut(&Testcase, &S) -> Vec, - // CB2: FnMut(&Testcase, &CorpusId) -> P, - // S: HasCorpus, - // M: HasTestcaseMetadata, - // { - // let (mut corpus_id, mut solutions_id) = - // if let Some(meta) = state.metadata_map().get::() { - // ( - // meta.last_corpus.and_then(|x| state.corpus().next(x)), - // meta.last_solution.and_then(|x| state.solutions().next(x)), - // ) - // } else { - // (state.corpus().first(), state.solutions().first()) - // }; - - // while let Some(i) = corpus_id { - // let testcase = state.corpus().get(i)?; - // // state.corpus().load_input_into(&mut testcase)?; - // let bytes = (self.to_bytes)(&testcase, state); - - // let fname = self - // .corpus_dir - // .join((self.generate_filename)(&testcase, &i)); - // let mut f = File::create(fname)?; - // drop(f.write_all(&bytes)); - - // corpus_id = state.corpus().next(i); - // } - - // while let Some(i) = solutions_id { - // let testcase = state.solutions().get(i)?; - // // state.solutions().load_input_into(&mut testcase)?; - // let bytes = (self.to_bytes)(&testcase, state); - - // let fname = self - // .solutions_dir - // .join((self.generate_filename)(&testcase, &i)); - // let mut f = File::create(fname)?; - // drop(f.write_all(&bytes)); - - // solutions_id = state.solutions().next(i); - // } - - // state.add_metadata(DumpToDiskMetadata { - // last_corpus: state.corpus().last(), - // last_solution: state.solutions().last(), - // }); - - // Ok(()) - // } + #[inline] + fn dump_state_to_disk>(&mut self, state: &mut S) -> Result<(), Error> + where + CB1: FnMut(&I, &TestcaseMetadata, &S) -> Vec, + CB2: FnMut(&I, &TestcaseMetadata, &CorpusId) -> P, + S: HasCorpus, + { + let (mut corpus_id, mut solutions_id) = + if let Some(meta) = state.metadata_map().get::() { + ( + meta.last_corpus.and_then(|x| state.corpus().next(x)), + meta.last_solution.and_then(|x| state.solutions().next(x)), + ) + } else { + (state.corpus().first(), state.solutions().first()) + }; + + while let Some(i) = corpus_id { + let testcase = state.corpus().get(i)?; + + let input = testcase.input(); + let md = testcase.testcase_metadata(); + + let bytes = (self.to_bytes)(input.as_ref(), &*md, state); + + let fname = self + .corpus_dir + .join((self.generate_filename)(input.as_ref(), &*md, &i)); + let mut f = File::create(fname)?; + drop(f.write_all(&bytes)); + + corpus_id = state.corpus().next(i); + } + + while let Some(i) = solutions_id { + let testcase = state.solutions().get(i)?; + + let input = testcase.input(); + let md = testcase.testcase_metadata(); + + let bytes = (self.to_bytes)(input.as_ref(), &*md, state); + + let fname = self + .solutions_dir + .join((self.generate_filename)(input.as_ref(), &*md, &i)); + let mut f = File::create(fname)?; + drop(f.write_all(&bytes)); + + solutions_id = state.solutions().next(i); + } + + state.add_metadata(DumpToDiskMetadata { + last_corpus: state.corpus().last(), + last_solution: state.solutions().last(), + }); + + Ok(()) + } } diff --git a/crates/libafl/src/stages/tmin.rs b/crates/libafl/src/stages/tmin.rs index f961fded791..73606ec3655 100644 --- a/crates/libafl/src/stages/tmin.rs +++ b/crates/libafl/src/stages/tmin.rs @@ -289,9 +289,13 @@ where .parent_id(Some(base_corpus_id)) .build(); - fuzzer - .feedback_mut() - .append_metadata(state, manager, &*observers, &mut tc_md)?; + fuzzer.feedback_mut().append_metadata( + state, + manager, + &*observers, + &base, + &mut tc_md, + )?; let prev = state.corpus_mut().replace_metadata(base_corpus_id, tc_md)?; diff --git a/crates/libafl_frida/src/asan/errors.rs b/crates/libafl_frida/src/asan/errors.rs index fc34941ed4e..d642cc6d3e8 100644 --- a/crates/libafl_frida/src/asan/errors.rs +++ b/crates/libafl_frida/src/asan/errors.rs @@ -13,7 +13,7 @@ use frida_gum::interceptor::Interceptor; use frida_gum::{Gum, Process}; use libafl::{ Error, HasMetadata, - corpus::Testcase, + corpus::{Testcase, TestcaseMetadata}, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, observers::Observer, @@ -710,10 +710,11 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _input: &I, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { if let Some(errors) = &self.errors { - testcase.add_metadata(errors.clone()); + md.add_metadata(errors.clone()); } Ok(()) diff --git a/crates/libafl_libfuzzer/runtime/src/feedbacks.rs b/crates/libafl_libfuzzer/runtime/src/feedbacks.rs index 130a29bfcb4..5612ca3adaa 100644 --- a/crates/libafl_libfuzzer/runtime/src/feedbacks.rs +++ b/crates/libafl_libfuzzer/runtime/src/feedbacks.rs @@ -140,7 +140,8 @@ where _state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _input: &I, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { match self.exit_kind { ExitKind::Crash | ExitKind::Oom if OomFeedback::oomed() => { diff --git a/fuzzers/baby/tutorial/src/metadata.rs b/fuzzers/baby/tutorial/src/metadata.rs index 0a778c4fc17..ef45a07f50d 100644 --- a/fuzzers/baby/tutorial/src/metadata.rs +++ b/fuzzers/baby/tutorial/src/metadata.rs @@ -64,11 +64,10 @@ impl Feedback for PacketLenFeedback { _state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _input: &PacketData, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { - testcase - .metadata_map_mut() - .insert(PacketLenMetadata { length: self.len }); + md.insert(PacketLenMetadata { length: self.len }); Ok(()) } } diff --git a/fuzzers/forkserver/libafl-fuzz/src/feedback/filepath.rs b/fuzzers/forkserver/libafl-fuzz/src/feedback/filepath.rs index 6f6b8c499a0..bc21fce4991 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/feedback/filepath.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/feedback/filepath.rs @@ -90,7 +90,8 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, - testcase: &mut Testcase, + _input: &I, + md: &mut TestcaseMetadata, ) -> Result<(), Error> { (self.func)(state, testcase, &self.out_dir)?; Ok(()) diff --git a/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs b/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs index 4746d03eb2d..a6b366c6e34 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/feedback/persistent_record.rs @@ -78,6 +78,7 @@ where state: &mut S, _manager: &mut EM, _observers: &OT, + _input: &I, _md: &mut TestcaseMetadata, ) -> Result<(), Error> { if self.should_run() { diff --git a/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs b/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs index 04b43273d88..8a145986796 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/feedback/seed.rs @@ -93,6 +93,7 @@ where state: &mut S, manager: &mut EM, observers: &OT, + _input: &I, md: &mut TestcaseMetadata, ) -> Result<(), Error> { self.inner From 5ed2c4e3a25adfee33b7723af8236a403a3c571c Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 16:34:32 +0200 Subject: [PATCH 07/12] possibility to disable corpus entry --- crates/libafl/src/corpus/combined.rs | 6 +++ crates/libafl/src/corpus/minimizer.rs | 40 ++++++++----------- crates/libafl/src/corpus/mod.rs | 3 ++ crates/libafl/src/corpus/single.rs | 4 ++ crates/libafl/src/corpus/store/inmemory.rs | 6 +++ crates/libafl/src/corpus/store/maps.rs | 24 +++++++++++ crates/libafl/src/corpus/store/mod.rs | 2 + crates/libafl/src/mutators/mutations.rs | 4 +- .../src/schedulers/probabilistic_sampling.rs | 3 +- 9 files changed, 65 insertions(+), 27 deletions(-) diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index e3d61a129a1..94740b0988e 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -78,6 +78,12 @@ where cache.get_from::(id, cache_store, &self.fallback_store) } + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.cache.borrow_mut().disable(id, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store) + } + fn replace_metadata( &mut self, id: CorpusId, diff --git a/crates/libafl/src/corpus/minimizer.rs b/crates/libafl/src/corpus/minimizer.rs index 9b5b82809b2..ad88175ce9a 100644 --- a/crates/libafl/src/corpus/minimizer.rs +++ b/crates/libafl/src/corpus/minimizer.rs @@ -14,7 +14,7 @@ use z3::{Optimize, ast::Bool}; use crate::{ Error, HasMetadata, HasScheduler, - corpus::Corpus, + corpus::{Corpus, IsTestcaseMetadataCell}, events::{Event, EventFirer, EventWithStats, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, inputs::Input, @@ -97,31 +97,24 @@ where let mut curr = 0; while let Some(id) = cur_id { let (weight, executions) = { - if state.corpus().get(id)?.borrow().scheduled_count() == 0 { - // Execute the input; we cannot rely on the metadata already being present. - - let input = state - .corpus() - .get(id)? - .borrow_mut() - .load_input(state.corpus())? - .clone(); - - let (exit_kind, mut total_time, _) = - run_target_with_timing(fuzzer, executor, state, mgr, &input, false)?; - if exit_kind != ExitKind::Ok { - total_time = Duration::from_secs(1); + { + let tc = state.corpus().get(id)?; + let md = tc.testcase_metadata(); + if md.scheduled_count() == 0 { + // Execute the input; we cannot rely on the metadata already being present. + let input = tc.input(); + + let (exit_kind, mut total_time, _) = + run_target_with_timing(fuzzer, executor, state, mgr, input.as_ref(), false)?; + if exit_kind != ExitKind::Ok { + total_time = Duration::from_secs(1); + } + md.set_exec_time(total_time); } - state - .corpus() - .get(id)? - .borrow_mut() - .set_exec_time(total_time); } - let mut testcase = state.corpus().get(id)?.borrow_mut(); ( - TS::compute(state, &mut *testcase)? + TS::compute(state, id)? .to_u64() .expect("Weight must be computable."), *state.executions(), @@ -214,7 +207,8 @@ where continue; } - let removed = state.corpus_mut().remove(id)?; + let removed = state.corpus().disable(id)?; + // scheduler needs to know we've removed the input, or it will continue to try // to use now-missing inputs fuzzer diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index 8a5acbf8ac2..1c99e0e2eb5 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -190,6 +190,9 @@ pub trait Corpus: Sized { id: CorpusId, ) -> Result, Error>; + /// Disable a corpus entry + fn disable(&mut self, id: CorpusId) -> Result<(), Error>; + /// Replace a [`TestcaseMetadata`] by another one. fn replace_metadata( &mut self, diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 584eee55e10..257744dcc90 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -88,6 +88,10 @@ where self.store.get_from::(id) } + fn disable(&mut self, id: CorpusId) { + + } + fn replace_metadata( &mut self, id: CorpusId, diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index 6c05e4b0d25..82ff8a294bf 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -95,6 +95,12 @@ where } } + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + let tc = self.enabled_map.remove(id).ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; + self.disabled_map.add(id, tc); + Ok(()) + } + fn replace_metadata( &mut self, _id: CorpusId, diff --git a/crates/libafl/src/corpus/store/maps.rs b/crates/libafl/src/corpus/store/maps.rs index 8f606c0c3a8..feedaf33f5d 100644 --- a/crates/libafl/src/corpus/store/maps.rs +++ b/crates/libafl/src/corpus/store/maps.rs @@ -33,6 +33,9 @@ pub trait InMemoryCorpusMap { testcase_metadata: TestcaseMetadata, ) -> Option; + /// Remove a testcase from the map, returning the removed testcase if present. + fn remove(&mut self, id: CorpusId) -> Option; + /// Get the prev corpus id in chronological order fn prev(&self, id: CorpusId) -> Option; @@ -175,6 +178,21 @@ where self.map.get_mut(&id).map(|storage| &mut storage.testcase) } + fn remove(&mut self, id: CorpusId) -> Option { + let entry = self.map.remove(&id)?; + self.history.remove(id); + + if let Some(prev) = &entry.prev { + self.map.get_mut(prev).unwrap().next = entry.next; + } + + if let Some(next) = &entry.next { + self.map.get_mut(next).unwrap().prev = entry.prev; + } + + Some(entry.testcase) + } + fn replace_metadata( &mut self, id: CorpusId, @@ -237,6 +255,12 @@ where self.map.get_mut(&id) } + fn remove(&mut self, id: CorpusId) -> Option { + let ret = self.map.remove(&id)?; + self.history.remove(id); + Some(ret) + } + fn replace_metadata( &mut self, id: CorpusId, diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index 1f2e163fcb5..4c1ca5f67b1 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -61,6 +61,8 @@ pub trait Store { id: CorpusId, ) -> Result, Error>; + fn disable(&mut self, id: CorpusId) -> Result<(), Error>; + /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. fn replace_metadata( &mut self, diff --git a/crates/libafl/src/mutators/mutations.rs b/crates/libafl/src/mutators/mutations.rs index c85d51a2a7e..d2f8f4c3c01 100644 --- a/crates/libafl/src/mutators/mutations.rs +++ b/crates/libafl/src/mutators/mutations.rs @@ -1793,8 +1793,6 @@ pub fn str_decode(item: &str) -> Result, Error> { #[cfg(test)] mod tests { - use alloc::rc::Rc; - use libafl_bolts::{ rands::StdRand, tuples::{HasConstLen, tuple_list, tuple_list_type}, @@ -1875,7 +1873,7 @@ mod tests { corpus .add( - Rc::new(BytesInput::new(vec![0x42; 0x1337])), + BytesInput::new(vec![0x42; 0x1337]), TestcaseMetadata::default(), ) .unwrap(); diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index e37bf46b242..c88cb16b52d 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -197,6 +197,7 @@ mod tests { schedulers::{ProbabilitySamplingScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, StdState}, }; + use crate::corpus::CorpusId; const FACTOR: f64 = 1337.0; @@ -209,7 +210,7 @@ mod tests { { fn compute( _state: &S, - _entry: &mut Testcase>::TestcaseMetadataCell>, + _corpus_id: CorpusId, ) -> Result { Ok(FACTOR) } From 600d1622fc0e6167038e12e1867d067b85711008 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 16:42:13 +0200 Subject: [PATCH 08/12] implemented disable everywhere --- crates/libafl/src/corpus/cache.rs | 18 ++++++++++++++++++ crates/libafl/src/corpus/nop.rs | 4 ++++ crates/libafl/src/corpus/single.rs | 4 ++-- crates/libafl/src/corpus/store/mod.rs | 1 + crates/libafl/src/corpus/store/ondisk.rs | 6 ++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index bebc4104062..359578264be 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -76,6 +76,14 @@ pub trait Cache { fallback_store: &FS, ) -> Result, Error>; + /// Disable an entry + fn disable( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error>; + /// Replace a testcase in the cache fn replace_metadata( &mut self, @@ -288,6 +296,11 @@ where } } + fn disable(&mut self, id: CorpusId, cache_store: &mut CS, fallback_store: &mut FS) -> Result<(), Error> { + cache_store.disable(id)?; + fallback_store.disable(id) + } + // fn replace( // &mut self, // id: CorpusId, @@ -408,6 +421,11 @@ where ) } + fn disable(&mut self, id: CorpusId, cache_store: &mut CS, fallback_store: &mut FS) -> Result<(), Error> { + cache_store.disable(id)?; + fallback_store.disable(id) + } + fn replace_metadata( &mut self, _id: CorpusId, diff --git a/crates/libafl/src/corpus/nop.rs b/crates/libafl/src/corpus/nop.rs index cbb08f07b62..d5410ebd9ca 100644 --- a/crates/libafl/src/corpus/nop.rs +++ b/crates/libafl/src/corpus/nop.rs @@ -57,6 +57,10 @@ impl Corpus for NopCorpus { Err(Error::unsupported("Unsupported by NopCorpus")) } + fn disable(&mut self, _id: CorpusId) -> Result<(), Error> { + Err(Error::unsupported("Unsupported by NopCorpus")) + } + /// Current testcase scheduled #[inline] fn current(&self) -> &Option { diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 257744dcc90..4024fafc8c3 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -88,8 +88,8 @@ where self.store.get_from::(id) } - fn disable(&mut self, id: CorpusId) { - + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.store.disable(id) } fn replace_metadata( diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index 4c1ca5f67b1..acaf81431fa 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -61,6 +61,7 @@ pub trait Store { id: CorpusId, ) -> Result, Error>; + /// Disable a testcase by id fn disable(&mut self, id: CorpusId) -> Result<(), Error>; /// Replaces the [`Testcase`] at the given idx in the enabled set, returning the existing. diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index a0a5b4af11a..d3712c6f265 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -281,6 +281,12 @@ where self.disk_mgr.load_testcase(&tc_id) } + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + let tc = self.enabled_map.remove(id).ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; + self.disabled_map.add(id, tc); + Ok(()) + } + fn replace_metadata( &mut self, _id: CorpusId, From 7b8745bf8c7c4eb1daeeafbb60ac7794061fdaf6 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Thu, 18 Sep 2025 16:42:56 +0200 Subject: [PATCH 09/12] fmt --- crates/libafl/src/corpus/cache.rs | 14 ++++++++++++-- crates/libafl/src/corpus/combined.rs | 8 +++++--- crates/libafl/src/corpus/minimizer.rs | 10 ++++++++-- crates/libafl/src/corpus/store/inmemory.rs | 5 ++++- crates/libafl/src/corpus/store/ondisk.rs | 5 ++++- .../src/schedulers/probabilistic_sampling.rs | 8 ++------ 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index 359578264be..6c0a12cb11e 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -296,7 +296,12 @@ where } } - fn disable(&mut self, id: CorpusId, cache_store: &mut CS, fallback_store: &mut FS) -> Result<(), Error> { + fn disable( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { cache_store.disable(id)?; fallback_store.disable(id) } @@ -421,7 +426,12 @@ where ) } - fn disable(&mut self, id: CorpusId, cache_store: &mut CS, fallback_store: &mut FS) -> Result<(), Error> { + fn disable( + &mut self, + id: CorpusId, + cache_store: &mut CS, + fallback_store: &mut FS, + ) -> Result<(), Error> { cache_store.disable(id)?; fallback_store.disable(id) } diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index 94740b0988e..5a7b823e67e 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -79,9 +79,11 @@ where } fn disable(&mut self, id: CorpusId) -> Result<(), Error> { - self.cache.borrow_mut().disable(id, - &mut *self.cache_store.borrow_mut(), - &mut self.fallback_store) + self.cache.borrow_mut().disable( + id, + &mut *self.cache_store.borrow_mut(), + &mut self.fallback_store, + ) } fn replace_metadata( diff --git a/crates/libafl/src/corpus/minimizer.rs b/crates/libafl/src/corpus/minimizer.rs index ad88175ce9a..2ea75eb6e72 100644 --- a/crates/libafl/src/corpus/minimizer.rs +++ b/crates/libafl/src/corpus/minimizer.rs @@ -104,8 +104,14 @@ where // Execute the input; we cannot rely on the metadata already being present. let input = tc.input(); - let (exit_kind, mut total_time, _) = - run_target_with_timing(fuzzer, executor, state, mgr, input.as_ref(), false)?; + let (exit_kind, mut total_time, _) = run_target_with_timing( + fuzzer, + executor, + state, + mgr, + input.as_ref(), + false, + )?; if exit_kind != ExitKind::Ok { total_time = Duration::from_secs(1); } diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index 82ff8a294bf..28cebb034e0 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -96,7 +96,10 @@ where } fn disable(&mut self, id: CorpusId) -> Result<(), Error> { - let tc = self.enabled_map.remove(id).ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; + let tc = self + .enabled_map + .remove(id) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; self.disabled_map.add(id, tc); Ok(()) } diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index d3712c6f265..4915e04615d 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -282,7 +282,10 @@ where } fn disable(&mut self, id: CorpusId) -> Result<(), Error> { - let tc = self.enabled_map.remove(id).ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; + let tc = self + .enabled_map + .remove(id) + .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))?; self.disabled_map.add(id, tc); Ok(()) } diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index c88cb16b52d..8bad64a47cf 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -191,13 +191,12 @@ mod tests { use crate::{ Error, - corpus::{Corpus, InMemoryCorpus, Testcase}, + corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase}, feedbacks::ConstFeedback, inputs::bytes::BytesInput, schedulers::{ProbabilitySamplingScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, StdState}, }; - use crate::corpus::CorpusId; const FACTOR: f64 = 1337.0; @@ -208,10 +207,7 @@ mod tests { where S: HasCorpus, { - fn compute( - _state: &S, - _corpus_id: CorpusId, - ) -> Result { + fn compute(_state: &S, _corpus_id: CorpusId) -> Result { Ok(FACTOR) } } From 286a23611b93d14215a4a79b1ee422e0f9952cc2 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 19 Sep 2025 15:03:15 +0200 Subject: [PATCH 10/12] introduce collection.rs, containing all the standard corpus --- crates/libafl/src/corpus/cache.rs | 61 +- crates/libafl/src/corpus/collection.rs | 613 ++++++++++++++++++ crates/libafl/src/corpus/minimizer.rs | 4 +- crates/libafl/src/corpus/mod.rs | 66 +- crates/libafl/src/corpus/single.rs | 21 +- crates/libafl/src/corpus/store/inmemory.rs | 21 +- crates/libafl/src/corpus/store/maps.rs | 66 +- crates/libafl/src/corpus/store/ondisk.rs | 58 +- crates/libafl/src/corpus/testcase.rs | 6 +- crates/libafl/src/events/llmp/restarting.rs | 5 +- crates/libafl/src/executors/inprocess/mod.rs | 2 +- crates/libafl/src/feedbacks/nautilus.rs | 6 +- crates/libafl/src/feedbacks/value_bloom.rs | 6 +- crates/libafl/src/fuzzer/mod.rs | 14 +- crates/libafl/src/mutators/list.rs | 8 +- crates/libafl/src/mutators/multi.rs | 16 +- crates/libafl/src/mutators/mutations.rs | 15 +- crates/libafl/src/mutators/numeric.rs | 6 +- crates/libafl/src/mutators/scheduled.rs | 38 +- crates/libafl/src/mutators/unicode/mod.rs | 14 +- crates/libafl/src/schedulers/minimizer.rs | 11 +- crates/libafl/src/schedulers/mod.rs | 11 +- .../src/schedulers/probabilistic_sampling.rs | 9 +- crates/libafl/src/schedulers/queue.rs | 6 +- crates/libafl/src/schedulers/weighted.rs | 9 +- crates/libafl/src/stages/afl_stats.rs | 43 +- crates/libafl/src/stages/concolic.rs | 5 +- .../runtime/Cargo.toml.template | 2 +- crates/libafl_libfuzzer/runtime/src/merge.rs | 12 +- .../runtime/src/schedulers.rs | 7 +- .../forkserver/libafl-fuzz/src/scheduler.rs | 11 +- 31 files changed, 881 insertions(+), 291 deletions(-) create mode 100644 crates/libafl/src/corpus/collection.rs diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index 6c0a12cb11e..56c46d0d9b5 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -9,6 +9,7 @@ use alloc::{rc::Rc, vec::Vec}; use std::{cell::RefCell, collections::VecDeque, marker::PhantomData}; use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; use crate::{ corpus::{ @@ -28,7 +29,7 @@ pub trait HasCachePolicy { /// Propagate the changes when the cell gets dropped. /// Expect more writes to the fallback store. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct WritebackOnDropPolicy; impl HasCachePolicy for WritebackOnDropPolicy { fn dirty(&self, _corpus_id: CorpusId) { @@ -42,7 +43,7 @@ impl HasCachePolicy for WritebackOnDropPolicy { /// flushing the cache regularly. /// If the cache is not flushed, no data will be written to the fallback store, resulting in /// data loss. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct WritebackOnFlushPolicy { dirty_entries: RefCell>, } @@ -108,7 +109,7 @@ pub trait Cache { } /// A composed testcase metadata cell, linking the cached cell with the fallback cell. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell, @@ -118,14 +119,14 @@ where write_access: RefCell, cache_policy: Rc, cache_cell: CC, - fallback_cell: FC, + fallback_cell: Rc, } impl Clone for CacheTestcaseMetadataCell where CC: IsTestcaseMetadataCell + Clone, CP: HasCachePolicy, - FC: IsTestcaseMetadataCell + Clone, + FC: IsTestcaseMetadataCell, { fn clone(&self) -> Self { Self { @@ -137,18 +138,26 @@ where } } +pub type StdIdentityCacheTestcaseMetadataCell, FS: Store> = Rc< + CacheTestcaseMetadataCell< + CS::TestcaseMetadataCell, + WritebackOnFlushPolicy, + FS::TestcaseMetadataCell, + >, +>; + /// An identity cache, storing everything both in the cache and the backing store. /// /// It only supports [`WritebackOnFlushPolicy`] since all the testcases are stored in memory on load /// forever. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct IdentityCache { cell_map: RefCell, cache_policy: Rc, } /// A `First In / First Out` cache policy. -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct FifoCache { cached_ids: VecDeque, cache_max_len: usize, @@ -167,7 +176,7 @@ where cache_policy, write_access: RefCell::new(false), cache_cell, - fallback_cell, + fallback_cell: Rc::new(fallback_cell), } } } @@ -236,28 +245,10 @@ where CS: RemovableStore, FS: Store, I: Input, - M: InMemoryCorpusMap< - Testcase< - I, - Rc< - CacheTestcaseMetadataCell< - CS::TestcaseMetadataCell, - WritebackOnFlushPolicy, - FS::TestcaseMetadataCell, - >, - >, - >, - >, + M: InMemoryCorpusMap>>, >::TestcaseMetadataCell: Clone, - >::TestcaseMetadataCell: Clone, { - type TestcaseMetadataCell = Rc< - CacheTestcaseMetadataCell< - CS::TestcaseMetadataCell, - WritebackOnFlushPolicy, - FS::TestcaseMetadataCell, - >, - >; + type TestcaseMetadataCell = StdIdentityCacheTestcaseMetadataCell; fn add_shared( &mut self, @@ -306,20 +297,6 @@ where fallback_store.disable(id) } - // fn replace( - // &mut self, - // id: CorpusId, - // input: Rc, - // md: TestcaseMetadata, - // cache_store: &mut CS, - // fallback_store: &mut FS, - // ) -> Result, Error> { - // let old_tc = cache_store.replace(id, input.clone(), md.clone())?; - // fallback_store.replace(id, input, md)?; - - // Ok(old_tc) - // } - fn replace_metadata( &mut self, _id: CorpusId, diff --git a/crates/libafl/src/corpus/collection.rs b/crates/libafl/src/corpus/collection.rs new file mode 100644 index 00000000000..54db96e97e5 --- /dev/null +++ b/crates/libafl/src/corpus/collection.rs @@ -0,0 +1,613 @@ +use alloc::{rc::Rc, string::String}; +use std::{cell::RefCell, path::PathBuf}; + +use libafl_bolts::Error; +use serde::{Deserialize, Serialize}; + +use crate::{ + corpus::{ + CombinedCorpus, Corpus, CorpusId, FifoCache, IdentityCache, InMemoryStore, OnDiskStore, + SingleCorpus, Testcase, TestcaseMetadata, + cache::StdIdentityCacheTestcaseMetadataCell, + maps::{self, InMemoryCorpusMap, InMemoryTestcaseMap}, + store::{OnDiskMetadataFormat, Store, ondisk::OnDiskTestcaseCell}, + }, + inputs::Input, +}; + +#[cfg(not(feature = "corpus_btreemap"))] +type StdInMemoryMap = maps::HashCorpusMap; +#[cfg(feature = "corpus_btreemap")] +type StdInMemoryMap = maps::BtreeCorpusMap; + +type StdInMemoryTestcaseMetadataCell = Rc>; +type StdInMemoryTestcase = Testcase; +type InnerStdInMemoryCorpusMap = StdInMemoryMap>; +type InnerStdInMemoryStore = + InMemoryStore, StdInMemoryTestcaseMetadataCell>; +type InnerInMemoryCorpus = SingleCorpus>; + +type StdOnDiskTestcaseMetadataCell = Rc>; +type StdOnDiskTestcase = Testcase>; +type InnerStdOnDiskCorpusMap = StdInMemoryMap>; +type InnerStdOnDiskStore = OnDiskStore>; +#[cfg(feature = "std")] +type InnerOnDiskCorpus = SingleCorpus>; + +type InnerInMemoryOnDiskCorpus = CombinedCorpus< + IdentityCache< + StdInMemoryMap< + Testcase< + I, + StdIdentityCacheTestcaseMetadataCell< + I, + InnerStdInMemoryStore, + InnerStdOnDiskStore, + >, + >, + >, + >, + InnerStdInMemoryStore, + InnerStdOnDiskStore, + I, +>; + +type InnerCachedOnDiskCorpus = CombinedCorpus< + FifoCache, InnerStdOnDiskStore, I>, + InnerStdInMemoryStore, + InnerStdOnDiskStore, + I, +>; + +/// The standard fully in-memory corpus map. +#[repr(transparent)] +#[derive(Debug, Serialize)] +pub struct StdInMemoryCorpusMap(InnerStdInMemoryCorpusMap); + +/// The standard fully in-memory store. +#[repr(transparent)] +#[derive(Debug, Serialize)] +pub struct StdInMemoryStore(InnerStdInMemoryStore); + +/// The standard fully on-disk store. +#[repr(transparent)] +#[derive(Debug, Serialize)] +pub struct StdOnDiskStore(InnerStdOnDiskStore); + +/// The standard in-memory corpus. +#[repr(transparent)] +#[derive(Debug, Serialize, Deserialize)] +pub struct InMemoryCorpus(InnerInMemoryCorpus); + +/// The standard fully on-disk corpus. +#[cfg(feature = "std")] +#[repr(transparent)] +#[derive(Debug, Serialize, Deserialize)] +pub struct OnDiskCorpus(InnerOnDiskCorpus); + +/// The standard corpus for storing on disk and in-memory with a cache. +/// Useful for very large corpuses. +#[repr(transparent)] +#[derive(Debug, Serialize)] +pub struct CachedOnDiskCorpus(InnerCachedOnDiskCorpus); + +/// The standard corpus for storing on disk and in-memory. +#[repr(transparent)] +#[derive(Debug, Serialize)] +pub struct InMemoryOnDiskCorpus(InnerInMemoryOnDiskCorpus); + +impl InMemoryCorpusMap> for StdInMemoryCorpusMap +where + I: Input, +{ + fn count(&self) -> usize { + self.0.count() + } + + fn add(&mut self, id: CorpusId, testcase: Testcase>>) { + self.0.add(id, testcase) + } + + fn get(&self, id: CorpusId) -> Option<&Testcase>>> { + self.0.get(id) + } + + fn get_mut(&mut self, id: CorpusId) -> Option<&mut Testcase>>> { + self.0.get_mut(id) + } + + fn remove(&mut self, id: CorpusId) -> Option>>> { + self.0.remove(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.0.nth(nth) + } +} + +impl InMemoryTestcaseMap> for StdInMemoryCorpusMap +where + I: Input, +{ + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option { + self.0.replace_metadata(id, testcase_metadata) + } +} + +impl Store for StdInMemoryStore +where + I: Input, +{ + type TestcaseMetadataCell = as Store>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn add_shared( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result<(), Error> { + self.0.add_shared::(id, input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + metadata: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, metadata) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.0.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} + +impl Store for StdOnDiskStore +where + I: Input, +{ + type TestcaseMetadataCell = as Store>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn add_shared( + &mut self, + id: CorpusId, + input: Rc, + md: TestcaseMetadata, + ) -> Result<(), Error> { + self.0.add_shared::(id, input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + metadata: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, metadata) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth(&self, nth: usize) -> CorpusId { + self.0.nth(nth) + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} + +impl Default for InMemoryCorpus { + fn default() -> Self { + InMemoryCorpus(InnerInMemoryCorpus::default()) + } +} + +impl InMemoryCorpus { + pub fn new() -> Self { + Self::default() + } +} + +impl Corpus for InMemoryCorpus +where + I: Input, +{ + type TestcaseMetadataCell = as Corpus>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn count_all(&self) -> usize { + self.0.count_all() + } + + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { + self.0.add_shared::(input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + md: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, md) + } + + fn current(&self) -> &Option { + self.0.current() + } + + fn current_mut(&mut self) -> &mut Option { + self.0.current_mut() + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} + +#[cfg(feature = "std")] +impl OnDiskCorpus +where + I: Input, +{ + pub fn new(root: PathBuf) -> Result { + Ok(OnDiskCorpus(InnerOnDiskCorpus::new( + InnerStdOnDiskStore::new(root)?, + ))) + } + + pub fn new_with_format(root: PathBuf, md_format: OnDiskMetadataFormat) -> Result { + Ok(OnDiskCorpus(InnerOnDiskCorpus::new( + InnerStdOnDiskStore::new_with_format(root, md_format)?, + ))) + } +} + +#[cfg(feature = "std")] +impl Corpus for OnDiskCorpus +where + I: Input, +{ + type TestcaseMetadataCell = as Corpus>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn count_all(&self) -> usize { + self.0.count_all() + } + + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { + self.0.add_shared::(input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + md: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, md) + } + + fn current(&self) -> &Option { + self.0.current() + } + + fn current_mut(&mut self) -> &mut Option { + self.0.current_mut() + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} + +impl Corpus for InMemoryOnDiskCorpus +where + I: Input, +{ + type TestcaseMetadataCell = as Corpus>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn count_all(&self) -> usize { + self.0.count_all() + } + + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { + self.0.add_shared::(input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + md: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, md) + } + + fn current(&self) -> &Option { + self.0.current() + } + + fn current_mut(&mut self) -> &mut Option { + self.0.current_mut() + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} + +impl Corpus for CachedOnDiskCorpus { + type TestcaseMetadataCell = as Corpus>::TestcaseMetadataCell; + + fn count(&self) -> usize { + self.0.count() + } + + fn count_disabled(&self) -> usize { + self.0.count_disabled() + } + + fn count_all(&self) -> usize { + self.0.count_all() + } + + fn add_shared( + &mut self, + input: Rc, + md: TestcaseMetadata, + ) -> Result { + self.0.add_shared::(input, md) + } + + fn get_from( + &self, + id: CorpusId, + ) -> Result, Error> { + self.0.get_from::(id) + } + + fn disable(&mut self, id: CorpusId) -> Result<(), Error> { + self.0.disable(id) + } + + fn replace_metadata( + &mut self, + id: CorpusId, + md: TestcaseMetadata, + ) -> Result { + self.0.replace_metadata(id, md) + } + + fn current(&self) -> &Option { + self.0.current() + } + + fn current_mut(&mut self) -> &mut Option { + self.0.current_mut() + } + + fn next(&self, id: CorpusId) -> Option { + self.0.next(id) + } + + fn prev(&self, id: CorpusId) -> Option { + self.0.prev(id) + } + + fn first(&self) -> Option { + self.0.first() + } + + fn last(&self) -> Option { + self.0.last() + } + + fn nth_from_all(&self, nth: usize) -> CorpusId { + self.0.nth_from_all(nth) + } +} diff --git a/crates/libafl/src/corpus/minimizer.rs b/crates/libafl/src/corpus/minimizer.rs index 2ea75eb6e72..0c35d4055fd 100644 --- a/crates/libafl/src/corpus/minimizer.rs +++ b/crates/libafl/src/corpus/minimizer.rs @@ -217,9 +217,7 @@ where // scheduler needs to know we've removed the input, or it will continue to try // to use now-missing inputs - fuzzer - .scheduler_mut() - .on_remove(state, id, &Some(removed))?; + fuzzer.scheduler_mut().on_remove(state, id)?; } *state.corpus_mut().current_mut() = None; //we may have removed the current ID from the corpus diff --git a/crates/libafl/src/corpus/mod.rs b/crates/libafl/src/corpus/mod.rs index 1c99e0e2eb5..4e6e21e70c7 100644 --- a/crates/libafl/src/corpus/mod.rs +++ b/crates/libafl/src/corpus/mod.rs @@ -2,7 +2,6 @@ use alloc::rc::Rc; use core::{fmt, marker::PhantomData}; -use std::{cell::RefCell, string::String}; use serde::{Deserialize, Serialize}; @@ -36,44 +35,11 @@ pub use nop::NopCorpus; pub mod store; pub use store::{InMemoryStore, OnDiskStore, maps}; -/// The standard fully in-memory corpus map. -#[cfg(not(feature = "corpus_btreemap"))] -pub type StdInMemoryCorpusMap = maps::HashCorpusMap>>>; - -/// The standard fully in-memory corpus map. -#[cfg(feature = "corpus_btreemap")] -pub type StdInMemoryCorpusMap = maps::BtreeCorpusMap>>>; - -/// The standard fully in-memory store. -pub type StdInMemoryStore = - InMemoryStore, Rc>>; - -/// The standard fully on-disk store. -pub type StdOnDiskStore = OnDiskStore>; - -/// The standard in-memory corpus. -pub type InMemoryCorpus = SingleCorpus>; - -/// The standard fully on-disk corpus. -#[cfg(feature = "std")] -pub type OnDiskCorpus = SingleCorpus>>; - -/// The standard corpus for storing on disk and in-memory. -pub type InMemoryOnDiskCorpus = CombinedCorpus< - IdentityCache>, - StdInMemoryStore, - StdOnDiskStore, - I, ->; - -/// The standard corpus for storing on disk and in-memory with a cache. -/// Useful for very large corpuses. -pub type CachedOnDiskCorpus = CombinedCorpus< - FifoCache, StdOnDiskStore, I>, - StdInMemoryStore, - StdOnDiskStore, - I, ->; +pub mod collection; +pub use collection::{ + CachedOnDiskCorpus, InMemoryCorpus, InMemoryOnDiskCorpus, OnDiskCorpus, StdInMemoryCorpusMap, + StdInMemoryStore, StdOnDiskStore, +}; /// An abstraction for the index that identify a testcase in the corpus #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] @@ -155,12 +121,30 @@ pub trait Corpus: Sized { } /// Add an enabled testcase to the corpus and return its index - fn add(&mut self, input: I, md: TestcaseMetadata) -> Result { + /// + /// The default [`TestcaseMetadata`] will be instantiated. + fn add(&mut self, input: I) -> Result { + self.add_shared::(Rc::new(input), TestcaseMetadata::default()) + } + + /// Add an enabled testcase to the corpus and return its index + fn add_with_metadata(&mut self, input: I, md: TestcaseMetadata) -> Result { self.add_shared::(Rc::new(input), md) } /// Add a disabled testcase to the corpus and return its index - fn add_disabled(&mut self, input: I, md: TestcaseMetadata) -> Result { + /// + /// The default [`TestcaseMetadata`] will be instantiated. + fn add_disabled(&mut self, input: I) -> Result { + self.add_shared::(Rc::new(input), TestcaseMetadata::default()) + } + + /// Add a disabled testcase to the corpus and return its index + fn add_disabled_with_metadata( + &mut self, + input: I, + md: TestcaseMetadata, + ) -> Result { self.add_shared::(Rc::new(input), md) } diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 4024fafc8c3..0ed72c7a0fd 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -32,26 +32,23 @@ where S: Default, { fn default() -> Self { + Self::new(S::default()) + } +} + +impl SingleCorpus { + /// Create a new [`SingleCorpus`] + pub fn new(store: S) -> Self { Self { - store: S::default(), + store, counter: CorpusCounter::default(), - keys: Vec::new(), + keys: Vec::default(), current: None, phantom: PhantomData, } } } -impl SingleCorpus -where - S: Default, -{ - /// Create a new [`SingleCorpus`] - pub fn new() -> Self { - Self::default() - } -} - impl Corpus for SingleCorpus where S: Store, diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index 28cebb034e0..7d4a2be882e 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -6,7 +6,7 @@ use core::marker::PhantomData; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; -use super::{InMemoryCorpusMap, Store}; +use super::{InMemoryCorpusMap, RemovableStore, Store}; use crate::{ corpus::{ CorpusId, Testcase, @@ -141,3 +141,22 @@ where } } } + +impl RemovableStore for InMemoryStore +where + M: InMemoryCorpusMap>, + TMC: HasInstantiableTestcaseMetadata + Clone, + I: Input, +{ + fn remove(&mut self, id: CorpusId) -> Result, Error> { + if let Some(tc) = self.enabled_map.remove(id) { + Ok(tc) + } else if let Some(tc) = self.disabled_map.remove(id) { + Ok(tc) + } else { + Err(Error::key_not_found(format!( + "Index {id} not found for remove" + ))) + } + } +} diff --git a/crates/libafl/src/corpus/store/maps.rs b/crates/libafl/src/corpus/store/maps.rs index feedaf33f5d..354b44a0cde 100644 --- a/crates/libafl/src/corpus/store/maps.rs +++ b/crates/libafl/src/corpus/store/maps.rs @@ -26,13 +26,6 @@ pub trait InMemoryCorpusMap { /// Get by id; considers only enabled testcases fn get_mut(&mut self, id: CorpusId) -> Option<&mut T>; - /// Replace the metadata of a given testcase - fn replace_metadata( - &mut self, - id: CorpusId, - testcase_metadata: TestcaseMetadata, - ) -> Option; - /// Remove a testcase from the map, returning the removed testcase if present. fn remove(&mut self, id: CorpusId) -> Option; @@ -52,6 +45,15 @@ pub trait InMemoryCorpusMap { fn nth(&self, nth: usize) -> CorpusId; } +pub trait InMemoryTestcaseMap: InMemoryCorpusMap { + /// Replace the metadata of a given testcase + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option; +} + /// A history for [`CorpusId`] #[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct CorpusIdHistory { @@ -132,10 +134,7 @@ impl CorpusIdHistory { } } -impl InMemoryCorpusMap for HashCorpusMap -where - T: IsTestcaseMetadataCell, -{ +impl InMemoryCorpusMap for HashCorpusMap { fn count(&self) -> usize { self.map.len() } @@ -193,15 +192,6 @@ where Some(entry.testcase) } - fn replace_metadata( - &mut self, - id: CorpusId, - testcase_metadata: TestcaseMetadata, - ) -> Option { - let old_tc = self.map.get_mut(&id)?; - Some(old_tc.testcase.replace_testcase_metadata(testcase_metadata)) - } - fn prev(&self, id: CorpusId) -> Option { match self.map.get(&id) { Some(item) => item.prev, @@ -229,10 +219,21 @@ where } } -impl InMemoryCorpusMap for BtreeCorpusMap +impl InMemoryTestcaseMap for HashCorpusMap where T: IsTestcaseMetadataCell, { + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option { + let old_tc = self.map.get_mut(&id)?; + Some(old_tc.testcase.replace_testcase_metadata(testcase_metadata)) + } +} + +impl InMemoryCorpusMap for BtreeCorpusMap { fn count(&self) -> usize { self.map.len() } @@ -261,15 +262,6 @@ where Some(ret) } - fn replace_metadata( - &mut self, - id: CorpusId, - testcase_metadata: TestcaseMetadata, - ) -> Option { - let tc = self.get_mut(id)?; - Some(tc.replace_testcase_metadata(testcase_metadata)) - } - fn prev(&self, id: CorpusId) -> Option { // TODO see if using self.keys is faster let mut range = self @@ -316,3 +308,17 @@ where self.history.nth(nth) } } + +impl InMemoryTestcaseMap for BtreeCorpusMap +where + T: IsTestcaseMetadataCell, +{ + fn replace_metadata( + &mut self, + id: CorpusId, + testcase_metadata: TestcaseMetadata, + ) -> Option { + let tc = self.get_mut(id)?; + Some(tc.replace_testcase_metadata(testcase_metadata)) + } +} diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index 4915e04615d..5c9af6ef67b 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -27,7 +27,7 @@ use crate::{ /// An on-disk store /// /// The maps only store the unique ID associated to the added [`Testcase`]s. -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct OnDiskStore { disk_mgr: Rc>, enabled_map: M, @@ -37,13 +37,22 @@ pub struct OnDiskStore { } /// A Disk Manager, able to load and store [`Testcase`]s -#[derive(Default, Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct DiskMgr { root_dir: PathBuf, md_format: OnDiskMetadataFormat, phantom: PhantomData, } +/// An on-disk [`Testcase`] cell. +#[derive(Debug)] +pub struct OnDiskTestcaseCell { + mgr: Rc>, + id: String, + testcase_md: RefCell, + modified: RefCell, +} + /// Options for the the format of the on-disk metadata #[derive(Default, Debug, Clone, Serialize, Deserialize)] pub enum OnDiskMetadataFormat { @@ -98,15 +107,6 @@ impl OnDiskMetadataFormat { } } -/// An on-disk [`Testcase`] cell. -#[derive(Debug)] -pub struct OnDiskTestcaseCell { - mgr: Rc>, - id: String, - testcase_md: RefCell, - modified: RefCell, -} - impl OnDiskTestcaseCell { /// Get a new [`OnDiskTestcaseCell`]. pub fn new(mgr: Rc>, id: String, testcase_md: TestcaseMetadata) -> Self { @@ -159,6 +159,21 @@ impl Drop for OnDiskTestcaseCell { } impl DiskMgr { + pub fn new(root_dir: PathBuf) -> Result { + Self::new_with_format(root_dir, OnDiskMetadataFormat::default()) + } + + pub fn new_with_format( + root_dir: PathBuf, + md_format: OnDiskMetadataFormat, + ) -> Result { + Ok(Self { + root_dir, + md_format, + phantom: PhantomData, + }) + } + fn testcase_path(&self, testcase_id: &String) -> PathBuf { self.root_dir.join(&testcase_id) } @@ -223,6 +238,27 @@ where } } +impl OnDiskStore +where + M: Default, +{ + pub fn new(root: PathBuf) -> Result { + Self::new_with_format(root, OnDiskMetadataFormat::default()) + } + + pub fn new_with_format(root: PathBuf, md_format: OnDiskMetadataFormat) -> Result { + let disk_mgr = Rc::new(DiskMgr::new_with_format(root, md_format)?); + + Ok(Self { + disk_mgr, + enabled_map: M::default(), + disabled_map: M::default(), + first: None, + last: None, + }) + } +} + impl Store for OnDiskStore where I: Input, diff --git a/crates/libafl/src/corpus/testcase.rs b/crates/libafl/src/corpus/testcase.rs index 265ac63648b..76a10809959 100644 --- a/crates/libafl/src/corpus/testcase.rs +++ b/crates/libafl/src/corpus/testcase.rs @@ -117,10 +117,6 @@ impl IsTestcaseMetadataCell for RefCell { type TestcaseMetadataRef<'a> = Ref<'a, TestcaseMetadata>; type TestcaseMetadataRefMut<'a> = RefMut<'a, TestcaseMetadata>; - // fn new(md: TestcaseMetadata) -> Self { - // RefCell::new(md) - // } - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { self.borrow() } @@ -253,9 +249,11 @@ pub struct TestcaseMetadata { objectives_found: usize, /// Vector of `Feedback` names that deemed this `Testcase` as corpus worthy #[cfg(feature = "track_hit_feedbacks")] + #[builder(default)] hit_feedbacks: Vec>, /// Vector of `Feedback` names that deemed this `Testcase` as solution worthy #[cfg(feature = "track_hit_feedbacks")] + #[builder(default)] hit_objectives: Vec>, } diff --git a/crates/libafl/src/events/llmp/restarting.rs b/crates/libafl/src/events/llmp/restarting.rs index 154e779ca55..f52606d5b93 100644 --- a/crates/libafl/src/events/llmp/restarting.rs +++ b/crates/libafl/src/events/llmp/restarting.rs @@ -989,7 +989,7 @@ mod tests { use crate::{ StdFuzzer, - corpus::{Corpus, InMemoryCorpus, Testcase}, + corpus::{Corpus, InMemoryCorpus}, events::llmp::restarting::{_ENV_FUZZER_SENDER, LlmpEventManagerBuilder}, executors::{ExitKind, InProcessExecutor}, feedbacks::ConstFeedback, @@ -1018,8 +1018,7 @@ mod tests { let time = TimeObserver::new("time"); let mut corpus = InMemoryCorpus::::new(); - let testcase = Testcase::new(vec![0; 4].into()); - corpus.add(testcase).unwrap(); + corpus.add(vec![0; 4].into()).unwrap(); let solutions = InMemoryCorpus::::new(); diff --git a/crates/libafl/src/executors/inprocess/mod.rs b/crates/libafl/src/executors/inprocess/mod.rs index 572850b3ace..c6d37889f5d 100644 --- a/crates/libafl/src/executors/inprocess/mod.rs +++ b/crates/libafl/src/executors/inprocess/mod.rs @@ -363,7 +363,7 @@ pub fn run_observers_and_save_state( state .solutions_mut() - .add(input.clone(), testcase_md) + .add_with_metadata(input.clone(), testcase_md) .expect("In run_observers_and_save_state solutions failure."); let event = Event::Objective { diff --git a/crates/libafl/src/feedbacks/nautilus.rs b/crates/libafl/src/feedbacks/nautilus.rs index 8a0f36f57be..ac760215000 100644 --- a/crates/libafl/src/feedbacks/nautilus.rs +++ b/crates/libafl/src/feedbacks/nautilus.rs @@ -7,7 +7,7 @@ use libafl_bolts::Named; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, + Error, HasMetadataMut, common::nautilus::grammartec::{chunkstore::ChunkStore, context::Context}, corpus::TestcaseMetadata, executors::ExitKind, @@ -68,7 +68,7 @@ impl<'a> NautilusFeedback<'a> { md: &mut TestcaseMetadata, ) -> Result<(), Error> where - S: HasCorpus + HasMetadata, + S: HasCorpus + HasMetadataMut, { let meta = state .metadata_map_mut() @@ -91,7 +91,7 @@ impl StateInitializer for NautilusFeedback<'_> {} impl Feedback for NautilusFeedback<'_> where - S: HasMetadata + HasCorpus, + S: HasMetadataMut + HasCorpus, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/value_bloom.rs b/crates/libafl/src/feedbacks/value_bloom.rs index 9c87cd24cc5..f267e843166 100644 --- a/crates/libafl/src/feedbacks/value_bloom.rs +++ b/crates/libafl/src/feedbacks/value_bloom.rs @@ -11,7 +11,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - HasNamedMetadata, + HasNamedMetadataMut, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, observers::{ObserversTuple, ValueObserver}, @@ -59,7 +59,7 @@ impl Named for ValueBloomFeedback<'_, T> { } } -impl StateInitializer for ValueBloomFeedback<'_, T> { +impl StateInitializer for ValueBloomFeedback<'_, T> { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { let _ = state.named_metadata_or_insert_with::(&self.name, || { @@ -71,7 +71,7 @@ impl StateInitializer for ValueBloomFeedback<'_, T> { } } -impl, S: HasNamedMetadata, T: Hash> Feedback +impl, S: HasNamedMetadataMut, T: Hash> Feedback for ValueBloomFeedback<'_, T> { fn is_interesting( diff --git a/crates/libafl/src/fuzzer/mod.rs b/crates/libafl/src/fuzzer/mod.rs index 4e37ae558af..e7e34140447 100644 --- a/crates/libafl/src/fuzzer/mod.rs +++ b/crates/libafl/src/fuzzer/mod.rs @@ -431,7 +431,7 @@ where self.feedback_mut() .append_metadata(state, manager, observers, input, &mut md)?; - let id = state.corpus_mut().add(input.clone(), md)?; + let id = state.corpus_mut().add_with_metadata(input.clone(), md)?; self.scheduler_mut().on_add(state, id)?; Ok(Some(id)) @@ -450,7 +450,7 @@ where .append_hit_feedbacks(md.hit_objectives_mut())?; self.objective_mut() .append_metadata(state, manager, observers, input, &mut md)?; - state.solutions_mut().add(input.clone(), md)?; + state.solutions_mut().add_with_metadata(input.clone(), md)?; Ok(None) } @@ -734,7 +734,9 @@ where &mut tc_md, )?; // we don't care about solution id - let id = state.solutions_mut().add(input.clone(), tc_md.clone())?; + let id = state + .solutions_mut() + .add_with_metadata(input.clone(), tc_md.clone())?; manager.fire( state, @@ -772,7 +774,7 @@ where // Add the input to the main corpus self.feedback_mut() .append_metadata(state, manager, &*observers, &input, &mut tc_md)?; - let id = state.corpus_mut().add(input.clone(), tc_md)?; + let id = state.corpus_mut().add_with_metadata(input.clone(), tc_md)?; self.scheduler_mut().on_add(state, id)?; let observers_buf = if manager.configuration() == EventConfig::AlwaysUnique { @@ -806,7 +808,9 @@ where .build(); // Add the disabled input to the main corpus - let id = state.corpus_mut().add_disabled(input.clone(), tc_md)?; + let id = state + .corpus_mut() + .add_disabled_with_metadata(input.clone(), tc_md)?; Ok(id) } } diff --git a/crates/libafl/src/mutators/list.rs b/crates/libafl/src/mutators/list.rs index 7278ce6f00d..195f0c8a8c4 100644 --- a/crates/libafl/src/mutators/list.rs +++ b/crates/libafl/src/mutators/list.rs @@ -179,8 +179,8 @@ where let other_idx_raw = state.rand_mut().next() as usize; let id = random_corpus_id!(state.corpus(), state.rand_mut()); - let mut testcase = state.corpus().get(id)?.borrow_mut(); - let other = testcase.load_input(state.corpus())?; + let testcase = state.corpus().get(id)?; + let other = testcase.input(); let other_len = other.len(); @@ -232,8 +232,8 @@ where let other_idx_raw = state.rand_mut().next() as usize; let id = random_corpus_id!(state.corpus(), state.rand_mut()); - let mut testcase = state.corpus().get(id)?.borrow_mut(); - let other = testcase.load_input(state.corpus())?; + let testcase = state.corpus().get(id)?; + let other = testcase.input(); let other_len = other.len(); diff --git a/crates/libafl/src/mutators/multi.rs b/crates/libafl/src/mutators/multi.rs index f5a31c882d6..22704dd3999 100644 --- a/crates/libafl/src/mutators/multi.rs +++ b/crates/libafl/src/mutators/multi.rs @@ -202,8 +202,8 @@ where } } - let mut other_testcase = state.corpus().get(id)?.borrow_mut(); - let other = other_testcase.load_input(state.corpus())?; + let other_testcase = state.corpus().get(id)?; + let other = other_testcase.input(); let other_len = other.len(); if other_len == 0 { return Ok(MutationResult::Skipped); @@ -237,9 +237,9 @@ where NonZero::new_unchecked(min(other_size, size - target)) }); - let other_testcase = state.corpus().get(id)?.borrow_mut(); + let other_testcase = state.corpus().get(id)?; // No need to load the input again, it'll still be cached. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); Ok(Self::crossover_insert( part, @@ -341,8 +341,8 @@ where } } - let mut other_testcase = state.corpus().get(id)?.borrow_mut(); - let other = other_testcase.load_input(state.corpus())?; + let other_testcase = state.corpus().get(id)?; + let other = other_testcase.input(); let other_len = other.len(); if other_len == 0 { @@ -376,9 +376,9 @@ where NonZero::new_unchecked(min(other_size, size - target)) }); - let other_testcase = state.corpus().get(id)?.borrow_mut(); + let other_testcase = state.corpus().get(id)?; // No need to load the input again, it'll still be cached. - let other = other_testcase.input().as_ref().unwrap(); + let other = other_testcase.input(); Ok(Self::crossover_replace( part, diff --git a/crates/libafl/src/mutators/mutations.rs b/crates/libafl/src/mutators/mutations.rs index d2f8f4c3c01..2e628437c01 100644 --- a/crates/libafl/src/mutators/mutations.rs +++ b/crates/libafl/src/mutators/mutations.rs @@ -1800,12 +1800,8 @@ mod tests { use super::*; use crate::{ - HasMetadata, - corpus::{InMemoryCorpus, TestcaseMetadata}, - feedbacks::ConstFeedback, - inputs::BytesInput, - mutators::MutatorsTuple, - state::StdState, + HasMetadata, corpus::InMemoryCorpus, feedbacks::ConstFeedback, inputs::BytesInput, + mutators::MutatorsTuple, state::StdState, }; type TestMutatorsTupleType = tuple_list_type!( @@ -1871,12 +1867,7 @@ mod tests { let mut feedback = ConstFeedback::new(false); let mut objective = ConstFeedback::new(false); - corpus - .add( - BytesInput::new(vec![0x42; 0x1337]), - TestcaseMetadata::default(), - ) - .unwrap(); + corpus.add(BytesInput::new(vec![0x42; 0x1337])).unwrap(); StdState::new( rand, diff --git a/crates/libafl/src/mutators/numeric.rs b/crates/libafl/src/mutators/numeric.rs index 3d381dcbc0b..feabb057b23 100644 --- a/crates/libafl/src/mutators/numeric.rs +++ b/crates/libafl/src/mutators/numeric.rs @@ -502,13 +502,13 @@ mod tests { use libafl_bolts::{ rands::{Rand, XkcdRand}, - tuples::IntoVec as _, + tuples::IntoVec, }; use serde::{Deserialize, Serialize}; use super::{Numeric, int_mutators}; use crate::{ - corpus::{Corpus as _, InMemoryCorpus, Testcase}, + corpus::{Corpus, InMemoryCorpus}, inputs::value::I16Input, mutators::MutationResult, state::StdState, @@ -544,7 +544,7 @@ mod tests { #[test] fn all_mutate_owned() { let mut corpus = InMemoryCorpus::new(); - corpus.add(Testcase::new(42_i16.into())).unwrap(); + corpus.add(42_i16.into()).unwrap(); let mut state = StdState::new( XkcdRand::new(), corpus, diff --git a/crates/libafl/src/mutators/scheduled.rs b/crates/libafl/src/mutators/scheduled.rs index e33fc9ca79d..d4f711bc498 100644 --- a/crates/libafl/src/mutators/scheduled.rs +++ b/crates/libafl/src/mutators/scheduled.rs @@ -392,7 +392,7 @@ mod tests { use libafl_bolts::rands::{StdRand, XkcdRand}; use crate::{ - corpus::{Corpus, InMemoryCorpus, Testcase}, + corpus::{Corpus, InMemoryCorpus}, feedbacks::ConstFeedback, inputs::{BytesInput, HasMutatorBytes}, mutators::{ @@ -408,14 +408,10 @@ mod tests { fn test_mut_scheduled() { let rand = XkcdRand::with_seed(0); let mut corpus: InMemoryCorpus = InMemoryCorpus::new(); - corpus - .add(Testcase::new(vec![b'a', b'b', b'c'].into())) - .unwrap(); - corpus - .add(Testcase::new(vec![b'd', b'e', b'f'].into())) - .unwrap(); + corpus.add(vec![b'a', b'b', b'c'].into()).unwrap(); + corpus.add(vec![b'd', b'e', b'f'].into()).unwrap(); - let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap(); + let mut input = corpus.get(corpus.first().unwrap()).unwrap().input().clone(); let mut feedback = ConstFeedback::new(false); let mut objective = ConstFeedback::new(false); @@ -442,10 +438,15 @@ mod tests { fn test_havoc() { let rand = StdRand::with_seed(0x1337); let mut corpus: InMemoryCorpus = InMemoryCorpus::new(); - corpus.add(Testcase::new(b"abc".to_vec().into())).unwrap(); - corpus.add(Testcase::new(b"def".to_vec().into())).unwrap(); - - let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap(); + corpus.add(b"abc".to_vec().into()).unwrap(); + corpus.add(b"def".to_vec().into()).unwrap(); + + let mut input = corpus + .get(corpus.first().unwrap()) + .unwrap() + .input() + .as_ref() + .clone(); let input_prior = input.clone(); let mut feedback = ConstFeedback::new(false); @@ -483,10 +484,15 @@ mod tests { fn test_single_choice() { let rand = StdRand::with_seed(0x1337); let mut corpus: InMemoryCorpus = InMemoryCorpus::new(); - corpus.add(Testcase::new(b"abc".to_vec().into())).unwrap(); - corpus.add(Testcase::new(b"def".to_vec().into())).unwrap(); - - let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap(); + corpus.add(b"abc".to_vec().into()).unwrap(); + corpus.add(b"def".to_vec().into()).unwrap(); + + let mut input = corpus + .get(corpus.first().unwrap()) + .unwrap() + .input() + .as_ref() + .clone(); let input_prior = input.clone(); let mut feedback = ConstFeedback::new(false); diff --git a/crates/libafl/src/mutators/unicode/mod.rs b/crates/libafl/src/mutators/unicode/mod.rs index 8c113aabb35..71828c0240f 100644 --- a/crates/libafl/src/mutators/unicode/mod.rs +++ b/crates/libafl/src/mutators/unicode/mod.rs @@ -10,7 +10,7 @@ use core::{ use libafl_bolts::{Error, HasLen, Named, rands::Rand}; use crate::{ - HasMetadata, + HasMetadata, HasMetadataMut, corpus::{Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, Testcase}, inputs::{BytesInput, HasMutatorBytes, ResizableMutator}, mutators::{MutationResult, Mutator, Tokens, rand_range}, @@ -39,8 +39,11 @@ where base: &Testcase, state: &S, ) -> Result { - let input = base.load_input(state.corpus())?.clone(); - let metadata = base.metadata::().cloned()?; + let input = base.input().as_ref().clone(); + let metadata = base + .testcase_metadata() + .metadata::() + .cloned()?; Ok((input, metadata)) } @@ -55,8 +58,9 @@ where { fn post_exec(self, state: &mut S, corpus_id: Option) -> Result<(), Error> { if let Some(corpus_id) = corpus_id { - let mut tc = state.testcase_mut(corpus_id)?; - tc.add_metadata(self); + let tc = state.testcase(corpus_id)?; + let mut md = tc.testcase_metadata_mut(); + md.add_metadata(self); } Ok(()) } diff --git a/crates/libafl/src/schedulers/minimizer.rs b/crates/libafl/src/schedulers/minimizer.rs index ba8ffc36097..b9eaf7f1333 100644 --- a/crates/libafl/src/schedulers/minimizer.rs +++ b/crates/libafl/src/schedulers/minimizer.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use super::HasQueueCycles; use crate::{ Error, HasMetadata, HasMetadataMut, - corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, feedbacks::MapIndexesMetadata, observers::CanTrack, require_index_tracking, @@ -87,13 +87,8 @@ where S: HasCorpus + HasMetadataMut + HasRand, { /// Removes an entry from the corpus - fn on_remove( - &mut self, - state: &mut S, - id: CorpusId, - testcase: &Testcase>::TestcaseMetadataCell>, - ) -> Result<(), Error> { - self.base.on_remove(state, id, testcase)?; + fn on_remove(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { + self.base.on_remove(state, id)?; let mut entries = if let Some(meta) = state.metadata_map_mut().get_mut::() { meta.map diff --git a/crates/libafl/src/schedulers/mod.rs b/crates/libafl/src/schedulers/mod.rs index 1409654d472..b4e78288689 100644 --- a/crates/libafl/src/schedulers/mod.rs +++ b/crates/libafl/src/schedulers/mod.rs @@ -11,9 +11,7 @@ use libafl_bolts::{ use crate::{ Error, HasMetadata, HasMetadataMut, - corpus::{ - Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, SchedulerTestcaseMetadata, Testcase, - }, + corpus::{Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, SchedulerTestcaseMetadata}, random_corpus_id, state::{HasCorpus, HasRand}, }; @@ -51,12 +49,7 @@ where { /// Removed the given entry from the corpus at the given index /// When you remove testcases, make sure that testcase is not currently fuzzed one! - fn on_remove( - &mut self, - _state: &mut S, - _id: CorpusId, - _testcase: &Testcase>::TestcaseMetadataCell>, - ) -> Result<(), Error> { + fn on_remove(&mut self, _state: &mut S, _id: CorpusId) -> Result<(), Error> { Ok(()) } diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index 8bad64a47cf..7b6ff979f33 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase}, + corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, schedulers::{RemovableScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, HasRand}, }; @@ -88,12 +88,7 @@ where F: TestcaseScore, S: HasCorpus + HasMetadataMut + HasRand, { - fn on_remove( - &mut self, - state: &mut S, - id: CorpusId, - _testcase: &Testcase>::TestcaseMetadataCell>, - ) -> Result<(), Error> { + fn on_remove(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { let meta = state .metadata_map_mut() .get_mut::() diff --git a/crates/libafl/src/schedulers/queue.rs b/crates/libafl/src/schedulers/queue.rs index 641555ef0ee..948175c5a16 100644 --- a/crates/libafl/src/schedulers/queue.rs +++ b/crates/libafl/src/schedulers/queue.rs @@ -101,7 +101,7 @@ mod tests { use libafl_bolts::rands::StdRand; use crate::{ - corpus::{Corpus, OnDiskCorpus, Testcase}, + corpus::{Corpus, OnDiskCorpus}, feedbacks::ConstFeedback, inputs::bytes::BytesInput, schedulers::{QueueScheduler, Scheduler}, @@ -115,8 +115,8 @@ mod tests { let mut q = OnDiskCorpus::::new(PathBuf::from("target/.test/fancy/path")).unwrap(); - let t = Testcase::with_filename(BytesInput::new(vec![0_u8; 4]), "fancyfile".into()); - q.add(t).unwrap(); + // let t = Testcase::with_filename(), "fancyfile".into()); + q.add(BytesInput::new(vec![0_u8; 4])).unwrap(); let objective_q = OnDiskCorpus::::new(PathBuf::from("target/.test/fancy/objective/path")) diff --git a/crates/libafl/src/schedulers/weighted.rs b/crates/libafl/src/schedulers/weighted.rs index e920acca6f2..a258d1a1201 100644 --- a/crates/libafl/src/schedulers/weighted.rs +++ b/crates/libafl/src/schedulers/weighted.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use crate::{ Error, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcase, Testcase}, + corpus::{Corpus, CorpusId, HasTestcase}, random_corpus_id, schedulers::{ AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler, on_add_metadata_default, @@ -261,12 +261,7 @@ where S: HasCorpus, { /// This will *NOT* neutralize the effect of this removed testcase from the global data such as `SchedulerMetadata` - fn on_remove( - &mut self, - _state: &mut S, - _id: CorpusId, - _testcase: &Testcase>::TestcaseMetadataCell>, - ) -> Result<(), Error> { + fn on_remove(&mut self, _state: &mut S, _id: CorpusId) -> Result<(), Error> { self.table_invalidated = true; Ok(()) } diff --git a/crates/libafl/src/stages/afl_stats.rs b/crates/libafl/src/stages/afl_stats.rs index 9d27b52b41c..a0aecead89e 100644 --- a/crates/libafl/src/stages/afl_stats.rs +++ b/crates/libafl/src/stages/afl_stats.rs @@ -27,7 +27,7 @@ use crate::feedbacks::{CRASH_FEEDBACK_NAME, TIMEOUT_FEEDBACK_NAME}; use crate::{ Error, HasMetadata, HasNamedMetadata, HasScheduler, corpus::{ - Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata, Testcase, + Corpus, HasCurrentCorpusId, SchedulerTestcaseMetadata, TestcaseMetadata, testcase::IsTestcaseMetadataCell, }, events::{Event, EventFirer, EventWithStats}, @@ -277,24 +277,25 @@ where )); }; let testcase = state.corpus().get(corpus_idx)?; + let md = &*testcase.testcase_metadata(); // NOTE: scheduled_count represents the amount of fuzz runs a // testcase has had. Since this stage is kept at the very end of stage list, // the entry would have been fuzzed already (and should contain IsFavoredMetadata) but would have a scheduled count of zero // since the scheduled count is incremented after all stages have been run. - if testcase.scheduled_count() == 0 { + if md.scheduled_count() == 0 { // New testcase! self.cycles_wo_finds = 0; self.update_last_find(); #[cfg(feature = "track_hit_feedbacks")] { - self.maybe_update_last_crash(&testcase, state); - self.maybe_update_last_hang(&testcase, state); + self.maybe_update_last_crash(md, state); + self.maybe_update_last_hang(md, state); } self.update_has_fuzzed_size(); - self.maybe_update_is_favored_size(&testcase); + self.maybe_update_is_favored_size(md); } - self.maybe_update_slowest_exec(&testcase); - self.maybe_update_max_depth(&testcase); + self.maybe_update_slowest_exec(md); + self.maybe_update_max_depth(md); // See if we actually need to run the stage, if not, avoid dynamic value computation. if !self.check_interval() { @@ -492,17 +493,14 @@ where Ok(()) } - fn maybe_update_is_favored_size( - &mut self, - testcase: &Testcase, - ) { - if testcase.has_metadata::() { + fn maybe_update_is_favored_size(&mut self, md: &TestcaseMetadata) { + if md.has_metadata::() { self.is_favored_size += 1; } } - fn maybe_update_slowest_exec(&mut self, testcase: &Testcase) { - if let Some(exec_time) = testcase.testcase_metadata().exec_time() { + fn maybe_update_slowest_exec(&mut self, md: &TestcaseMetadata) { + if let Some(exec_time) = md.exec_time() { if exec_time > &self.slowest_exec { self.slowest_exec = *exec_time; } @@ -513,11 +511,8 @@ where self.has_fuzzed_size += 1; } - fn maybe_update_max_depth(&mut self, testcase: &Testcase) { - if let Ok(metadata) = testcase - .testcase_metadata() - .metadata::() - { + fn maybe_update_max_depth(&mut self, md: &TestcaseMetadata) { + if let Ok(metadata) = md.metadata::() { if metadata.depth() > self.max_depth { self.max_depth = metadata.depth(); } @@ -529,13 +524,12 @@ where } #[cfg(feature = "track_hit_feedbacks")] - fn maybe_update_last_crash(&mut self, testcase: &Testcase, state: &S) + fn maybe_update_last_crash(&mut self, md: &TestcaseMetadata, state: &S) where - M: IsTestcaseMetadataCell, S: HasExecutions, { #[cfg(feature = "track_hit_feedbacks")] - if testcase + if md .hit_objectives() .contains(&Cow::Borrowed(CRASH_FEEDBACK_NAME)) { @@ -545,12 +539,11 @@ where } #[cfg(feature = "track_hit_feedbacks")] - fn maybe_update_last_hang(&mut self, testcase: &Testcase, state: &S) + fn maybe_update_last_hang(&mut self, md: &TestcaseMetadata, state: &S) where - M: IsTestcaseMetadataCell, S: HasExecutions, { - if testcase + if md .hit_objectives() .contains(&Cow::Borrowed(TIMEOUT_FEEDBACK_NAME)) { diff --git a/crates/libafl/src/stages/concolic.rs b/crates/libafl/src/stages/concolic.rs index 1805233acbf..e38d80ace12 100644 --- a/crates/libafl/src/stages/concolic.rs +++ b/crates/libafl/src/stages/concolic.rs @@ -394,9 +394,10 @@ where start_timer!(state); mark_feature_time!(state, PerfFeature::GetInputFromCorpus); } - let testcase = state.current_testcase()?.clone(); + let testcase = state.current_testcase()?; + let md = testcase.testcase_metadata(); - let mutations = testcase.metadata::().ok().map(|meta| { + let mutations = md.metadata::().ok().map(|meta| { start_timer!(state); let mutations = { generate_mutations(meta.iter_messages()) }; mark_feature_time!(state, PerfFeature::Mutate); diff --git a/crates/libafl_libfuzzer/runtime/Cargo.toml.template b/crates/libafl_libfuzzer/runtime/Cargo.toml.template index 09f8ff2d52d..f201cdd3119 100644 --- a/crates/libafl_libfuzzer/runtime/Cargo.toml.template +++ b/crates/libafl_libfuzzer/runtime/Cargo.toml.template @@ -14,7 +14,7 @@ track_hit_feedbacks = [ ] tui_monitor = ["libafl/tui_monitor"] -[target.'cfg(not(windows))'.features] +[target.'cfg(not(target_os = "windows"))'.features] ## Enable the `fork` feature on non-windows platforms default = ["fork", "tui_monitor"] diff --git a/crates/libafl_libfuzzer/runtime/src/merge.rs b/crates/libafl_libfuzzer/runtime/src/merge.rs index aacf7ac7048..4808bc5b03e 100644 --- a/crates/libafl_libfuzzer/runtime/src/merge.rs +++ b/crates/libafl_libfuzzer/runtime/src/merge.rs @@ -204,10 +204,8 @@ pub fn merge( } for id in fuzzer.scheduler().removable() { - let testcase = state.corpus_mut().remove(id)?; - fuzzer - .scheduler_mut() - .on_remove(&mut state, id, &Some(testcase))?; + state.corpus_mut().disable(id)?; + fuzzer.scheduler_mut().on_remove(&mut state, id)?; } for id in fuzzer.scheduler().current().clone() { @@ -228,10 +226,8 @@ pub fn merge( new_file_path.push(base); if new_file_path.exists() { drop(testcase); - let testcase = state.corpus_mut().remove(id)?; - fuzzer - .scheduler_mut() - .on_remove(&mut state, id, &Some(testcase))?; + state.corpus_mut().remove(id)?; + fuzzer.scheduler_mut().on_remove(&mut state, id)?; } else { // False-positive: file_path is used just below rename(&file_path, &new_file_path)?; diff --git a/crates/libafl_libfuzzer/runtime/src/schedulers.rs b/crates/libafl_libfuzzer/runtime/src/schedulers.rs index ad8543eff17..2a09928a9bf 100644 --- a/crates/libafl_libfuzzer/runtime/src/schedulers.rs +++ b/crates/libafl_libfuzzer/runtime/src/schedulers.rs @@ -22,12 +22,7 @@ where I: Input, S: HasCorpus, { - fn on_remove( - &mut self, - _state: &mut S, - id: CorpusId, - _testcase: &Option>, - ) -> Result<(), Error> { + fn on_remove(&mut self, _state: &mut S, id: CorpusId) -> Result<(), Error> { self.all.remove(&id); Ok(()) } diff --git a/fuzzers/forkserver/libafl-fuzz/src/scheduler.rs b/fuzzers/forkserver/libafl-fuzz/src/scheduler.rs index bb84de69372..5933973f89e 100644 --- a/fuzzers/forkserver/libafl-fuzz/src/scheduler.rs +++ b/fuzzers/forkserver/libafl-fuzz/src/scheduler.rs @@ -20,15 +20,10 @@ where W: Scheduler + RemovableScheduler, S: HasTestcase, { - fn on_remove( - &mut self, - state: &mut S, - id: CorpusId, - testcase: &Option>, - ) -> Result<(), Error> { + fn on_remove(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { match self { - Self::Queue(queue, _) => queue.on_remove(state, id, testcase), - Self::Weighted(weighted, _) => weighted.on_remove(state, id, testcase), + Self::Queue(queue, _) => queue.on_remove(state, id), + Self::Weighted(weighted, _) => weighted.on_remove(state, id), } } From f8f07b8e6612cccaa35f3cf96bab01c76d4bba49 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 19 Sep 2025 15:55:13 +0200 Subject: [PATCH 11/12] first clippy pass --- .../common/nautilus/grammartec/chunkstore.rs | 9 ++-- .../src/common/nautilus/grammartec/mutator.rs | 2 +- crates/libafl/src/corpus/cache.rs | 26 +++++----- crates/libafl/src/corpus/collection.rs | 19 +++++--- crates/libafl/src/corpus/combined.rs | 3 +- crates/libafl/src/corpus/minimizer.rs | 4 +- crates/libafl/src/corpus/single.rs | 3 +- crates/libafl/src/corpus/store/inmemory.rs | 12 ++--- crates/libafl/src/corpus/store/maps.rs | 5 +- crates/libafl/src/corpus/store/mod.rs | 2 +- crates/libafl/src/corpus/store/ondisk.rs | 48 ++++++++++--------- crates/libafl/src/corpus/testcase.rs | 45 ++++++++++------- crates/libafl/src/feedbacks/nautilus.rs | 13 +++-- crates/libafl/src/lib.rs | 5 +- crates/libafl/src/mutators/scheduled.rs | 9 ++-- crates/libafl/src/mutators/token_mutations.rs | 9 +--- crates/libafl/src/mutators/unicode/mod.rs | 4 +- .../src/schedulers/probabilistic_sampling.rs | 10 ++-- crates/libafl/src/schedulers/queue.rs | 27 ++++++----- crates/libafl/src/stages/afl_stats.rs | 2 - crates/libafl/src/stages/dump.rs | 8 ++-- crates/libafl/src/stages/generalization.rs | 2 +- crates/libafl/src/stages/logics.rs | 14 +++--- crates/libafl/src/stages/mod.rs | 4 +- crates/libafl/src/stages/tmin.rs | 2 +- crates/libafl/src/stages/unicode.rs | 13 +++-- crates/libafl_bolts/src/tuples.rs | 12 +---- 27 files changed, 161 insertions(+), 151 deletions(-) diff --git a/crates/libafl/src/common/nautilus/grammartec/chunkstore.rs b/crates/libafl/src/common/nautilus/grammartec/chunkstore.rs index f33a5834f07..07f6eb42383 100644 --- a/crates/libafl/src/common/nautilus/grammartec/chunkstore.rs +++ b/crates/libafl/src/common/nautilus/grammartec/chunkstore.rs @@ -49,7 +49,7 @@ impl ChunkStore { } } - pub fn add_tree(&mut self, tree: Tree, ctx: &Context) { + pub fn add_tree(&mut self, tree: &Tree, ctx: &Context) { let mut buffer = vec![]; let id = self.trees.len(); let mut contains_new_chunk = false; @@ -76,8 +76,9 @@ impl ChunkStore { contains_new_chunk = true; } } + if contains_new_chunk { - self.trees.push(tree); + self.trees.push(tree.clone()); } } @@ -129,7 +130,7 @@ mod tests { let tree = ctx.generate_tree_from_rule(&mut rand, r1, random_size); fs::create_dir_all("/tmp/outputs/chunks").expect("40234068"); let mut cks = ChunkStore::new("/tmp/".to_string()); - cks.add_tree(tree, &ctx); + cks.add_tree(&tree, &ctx); // assert!(cks.seen_outputs.contains("a b c".as_bytes())); // assert!(cks.seen_outputs.contains("b c".as_bytes())); // assert!(cks.seen_outputs.contains("c".as_bytes())); @@ -139,7 +140,7 @@ mod tests { let random_size = ctx.get_random_len_for_ruleid(&r2); let tree = ctx.generate_tree_from_rule(&mut rand, r2, random_size); - cks.add_tree(tree, &ctx); + cks.add_tree(&tree, &ctx); // assert_eq!(cks.seen_outputs.len(), 3); // assert_eq!(cks.nts_to_chunks[&ctx.nt_id("B")].len(), 1); let (tree_id, node_id) = cks.nts_to_chunks[&ctx.nt_id("B")][0]; diff --git a/crates/libafl/src/common/nautilus/grammartec/mutator.rs b/crates/libafl/src/common/nautilus/grammartec/mutator.rs index d59071f0bd0..4634242faaf 100644 --- a/crates/libafl/src/common/nautilus/grammartec/mutator.rs +++ b/crates/libafl/src/common/nautilus/grammartec/mutator.rs @@ -538,7 +538,7 @@ mod tests { let _ = ctx.add_rule("A", b"a"); ctx.initialize(101); let tree = ctx.generate_tree_from_rule(&mut rand, r3, 100); - cks.add_tree(tree, &ctx); + cks.add_tree(&tree, &ctx); for _ in 0..100 { let tree = ctx.generate_tree_from_rule(&mut rand, r1, 100); let mut mutator = Mutator::new(&ctx); diff --git a/crates/libafl/src/corpus/cache.rs b/crates/libafl/src/corpus/cache.rs index 56c46d0d9b5..05052c614a2 100644 --- a/crates/libafl/src/corpus/cache.rs +++ b/crates/libafl/src/corpus/cache.rs @@ -5,8 +5,8 @@ //! - a **cache store** holding on the testcases with quick access. //! - a **backing store** with more expensive access, used when the testcase cannot be found in the cache store. -use alloc::{rc::Rc, vec::Vec}; -use std::{cell::RefCell, collections::VecDeque, marker::PhantomData}; +use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; +use core::{cell::RefCell, marker::PhantomData}; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; @@ -138,11 +138,12 @@ where } } -pub type StdIdentityCacheTestcaseMetadataCell, FS: Store> = Rc< +/// The standard cell for testcase metadata in an identity cache. +pub type StdIdentityCacheTestcaseMetadataCell = Rc< CacheTestcaseMetadataCell< - CS::TestcaseMetadataCell, + >::TestcaseMetadataCell, WritebackOnFlushPolicy, - FS::TestcaseMetadataCell, + >::TestcaseMetadataCell, >, >; @@ -196,11 +197,11 @@ where where Self: 'a; - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_> { self.cache_cell.testcase_metadata() } - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_> { *self.write_access.borrow_mut() = true; self.cache_cell.testcase_metadata_mut() } @@ -317,7 +318,7 @@ where } fn written(&self, id: CorpusId) { - self.cache_policy.dirty(id) + self.cache_policy.dirty(id); } } @@ -327,6 +328,7 @@ where FS: Store, I: Clone, { + #[expect(clippy::too_many_arguments)] fn get_inner( &mut self, id: CorpusId, @@ -354,7 +356,7 @@ where debug_assert!(self.cached_ids.len() < self.cache_max_len); // tescase is not cached, fetch it from fallback - let fb_tc = fallback_get_fn(&fallback_store, id)?.cloned(); + let fb_tc = fallback_get_fn(fallback_store, id)?.cloned(); cache_add_fn(cache_store, id, fb_tc)?; self.cached_ids.push_front(id); @@ -397,9 +399,9 @@ where let (input, md) = testcase.into_inner(); cache_store.add_shared::(corpus_id, input, md.into_testcase_metadata()) }, - |cache_store, corpus_id| cache_store.get(corpus_id), - |cache_store, corpus_id| cache_store.remove(corpus_id), - |fallback_store, corpus_id| fallback_store.get_from::(corpus_id), + Store::get, + RemovableStore::remove, + Store::get_from::, ) } diff --git a/crates/libafl/src/corpus/collection.rs b/crates/libafl/src/corpus/collection.rs index 54db96e97e5..e3d3e041187 100644 --- a/crates/libafl/src/corpus/collection.rs +++ b/crates/libafl/src/corpus/collection.rs @@ -1,5 +1,8 @@ +//! A collection of various [`Corpus`]. + use alloc::{rc::Rc, string::String}; -use std::{cell::RefCell, path::PathBuf}; +use core::cell::RefCell; +use std::path::PathBuf; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; @@ -10,7 +13,7 @@ use crate::{ SingleCorpus, Testcase, TestcaseMetadata, cache::StdIdentityCacheTestcaseMetadataCell, maps::{self, InMemoryCorpusMap, InMemoryTestcaseMap}, - store::{OnDiskMetadataFormat, Store, ondisk::OnDiskTestcaseCell}, + store::{OnDiskMetadataFormat, Store}, }, inputs::Input, }; @@ -27,9 +30,9 @@ type InnerStdInMemoryStore = InMemoryStore, StdInMemoryTestcaseMetadataCell>; type InnerInMemoryCorpus = SingleCorpus>; -type StdOnDiskTestcaseMetadataCell = Rc>; -type StdOnDiskTestcase = Testcase>; -type InnerStdOnDiskCorpusMap = StdInMemoryMap>; +//type StdOnDiskTestcaseMetadataCell = Rc>; +//type StdOnDiskTestcase = Testcase>; +//type InnerStdOnDiskCorpusMap = StdInMemoryMap>; type InnerStdOnDiskStore = OnDiskStore>; #[cfg(feature = "std")] type InnerOnDiskCorpus = SingleCorpus>; @@ -105,7 +108,7 @@ where } fn add(&mut self, id: CorpusId, testcase: Testcase>>) { - self.0.add(id, testcase) + self.0.add(id, testcase); } fn get(&self, id: CorpusId) -> Option<&Testcase>>> { @@ -295,6 +298,8 @@ impl Default for InMemoryCorpus { } impl InMemoryCorpus { + /// Create a new [`InMemoryCorpus`]. + #[must_use] pub fn new() -> Self { Self::default() } @@ -379,12 +384,14 @@ impl OnDiskCorpus where I: Input, { + /// Create a new [`OnDiskCorpus`] pub fn new(root: PathBuf) -> Result { Ok(OnDiskCorpus(InnerOnDiskCorpus::new( InnerStdOnDiskStore::new(root)?, ))) } + /// Create a new [`OnDiskCorpus`] with a specific [`OnDiskMetadataFormat`] pub fn new_with_format(root: PathBuf, md_format: OnDiskMetadataFormat) -> Result { Ok(OnDiskCorpus(InnerOnDiskCorpus::new( InnerStdOnDiskStore::new_with_format(root, md_format)?, diff --git a/crates/libafl/src/corpus/combined.rs b/crates/libafl/src/corpus/combined.rs index 5a7b823e67e..39895ecb694 100644 --- a/crates/libafl/src/corpus/combined.rs +++ b/crates/libafl/src/corpus/combined.rs @@ -1,8 +1,7 @@ //! A cached corpus, using a given [`Cache`] policy and two [`Store`]s. -use alloc::rc::Rc; +use alloc::{rc::Rc, vec::Vec}; use core::{cell::RefCell, marker::PhantomData}; -use std::vec::Vec; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; diff --git a/crates/libafl/src/corpus/minimizer.rs b/crates/libafl/src/corpus/minimizer.rs index 0c35d4055fd..362ca0d1588 100644 --- a/crates/libafl/src/corpus/minimizer.rs +++ b/crates/libafl/src/corpus/minimizer.rs @@ -99,7 +99,7 @@ where let (weight, executions) = { { let tc = state.corpus().get(id)?; - let md = tc.testcase_metadata(); + let mut md = tc.testcase_metadata_mut(); if md.scheduled_count() == 0 { // Execute the input; we cannot rely on the metadata already being present. let input = tc.input(); @@ -213,7 +213,7 @@ where continue; } - let removed = state.corpus().disable(id)?; + state.corpus_mut().disable(id)?; // scheduler needs to know we've removed the input, or it will continue to try // to use now-missing inputs diff --git a/crates/libafl/src/corpus/single.rs b/crates/libafl/src/corpus/single.rs index 0ed72c7a0fd..a86c5e21306 100644 --- a/crates/libafl/src/corpus/single.rs +++ b/crates/libafl/src/corpus/single.rs @@ -2,9 +2,8 @@ //! //! A [`SingleCorpus`] owns a single store, in which every testcase is added. -use alloc::rc::Rc; +use alloc::{rc::Rc, vec::Vec}; use core::marker::PhantomData; -use std::vec::Vec; use libafl_bolts::Error; use serde::{Deserialize, Serialize}; diff --git a/crates/libafl/src/corpus/store/inmemory.rs b/crates/libafl/src/corpus/store/inmemory.rs index 7d4a2be882e..210db6cebca 100644 --- a/crates/libafl/src/corpus/store/inmemory.rs +++ b/crates/libafl/src/corpus/store/inmemory.rs @@ -63,13 +63,13 @@ where md: TestcaseMetadata, ) -> Result<(), Error> { if ENABLED { - Ok(self - .enabled_map - .add(id, Testcase::new(input, TMC::instantiate(md)))) + self.enabled_map + .add(id, Testcase::new(input, TMC::instantiate(md))); + Ok(()) } else { - Ok(self - .disabled_map - .add(id, Testcase::new(input, TMC::instantiate(md)))) + self.disabled_map + .add(id, Testcase::new(input, TMC::instantiate(md))); + Ok(()) } } diff --git a/crates/libafl/src/corpus/store/maps.rs b/crates/libafl/src/corpus/store/maps.rs index 354b44a0cde..65fe6e80f52 100644 --- a/crates/libafl/src/corpus/store/maps.rs +++ b/crates/libafl/src/corpus/store/maps.rs @@ -1,6 +1,6 @@ //! Multiple map implementations for the in-memory store. -use std::{collections::BTreeMap, vec::Vec}; +use alloc::{collections::BTreeMap, vec::Vec}; use num_traits::Zero; use serde::{Deserialize, Serialize}; @@ -17,7 +17,7 @@ pub trait InMemoryCorpusMap { self.count().is_zero() } - /// Store the testcase associated to corpus_id. + /// Store the testcase associated to `corpus_id`. fn add(&mut self, id: CorpusId, testcase: T); /// Get by id; considers only enabled testcases @@ -45,6 +45,7 @@ pub trait InMemoryCorpusMap { fn nth(&self, nth: usize) -> CorpusId; } +/// A corpus map for testcases. pub trait InMemoryTestcaseMap: InMemoryCorpusMap { /// Replace the metadata of a given testcase fn replace_metadata( diff --git a/crates/libafl/src/corpus/store/mod.rs b/crates/libafl/src/corpus/store/mod.rs index acaf81431fa..ac2e43cafdc 100644 --- a/crates/libafl/src/corpus/store/mod.rs +++ b/crates/libafl/src/corpus/store/mod.rs @@ -37,7 +37,7 @@ pub trait Store { self.count() == 0 } - /// Store the testcase associated to corpus_id to the set. + /// Store the testcase associated to `corpus_id` to the set. fn add_shared( &mut self, id: CorpusId, diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index 5c9af6ef67b..735f7f5cc65 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -1,15 +1,14 @@ //! An on-disk store -use alloc::rc::Rc; -use core::marker::PhantomData; -use std::{ +use alloc::{rc::Rc, string::String, vec::Vec}; +use core::{ cell::{Ref, RefCell, RefMut}, - fs, - fs::OpenOptions, + marker::PhantomData, +}; +use std::{ + fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, - string::String, - vec::Vec, }; use libafl_bolts::{Error, compress::GzipCompressor}; @@ -109,6 +108,7 @@ impl OnDiskMetadataFormat { impl OnDiskTestcaseCell { /// Get a new [`OnDiskTestcaseCell`]. + #[must_use] pub fn new(mgr: Rc>, id: String, testcase_md: TestcaseMetadata) -> Self { Self { mgr, @@ -129,11 +129,11 @@ impl IsTestcaseMetadataCell for OnDiskTestcaseCell { where I: 'a; - fn testcase_metadata<'a>(&'a self) -> Ref<'a, TestcaseMetadata> { + fn testcase_metadata(&self) -> Ref<'_, TestcaseMetadata> { self.testcase_md.borrow() } - fn testcase_metadata_mut<'a>(&'a self) -> RefMut<'a, TestcaseMetadata> { + fn testcase_metadata_mut(&self) -> RefMut<'_, TestcaseMetadata> { *self.modified.borrow_mut() = true; self.testcase_md.borrow_mut() } @@ -147,8 +147,7 @@ impl IsTestcaseMetadataCell for OnDiskTestcaseCell { } fn flush(&self) -> Result<(), Error> { - self.mgr - .save_metadata(&self.id, &*self.testcase_md.borrow()) + self.mgr.save_metadata(&self.id, &self.testcase_md.borrow()) } } @@ -159,10 +158,12 @@ impl Drop for OnDiskTestcaseCell { } impl DiskMgr { + /// Create a new [`DiskMgr`] pub fn new(root_dir: PathBuf) -> Result { Self::new_with_format(root_dir, OnDiskMetadataFormat::default()) } + /// Create a new [`DiskMgr`], with a given [`OnDiskMetadataFormat`] pub fn new_with_format( root_dir: PathBuf, md_format: OnDiskMetadataFormat, @@ -175,11 +176,11 @@ impl DiskMgr { } fn testcase_path(&self, testcase_id: &String) -> PathBuf { - self.root_dir.join(&testcase_id) + self.root_dir.join(testcase_id) } fn testcase_md_path(&self, testcase_id: &String) -> PathBuf { - self.root_dir.join(format!(".{}.metadata", testcase_id)) + self.root_dir.join(format!(".{testcase_id}.metadata")) } /// The file is created if it does not exist, or reused if it's already there @@ -189,9 +190,10 @@ impl DiskMgr { let mut testcase_md_f = OpenOptions::new() .write(true) .create(true) + .truncate(true) .open(&testcase_md_path)?; - let testcase_md_ser = self.md_format.to_vec(&md)?; + let testcase_md_ser = self.md_format.to_vec(md)?; testcase_md_f.write_all(&testcase_md_ser)?; Ok(()) @@ -202,15 +204,15 @@ impl DiskMgr where I: Input, { - fn save_input(&self, input: Rc) -> Result { - let testcase_id = Testcase::>::compute_id(input.as_ref()); + fn save_input(&self, input: &I) -> Result { + let testcase_id = Testcase::>::compute_id(input); let testcase_path = self.testcase_path(&testcase_id); - input.as_ref().to_file(testcase_path.as_path())?; + input.to_file(testcase_path.as_path())?; Ok(testcase_id) } - fn save_testcase(&self, input: Rc, md: &TestcaseMetadata) -> Result { + fn save_testcase(&self, input: &I, md: &TestcaseMetadata) -> Result { let id = self.save_input(input)?; self.save_metadata(&id, md)?; Ok(id) @@ -222,8 +224,8 @@ where self: &Rc, testcase_id: &String, ) -> Result>, Error> { - let testcase_path = self.testcase_path(testcase_id); - let testcase_md_path = self.testcase_md_path(testcase_id); + let testcase_path = self.as_ref().testcase_path(testcase_id); + let testcase_md_path = self.as_ref().testcase_md_path(testcase_id); let ser_fmt = self.md_format.clone(); // let _lockfile = TestcaseLockfile::new(self, testcase_id)?; @@ -242,10 +244,12 @@ impl OnDiskStore where M: Default, { + /// Create a new [`OnDiskStore`] pub fn new(root: PathBuf) -> Result { Self::new_with_format(root, OnDiskMetadataFormat::default()) } + /// Create a new [`OnDiskStore`], with a specified [`OnDiskMetadataFormat`]. pub fn new_with_format(root: PathBuf, md_format: OnDiskMetadataFormat) -> Result { let disk_mgr = Rc::new(DiskMgr::new_with_format(root, md_format)?); @@ -288,7 +292,7 @@ where input: Rc, md: TestcaseMetadata, ) -> Result<(), Error> { - let testcase_id = self.disk_mgr.save_testcase(input, &md)?; + let testcase_id = self.disk_mgr.save_testcase(input.as_ref(), &md)?; if ENABLED { self.enabled_map.add(id, testcase_id); @@ -314,7 +318,7 @@ where .ok_or_else(|| Error::key_not_found(format!("Index {id} not found")))? }; - self.disk_mgr.load_testcase(&tc_id) + self.disk_mgr.load_testcase(tc_id) } fn disable(&mut self, id: CorpusId) -> Result<(), Error> { diff --git a/crates/libafl/src/corpus/testcase.rs b/crates/libafl/src/corpus/testcase.rs index 76a10809959..7927ba40365 100644 --- a/crates/libafl/src/corpus/testcase.rs +++ b/crates/libafl/src/corpus/testcase.rs @@ -5,15 +5,13 @@ use alloc::{borrow::Cow, vec::Vec}; use alloc::{rc::Rc, string::String}; use core::{ + cell::{Ref, RefCell, RefMut}, fmt::{Debug, Formatter}, hash::Hasher, + marker::PhantomData, ops::{Deref, DerefMut}, time::Duration, }; -use std::{ - cell::{Ref, RefCell, RefMut}, - marker::PhantomData, -}; use libafl_bolts::{ HasLen, hasher_std, @@ -48,10 +46,10 @@ pub trait IsTestcaseMetadataCell { Self: 'a; /// Get a reference to the testcase metadata. - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a>; + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_>; /// Get a mutable reference to the testcase metadata. - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a>; + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_>; /// Consume the cell, and get the inner testcase metadata. fn into_testcase_metadata(self) -> TestcaseMetadata; @@ -78,7 +76,7 @@ pub struct NopTestcaseMetadataRef<'a>(PhantomData<&'a ()>); #[derive(Default, Clone, Copy, Debug)] pub struct NopTestcaseMetadataCell; -impl<'a> Deref for NopTestcaseMetadataRef<'a> { +impl Deref for NopTestcaseMetadataRef<'_> { type Target = TestcaseMetadata; fn deref(&self) -> &Self::Target { @@ -86,7 +84,7 @@ impl<'a> Deref for NopTestcaseMetadataRef<'a> { } } -impl<'a> DerefMut for NopTestcaseMetadataRef<'a> { +impl DerefMut for NopTestcaseMetadataRef<'_> { fn deref_mut(&mut self) -> &mut Self::Target { panic!("Invalid testcase metadata ref mut") } @@ -96,11 +94,11 @@ impl IsTestcaseMetadataCell for NopTestcaseMetadataCell { type TestcaseMetadataRef<'a> = NopTestcaseMetadataRef<'a>; type TestcaseMetadataRefMut<'a> = NopTestcaseMetadataRef<'a>; - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_> { NopTestcaseMetadataRef::default() } - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_> { NopTestcaseMetadataRef::default() } @@ -117,11 +115,11 @@ impl IsTestcaseMetadataCell for RefCell { type TestcaseMetadataRef<'a> = Ref<'a, TestcaseMetadata>; type TestcaseMetadataRefMut<'a> = RefMut<'a, TestcaseMetadata>; - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_> { self.borrow() } - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_> { self.borrow_mut() } @@ -157,11 +155,11 @@ where // fn new(md: TestcaseMetadata) -> Self { // Rc::new(T::new(md)) // } - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_> { self.deref().testcase_metadata() } - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_> { self.deref().testcase_metadata_mut() } @@ -196,11 +194,11 @@ where where Self: 'a; - fn testcase_metadata<'a>(&'a self) -> Self::TestcaseMetadataRef<'a> { + fn testcase_metadata(&self) -> Self::TestcaseMetadataRef<'_> { self.metadata.testcase_metadata() } - fn testcase_metadata_mut<'a>(&'a self) -> Self::TestcaseMetadataRefMut<'a> { + fn testcase_metadata_mut(&self) -> Self::TestcaseMetadataRefMut<'_> { self.metadata.testcase_metadata_mut() } @@ -500,30 +498,35 @@ where impl TestcaseMetadata { /// Get the executions #[inline] + #[must_use] pub fn executions(&self) -> u64 { self.executions } /// Get the execution time of the testcase #[inline] + #[must_use] pub fn exec_time(&self) -> &Option { &self.exec_time } /// Get the `scheduled_count` #[inline] + #[must_use] pub fn scheduled_count(&self) -> usize { self.scheduled_count } /// Get `disabled` #[inline] + #[must_use] pub fn disabled(&mut self) -> bool { self.disabled } /// Get the hit feedbacks #[inline] + #[must_use] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_feedbacks(&self) -> &Vec> { &self.hit_feedbacks @@ -531,6 +534,7 @@ impl TestcaseMetadata { /// Get the hit objectives #[inline] + #[must_use] #[cfg(feature = "track_hit_feedbacks")] pub fn hit_objectives(&self) -> &Vec> { &self.hit_objectives @@ -543,12 +547,14 @@ impl TestcaseMetadata { } /// Gets how many objectives were found by mutating this testcase + #[must_use] pub fn objectives_found(&self) -> usize { self.objectives_found } /// Get the executions (mutable) #[inline] + #[must_use] pub fn executions_mut(&mut self) -> &mut u64 { &mut self.executions } @@ -560,6 +566,7 @@ impl TestcaseMetadata { } /// Get a mutable reference to the execution time + #[must_use] pub fn exec_time_mut(&mut self) -> &mut Option { &mut self.exec_time } @@ -583,15 +590,17 @@ impl TestcaseMetadata { } /// Get the hit feedbacks (mutable) - #[inline] #[cfg(feature = "track_hit_feedbacks")] + #[inline] + #[must_use] pub fn hit_feedbacks_mut(&mut self) -> &mut Vec> { &mut self.hit_feedbacks } /// Get the hit objectives (mutable) - #[inline] #[cfg(feature = "track_hit_feedbacks")] + #[inline] + #[must_use] pub fn hit_objectives_mut(&mut self) -> &mut Vec> { &mut self.hit_objectives } diff --git a/crates/libafl/src/feedbacks/nautilus.rs b/crates/libafl/src/feedbacks/nautilus.rs index ac760215000..877297c08ea 100644 --- a/crates/libafl/src/feedbacks/nautilus.rs +++ b/crates/libafl/src/feedbacks/nautilus.rs @@ -65,18 +65,15 @@ impl<'a> NautilusFeedback<'a> { &mut self, state: &mut S, input: &NautilusInput, - md: &mut TestcaseMetadata, - ) -> Result<(), Error> - where + _md: &mut TestcaseMetadata, + ) where S: HasCorpus + HasMetadataMut, { let meta = state .metadata_map_mut() .get_mut::() .expect("NautilusChunksMetadata not in the state"); - meta.cks.add_tree(input.tree, self.ctx); - - Ok(()) + meta.cks.add_tree(&input.tree, self.ctx); } } @@ -112,7 +109,9 @@ where input: &NautilusInput, md: &mut TestcaseMetadata, ) -> Result<(), Error> { - self.append_nautilus_metadata_to_state(state, input, md) + self.append_nautilus_metadata_to_state(state, input, md); + + Ok(()) } #[cfg(feature = "track_hit_feedbacks")] diff --git a/crates/libafl/src/lib.rs b/crates/libafl/src/lib.rs index d3389e8cef3..25cf622d81a 100644 --- a/crates/libafl/src/lib.rs +++ b/crates/libafl/src/lib.rs @@ -113,7 +113,7 @@ mod tests { use crate::stages::ExecutionCountRestartHelperMetadata; use crate::{ StdFuzzer, - corpus::{Corpus, InMemoryCorpus, Testcase}, + corpus::{Corpus, InMemoryCorpus}, events::NopEventManager, executors::{ExitKind, InProcessExecutor}, feedbacks::ConstFeedback, @@ -138,8 +138,7 @@ mod tests { let rand = StdRand::with_seed(0); let mut corpus = InMemoryCorpus::::new(); - let testcase = Testcase::new(vec![0; 4].into()); - corpus.add(testcase).unwrap(); + corpus.add(vec![0; 4].into()).unwrap(); let mut feedback = ConstFeedback::new(false); let mut objective = ConstFeedback::new(false); diff --git a/crates/libafl/src/mutators/scheduled.rs b/crates/libafl/src/mutators/scheduled.rs index d4f711bc498..fe8d67aef6f 100644 --- a/crates/libafl/src/mutators/scheduled.rs +++ b/crates/libafl/src/mutators/scheduled.rs @@ -411,7 +411,7 @@ mod tests { corpus.add(vec![b'a', b'b', b'c'].into()).unwrap(); corpus.add(vec![b'd', b'e', b'f'].into()).unwrap(); - let mut input = corpus.get(corpus.first().unwrap()).unwrap().input().clone(); + let input = corpus.get(corpus.first().unwrap()).unwrap().input().clone(); let mut feedback = ConstFeedback::new(false); let mut objective = ConstFeedback::new(false); @@ -426,12 +426,13 @@ mod tests { .unwrap(); let mut splice = SpliceMutator::new(); - splice.mutate(&mut state, &mut input).unwrap(); + let mut spliced_input = input.as_ref().clone(); + splice.mutate(&mut state, &mut spliced_input).unwrap(); - log::trace!("{:?}", input.mutator_bytes()); + log::trace!("{:?}", spliced_input.mutator_bytes()); // The pre-seeded rand should have spliced at position 2. - assert_eq!(input.mutator_bytes(), b"abf"); + assert_eq!(spliced_input.mutator_bytes(), b"abf"); } #[test] diff --git a/crates/libafl/src/mutators/token_mutations.rs b/crates/libafl/src/mutators/token_mutations.rs index a9c47a69ea6..500d509a206 100644 --- a/crates/libafl/src/mutators/token_mutations.rs +++ b/crates/libafl/src/mutators/token_mutations.rs @@ -1977,19 +1977,14 @@ impl AflppRedQueen { tokens.add_token(&v); } } -#[derive(Debug, Copy, Clone)] +#[derive(Default, Debug, Copy, Clone)] enum TextType { + #[default] None, Ascii(usize), UTF8(usize), } -impl Default for TextType { - fn default() -> Self { - Self::None - } -} - impl TextType { fn is_ascii_or_utf8(self) -> bool { match self { diff --git a/crates/libafl/src/mutators/unicode/mod.rs b/crates/libafl/src/mutators/unicode/mod.rs index 71828c0240f..32d14d504ce 100644 --- a/crates/libafl/src/mutators/unicode/mod.rs +++ b/crates/libafl/src/mutators/unicode/mod.rs @@ -11,7 +11,7 @@ use libafl_bolts::{Error, HasLen, Named, rands::Rand}; use crate::{ HasMetadata, HasMetadataMut, - corpus::{Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, Testcase}, + corpus::{CorpusId, HasTestcase, IsTestcaseMetadataCell, Testcase}, inputs::{BytesInput, HasMutatorBytes, ResizableMutator}, mutators::{MutationResult, Mutator, Tokens, rand_range}, nonzero, @@ -37,7 +37,7 @@ where fn try_transform_from( base: &Testcase, - state: &S, + _state: &S, ) -> Result { let input = base.input().as_ref().clone(); let metadata = base diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index 7b6ff979f33..c4196529615 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -186,7 +186,7 @@ mod tests { use crate::{ Error, - corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase}, + corpus::{Corpus, CorpusId, InMemoryCorpus}, feedbacks::ConstFeedback, inputs::bytes::BytesInput, schedulers::{ProbabilitySamplingScheduler, Scheduler, TestcaseScore}, @@ -229,11 +229,11 @@ mod tests { let mut objective = ConstFeedback::new(false); let mut corpus = InMemoryCorpus::new(); - let t1 = Testcase::with_filename(BytesInput::new(vec![0_u8; 4]), "1".into()); - let t2 = Testcase::with_filename(BytesInput::new(vec![1_u8; 4]), "2".into()); + // let t1 = Testcase::with_filename(, "1".into()); + // let t2 = Testcase::with_filename(, "2".into()); - let idx1 = corpus.add(t1).unwrap(); - let idx2 = corpus.add(t2).unwrap(); + let idx1 = corpus.add(BytesInput::new(vec![0_u8; 4])).unwrap(); + let idx2 = corpus.add(BytesInput::new(vec![1_u8; 4])).unwrap(); let mut state = StdState::new( rand, diff --git a/crates/libafl/src/schedulers/queue.rs b/crates/libafl/src/schedulers/queue.rs index 948175c5a16..e9ed6c5153a 100644 --- a/crates/libafl/src/schedulers/queue.rs +++ b/crates/libafl/src/schedulers/queue.rs @@ -105,7 +105,7 @@ mod tests { feedbacks::ConstFeedback, inputs::bytes::BytesInput, schedulers::{QueueScheduler, Scheduler}, - state::{HasCorpus, StdState}, + state::StdState, }; #[test] @@ -116,7 +116,7 @@ mod tests { let mut q = OnDiskCorpus::::new(PathBuf::from("target/.test/fancy/path")).unwrap(); // let t = Testcase::with_filename(), "fancyfile".into()); - q.add(BytesInput::new(vec![0_u8; 4])).unwrap(); + let added_id = q.add(BytesInput::new(vec![0_u8; 4])).unwrap(); let objective_q = OnDiskCorpus::::new(PathBuf::from("target/.test/fancy/objective/path")) @@ -129,17 +129,18 @@ mod tests { let next_id = >::next(&mut scheduler, &mut state).unwrap(); - let filename = state - .corpus() - .get(next_id) - .unwrap() - .borrow() - .filename() - .as_ref() - .unwrap() - .clone(); - - assert_eq!(filename, "fancyfile"); + // let filename = state + // .corpus() + // .get(next_id) + // .unwrap() + // .borrow() + // .filename() + // .as_ref() + // .unwrap() + // .clone(); + + // assert_eq!(filename, "fancyfile"); + assert_eq!(added_id, next_id); fs::remove_dir_all("target/.test/fancy/path").unwrap(); } diff --git a/crates/libafl/src/stages/afl_stats.rs b/crates/libafl/src/stages/afl_stats.rs index a0aecead89e..b098a6b7049 100644 --- a/crates/libafl/src/stages/afl_stats.rs +++ b/crates/libafl/src/stages/afl_stats.rs @@ -417,8 +417,6 @@ where self.write_plot_data(&plot_data)?; } - drop(testcase); - // We construct this simple json by hand to squeeze out some extra speed. let json = format!( "{{\ diff --git a/crates/libafl/src/stages/dump.rs b/crates/libafl/src/stages/dump.rs index 1b4ede11dbd..2be35e4c49e 100644 --- a/crates/libafl/src/stages/dump.rs +++ b/crates/libafl/src/stages/dump.rs @@ -205,11 +205,11 @@ where let input = testcase.input(); let md = testcase.testcase_metadata(); - let bytes = (self.to_bytes)(input.as_ref(), &*md, state); + let bytes = (self.to_bytes)(input.as_ref(), &md, state); let fname = self .corpus_dir - .join((self.generate_filename)(input.as_ref(), &*md, &i)); + .join((self.generate_filename)(input.as_ref(), &md, &i)); let mut f = File::create(fname)?; drop(f.write_all(&bytes)); @@ -222,11 +222,11 @@ where let input = testcase.input(); let md = testcase.testcase_metadata(); - let bytes = (self.to_bytes)(input.as_ref(), &*md, state); + let bytes = (self.to_bytes)(input.as_ref(), &md, state); let fname = self .solutions_dir - .join((self.generate_filename)(input.as_ref(), &*md, &i)); + .join((self.generate_filename)(input.as_ref(), &md, &i)); let mut f = File::create(fname)?; drop(f.write_all(&bytes)); diff --git a/crates/libafl/src/stages/generalization.rs b/crates/libafl/src/stages/generalization.rs index 6a00036604c..d09ef199da9 100644 --- a/crates/libafl/src/stages/generalization.rs +++ b/crates/libafl/src/stages/generalization.rs @@ -125,7 +125,7 @@ where .clone() .into_inner() .into_iter() - .map(|x| Some(x)) + .map(Some) .collect(); if payload.len() > MAX_GENERALIZED_LEN { diff --git a/crates/libafl/src/stages/logics.rs b/crates/libafl/src/stages/logics.rs index a5f0c22f638..e82811a3109 100644 --- a/crates/libafl/src/stages/logics.rs +++ b/crates/libafl/src/stages/logics.rs @@ -310,7 +310,7 @@ mod test { #[cfg(any(not(feature = "serdeany_autoreg"), miri))] use crate::stages::RetryCountRestartHelper; use crate::{ - HasMetadata, NopFuzzer, + HasMetadataMut, NopFuzzer, events::NopEventManager, executors::nop::NopExecutor, stages::{ @@ -340,7 +340,7 @@ mod test { #[expect(clippy::unnecessary_wraps)] fn should_restart(state: &mut S, _stage: &ST) -> Result where - S: HasMetadata, + S: HasMetadataMut, { // check if we're resuming let _metadata = state.metadata_or_insert_with(|| Self { count: 0 }); @@ -349,7 +349,7 @@ mod test { fn clear_progress(state: &mut S, _stage: &ST) -> Result<(), Error> where - S: HasMetadata, + S: HasMetadataMut, { if state.remove_metadata::().is_none() { return Err(Error::illegal_state( @@ -362,7 +362,7 @@ mod test { impl Stage for ResumeSucceededStage where - S: HasMetadata, + S: HasMetadataMut, { fn perform( &mut self, @@ -384,7 +384,7 @@ mod test { impl Restartable for ResumeSucceededStage where - S: HasMetadata, + S: HasMetadataMut, { fn should_restart(&mut self, state: &mut S) -> Result { TestProgress::should_restart(state, self) @@ -397,7 +397,7 @@ mod test { impl Stage for ResumeFailedStage where - S: HasMetadata, + S: HasMetadataMut, { fn perform( &mut self, @@ -422,7 +422,7 @@ mod test { impl Restartable for ResumeFailedStage where - S: HasMetadata, + S: HasMetadataMut, { fn should_restart(&mut self, state: &mut S) -> Result { TestProgress::should_restart(state, self) diff --git a/crates/libafl/src/stages/mod.rs b/crates/libafl/src/stages/mod.rs index fb613fe3e16..c6fb97a11aa 100644 --- a/crates/libafl/src/stages/mod.rs +++ b/crates/libafl/src/stages/mod.rs @@ -541,7 +541,7 @@ mod test { use libafl_bolts::{Error, Named}; use crate::{ - corpus::{Corpus, HasCurrentCorpusId, Testcase}, + corpus::{Corpus, HasCurrentCorpusId}, inputs::NopInput, stages::RetryCountRestartHelper, state::{HasCorpus, StdState}, @@ -569,7 +569,7 @@ mod test { let mut state = StdState::nop()?; let stage = StageWithOneTry; - let corpus_id = state.corpus_mut().add(Testcase::new(NopInput {}))?; + let corpus_id = state.corpus_mut().add(NopInput {})?; state.set_corpus_id(corpus_id)?; diff --git a/crates/libafl/src/stages/tmin.rs b/crates/libafl/src/stages/tmin.rs index 73606ec3655..0e410975b27 100644 --- a/crates/libafl/src/stages/tmin.rs +++ b/crates/libafl/src/stages/tmin.rs @@ -191,7 +191,7 @@ where } start_timer!(state); - let transformed = I::try_transform_from(&mut state.current_testcase()?, state)?; + let transformed = I::try_transform_from(&state.current_testcase()?, state)?; let mut base = state.current_input_cloned()?; // potential post operation if base is replaced by a shorter input let mut base_post = None; diff --git a/crates/libafl/src/stages/unicode.rs b/crates/libafl/src/stages/unicode.rs index a4aa9051107..41aca09e394 100644 --- a/crates/libafl/src/stages/unicode.rs +++ b/crates/libafl/src/stages/unicode.rs @@ -8,7 +8,8 @@ use libafl_bolts::{Error, impl_serdeany}; use serde::{Deserialize, Serialize}; use crate::{ - HasMetadata, + HasMetadataMut, + corpus::IsTestcaseMetadataCell, inputs::{BytesInput, HasTargetBytes}, stages::{Restartable, Stage}, state::{HasCorpus, HasCurrentTestcase}, @@ -93,16 +94,18 @@ impl UnicodeIdentificationStage { S: HasCurrentTestcase, I: HasTargetBytes, { - let mut tc = state.current_testcase_mut()?; + let tc = state.current_testcase()?; + let input = tc.input(); + if tc.has_metadata::() { return Ok(()); // skip recompute } - let input = tc.load_input(state.corpus())?; - let bytes = input.target_bytes(); let metadata = extract_metadata(&bytes); - tc.add_metadata(metadata); + + let mut md = tc.testcase_metadata_mut(); + md.add_metadata(metadata); Ok(()) } diff --git a/crates/libafl_bolts/src/tuples.rs b/crates/libafl_bolts/src/tuples.rs index 02b9170b18d..0a7cffa7f22 100644 --- a/crates/libafl_bolts/src/tuples.rs +++ b/crates/libafl_bolts/src/tuples.rs @@ -1299,11 +1299,7 @@ mod test { let (tuple, _handles) = get_tuple(); #[expect(clippy::let_unit_value)] let recovered = tuple.get_all(tuple_list!()); - #[expect(clippy::unit_cmp)] - // needs its own scope to make the clippy expect work - { - assert_eq!(recovered, ()); - } + assert_eq!(recovered, ()); } #[test] @@ -1346,11 +1342,7 @@ mod test { let mut tuple = get_tuple().0; #[expect(clippy::let_unit_value)] let recovered = tuple.get_all_mut(tuple_list!()); - #[expect(clippy::unit_cmp)] - // needs its own scope to make the clippy expect work - { - assert_eq!(recovered, tuple_list!()); - } + assert_eq!(recovered, tuple_list!()); } #[test] From ea6b9e1ea2a77f17d9edce60f790cd4436659fc5 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Fri, 19 Sep 2025 16:15:45 +0200 Subject: [PATCH 12/12] remove HasMetadataMut / HasNamedMetadataMut --- crates/libafl/src/common/mod.rs | 6 --- crates/libafl/src/corpus/store/ondisk.rs | 4 +- crates/libafl/src/corpus/testcase.rs | 4 +- crates/libafl/src/executors/inprocess/mod.rs | 2 +- .../libafl/src/feedbacks/capture_feedback.rs | 4 +- crates/libafl/src/feedbacks/concolic.rs | 2 +- crates/libafl/src/feedbacks/list.rs | 10 ++--- crates/libafl/src/feedbacks/map.rs | 8 ++-- crates/libafl/src/feedbacks/nautilus.rs | 6 +-- .../libafl/src/feedbacks/new_hash_feedback.rs | 8 ++-- crates/libafl/src/feedbacks/simd.rs | 8 ++-- crates/libafl/src/feedbacks/stdio.rs | 2 +- crates/libafl/src/feedbacks/transferred.rs | 6 +-- crates/libafl/src/feedbacks/value_bloom.rs | 6 +-- crates/libafl/src/mutators/gramatron.rs | 4 +- crates/libafl/src/mutators/mopt_mutator.rs | 12 +++--- crates/libafl/src/mutators/scheduled.rs | 2 +- crates/libafl/src/mutators/token_mutations.rs | 4 +- crates/libafl/src/mutators/tuneable.rs | 26 ++++++------ crates/libafl/src/mutators/unicode/mod.rs | 2 +- crates/libafl/src/observers/cmp.rs | 4 +- crates/libafl/src/schedulers/accounting.rs | 12 +++--- crates/libafl/src/schedulers/minimizer.rs | 8 ++-- crates/libafl/src/schedulers/mod.rs | 6 +-- crates/libafl/src/schedulers/powersched.rs | 6 +-- .../src/schedulers/probabilistic_sampling.rs | 8 ++-- crates/libafl/src/schedulers/queue.rs | 4 +- crates/libafl/src/schedulers/tuneable.rs | 10 ++--- crates/libafl/src/schedulers/weighted.rs | 10 ++--- crates/libafl/src/stages/calibrate.rs | 8 ++-- crates/libafl/src/stages/colorization.rs | 8 ++-- crates/libafl/src/stages/concolic.rs | 10 ++--- crates/libafl/src/stages/dump.rs | 14 ++----- crates/libafl/src/stages/generalization.rs | 4 +- crates/libafl/src/stages/logics.rs | 14 +++---- crates/libafl/src/stages/mod.rs | 14 +++---- crates/libafl/src/stages/mutational.rs | 6 +-- crates/libafl/src/stages/power.rs | 4 +- crates/libafl/src/stages/push/mod.rs | 5 +-- crates/libafl/src/stages/replay.rs | 6 +-- crates/libafl/src/stages/shadow.rs | 4 +- crates/libafl/src/stages/sync.rs | 8 ++-- crates/libafl/src/stages/time_tracker.rs | 4 +- crates/libafl/src/stages/tmin.rs | 4 +- crates/libafl/src/stages/tracing.rs | 4 +- crates/libafl/src/stages/tuneable.rs | 40 +++++++++---------- crates/libafl/src/stages/unicode.rs | 2 +- crates/libafl/src/stages/verify_timeouts.rs | 4 +- crates/libafl/src/state/mod.rs | 10 +---- crates/libafl_frida/src/asan/errors.rs | 2 +- crates/libafl_frida/src/lib.rs | 5 +-- 51 files changed, 176 insertions(+), 198 deletions(-) diff --git a/crates/libafl/src/common/mod.rs b/crates/libafl/src/common/mod.rs index dc050389968..5f067e9b693 100644 --- a/crates/libafl/src/common/mod.rs +++ b/crates/libafl/src/common/mod.rs @@ -38,10 +38,7 @@ pub trait HasMetadata { .get::() .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) } -} -/// Trait for elements offering mutable metadata -pub trait HasMetadataMut: HasMetadata { /// A map, storing all metadata (mutable) fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap; @@ -120,10 +117,7 @@ pub trait HasNamedMetadata { .get::(name) .ok_or_else(|| Error::key_not_found(format!("{} not found", type_name::()))) } -} -/// Trait for elements offering named metadata -pub trait HasNamedMetadataMut: HasNamedMetadata { /// A map, storing all metadata (mutable) fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap; diff --git a/crates/libafl/src/corpus/store/ondisk.rs b/crates/libafl/src/corpus/store/ondisk.rs index 735f7f5cc65..a212b88f576 100644 --- a/crates/libafl/src/corpus/store/ondisk.rs +++ b/crates/libafl/src/corpus/store/ondisk.rs @@ -11,7 +11,9 @@ use std::{ path::{Path, PathBuf}, }; -use libafl_bolts::{Error, compress::GzipCompressor}; +#[cfg(feature = "gzip")] +use libafl_bolts::compress::GzipCompressor; +use libafl_bolts::Error; use serde::{Deserialize, Serialize}; use super::{InMemoryCorpusMap, Store}; diff --git a/crates/libafl/src/corpus/testcase.rs b/crates/libafl/src/corpus/testcase.rs index 7927ba40365..15d00e3ecef 100644 --- a/crates/libafl/src/corpus/testcase.rs +++ b/crates/libafl/src/corpus/testcase.rs @@ -21,7 +21,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use typed_builder::TypedBuilder; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId}, inputs::Input, state::HasCorpus, @@ -323,9 +323,7 @@ impl HasMetadata for TestcaseMetadata { fn metadata_map(&self) -> &SerdeAnyMap { &self.metadata } -} -impl HasMetadataMut for TestcaseMetadata { fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { &mut self.metadata } diff --git a/crates/libafl/src/executors/inprocess/mod.rs b/crates/libafl/src/executors/inprocess/mod.rs index c6d37889f5d..75de7c8e104 100644 --- a/crates/libafl/src/executors/inprocess/mod.rs +++ b/crates/libafl/src/executors/inprocess/mod.rs @@ -16,7 +16,7 @@ use libafl_bolts::tuples::{RefIndexable, tuple_list}; use crate::{ Error, - common::HasMetadataMut, + common::HasMetadata, corpus::{Corpus, IsTestcaseMetadataCell, testcase::TestcaseMetadata}, events::{Event, EventFirer, EventRestarter, EventWithStats}, executors::{ diff --git a/crates/libafl/src/feedbacks/capture_feedback.rs b/crates/libafl/src/feedbacks/capture_feedback.rs index c706cfe7391..5e257f3cc10 100644 --- a/crates/libafl/src/feedbacks/capture_feedback.rs +++ b/crates/libafl/src/feedbacks/capture_feedback.rs @@ -6,7 +6,7 @@ use libafl_bolts::{Error, Named}; use serde::{Serialize, de::DeserializeOwned}; use crate::{ - HasMetadataMut, + HasMetadata, corpus::testcase::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, @@ -39,7 +39,7 @@ impl StateInitializer for CaptureTimeoutFeedback {} impl Feedback for CaptureTimeoutFeedback where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, I: Debug + Serialize + DeserializeOwned + Default + 'static + Clone, { #[inline] diff --git a/crates/libafl/src/feedbacks/concolic.rs b/crates/libafl/src/feedbacks/concolic.rs index ce3ad31e427..ff885e8d82b 100644 --- a/crates/libafl/src/feedbacks/concolic.rs +++ b/crates/libafl/src/feedbacks/concolic.rs @@ -13,7 +13,7 @@ use libafl_bolts::{ }; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::testcase::TestcaseMetadata, feedbacks::{Feedback, StateInitializer}, observers::concolic::ConcolicObserver, diff --git a/crates/libafl/src/feedbacks/list.rs b/crates/libafl/src/feedbacks/list.rs index 4f710dafea8..2ddc13cdc44 100644 --- a/crates/libafl/src/feedbacks/list.rs +++ b/crates/libafl/src/feedbacks/list.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ - HasNamedMetadataMut, + HasNamedMetadata, corpus::testcase::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, @@ -89,7 +89,7 @@ where ) -> bool where OT: MatchName, - S: HasNamedMetadataMut, + S: HasNamedMetadata, { let observer = observers.get(&self.observer_handle).unwrap(); // TODO register the list content in a testcase metadata @@ -116,7 +116,7 @@ where } } - fn append_list_observer_metadata(&mut self, state: &mut S) { + fn append_list_observer_metadata(&mut self, state: &mut S) { let history_set = state .named_metadata_map_mut() .get_mut::>(self.name()) @@ -133,7 +133,7 @@ where impl StateInitializer for ListFeedback where - S: HasNamedMetadataMut, + S: HasNamedMetadata, T: Debug + Eq + Hash + for<'a> Deserialize<'a> + Serialize + Default + Copy + 'static, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { @@ -145,7 +145,7 @@ where impl Feedback for ListFeedback where OT: MatchName, - S: HasNamedMetadataMut, + S: HasNamedMetadata, T: Debug + Eq + Hash diff --git a/crates/libafl/src/feedbacks/map.rs b/crates/libafl/src/feedbacks/map.rs index c073622e126..cae239bda55 100644 --- a/crates/libafl/src/feedbacks/map.rs +++ b/crates/libafl/src/feedbacks/map.rs @@ -26,7 +26,7 @@ use super::simd::SimdMapFeedback; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - Error, HasMetadataMut, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::testcase::TestcaseMetadata, events::{Event, EventFirer, EventWithStats}, executors::ExitKind, @@ -320,7 +320,7 @@ impl StateInitializer for MapFeedback where O: MapObserver, O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, - S: HasNamedMetadataMut, + S: HasNamedMetadata, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { // Initialize `MapFeedbackMetadata` with an empty vector and add it to the state. @@ -339,7 +339,7 @@ where O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, OT: MatchName, R: Reducer, - S: HasNamedMetadataMut + HasExecutions, + S: HasNamedMetadata + HasExecutions, { fn is_interesting( &mut self, @@ -532,7 +532,7 @@ where { fn is_interesting_default(&mut self, state: &mut S, observers: &OT) -> bool where - S: HasNamedMetadataMut, + S: HasNamedMetadata, OT: MatchName, { let mut interesting = false; diff --git a/crates/libafl/src/feedbacks/nautilus.rs b/crates/libafl/src/feedbacks/nautilus.rs index 877297c08ea..0c70a039fec 100644 --- a/crates/libafl/src/feedbacks/nautilus.rs +++ b/crates/libafl/src/feedbacks/nautilus.rs @@ -7,7 +7,7 @@ use libafl_bolts::Named; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, common::nautilus::grammartec::{chunkstore::ChunkStore, context::Context}, corpus::TestcaseMetadata, executors::ExitKind, @@ -67,7 +67,7 @@ impl<'a> NautilusFeedback<'a> { input: &NautilusInput, _md: &mut TestcaseMetadata, ) where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { let meta = state .metadata_map_mut() @@ -88,7 +88,7 @@ impl StateInitializer for NautilusFeedback<'_> {} impl Feedback for NautilusFeedback<'_> where - S: HasMetadataMut + HasCorpus, + S: HasMetadata + HasCorpus, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/new_hash_feedback.rs b/crates/libafl/src/feedbacks/new_hash_feedback.rs index 8e2afc3da54..e7a8f825968 100644 --- a/crates/libafl/src/feedbacks/new_hash_feedback.rs +++ b/crates/libafl/src/feedbacks/new_hash_feedback.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - Error, HasNamedMetadataMut, + Error, HasNamedMetadata, executors::ExitKind, feedbacks::{Feedback, HasObserverHandle, StateInitializer}, observers::ObserverWithHashField, @@ -103,7 +103,7 @@ impl NewHashFeedback where O: ObserverWithHashField + Named, { - fn has_interesting_backtrace_hash_observation( + fn has_interesting_backtrace_hash_observation( &mut self, state: &mut S, observers: &OT, @@ -137,7 +137,7 @@ where impl StateInitializer for NewHashFeedback where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { state.add_named_metadata_checked( @@ -152,7 +152,7 @@ impl Feedback for NewHashFeedback where O: ObserverWithHashField + Named, OT: MatchName, - S: HasNamedMetadataMut, + S: HasNamedMetadata, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/simd.rs b/crates/libafl/src/feedbacks/simd.rs index 7876e8d3497..dc64c17a483 100644 --- a/crates/libafl/src/feedbacks/simd.rs +++ b/crates/libafl/src/feedbacks/simd.rs @@ -20,7 +20,7 @@ use super::{DifferentIsNovel, Feedback, HasObserverHandle, MapFeedback, StateIni #[cfg(feature = "introspection")] use crate::state::HasClientPerfMonitor; use crate::{ - HasNamedMetadataMut, + HasNamedMetadata, corpus::testcase::TestcaseMetadata, events::EventFirer, executors::ExitKind, @@ -49,7 +49,7 @@ where { fn is_interesting_u8_simd_optimized(&mut self, state: &mut S, observers: &OT) -> bool where - S: HasNamedMetadataMut, + S: HasNamedMetadata, OT: MatchName, { // TODO Replace with match_name_type when stable @@ -155,7 +155,7 @@ impl StateInitializer for SimdMapFeedback where O: MapObserver, O::Entry: 'static + Default + Debug + DeserializeOwned + Serialize, - S: HasNamedMetadataMut, + S: HasNamedMetadata, R: SimdReducer, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { @@ -192,7 +192,7 @@ where EM: EventFirer, O: MapObserver + for<'a> AsSlice<'a, Entry = u8> + for<'a> AsIter<'a, Item = u8>, OT: MatchName, - S: HasNamedMetadataMut + HasExecutions, + S: HasNamedMetadata + HasExecutions, R: SimdReducer, V: VectorType + Copy + Eq, R::PrimitiveReducer: Reducer, diff --git a/crates/libafl/src/feedbacks/stdio.rs b/crates/libafl/src/feedbacks/stdio.rs index 4781d15e837..159f25706a4 100644 --- a/crates/libafl/src/feedbacks/stdio.rs +++ b/crates/libafl/src/feedbacks/stdio.rs @@ -9,7 +9,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::testcase::TestcaseMetadata, feedbacks::{Feedback, StateInitializer}, observers::{StdErrObserver, StdOutObserver}, diff --git a/crates/libafl/src/feedbacks/transferred.rs b/crates/libafl/src/feedbacks/transferred.rs index ed801306bed..76522ab0ba9 100644 --- a/crates/libafl/src/feedbacks/transferred.rs +++ b/crates/libafl/src/feedbacks/transferred.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "track_hit_feedbacks")] use crate::feedbacks::premature_last_result_err; use crate::{ - HasMetadataMut, + HasMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, state::HasCorpus, @@ -56,7 +56,7 @@ impl Named for TransferredFeedback { impl StateInitializer for TransferredFeedback where - S: HasMetadataMut, + S: HasMetadata, { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { state.add_metadata(TransferringMetadata { transferring: true }); @@ -66,7 +66,7 @@ where impl Feedback for TransferredFeedback where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { fn is_interesting( &mut self, diff --git a/crates/libafl/src/feedbacks/value_bloom.rs b/crates/libafl/src/feedbacks/value_bloom.rs index f267e843166..9c87cd24cc5 100644 --- a/crates/libafl/src/feedbacks/value_bloom.rs +++ b/crates/libafl/src/feedbacks/value_bloom.rs @@ -11,7 +11,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - HasNamedMetadataMut, + HasNamedMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, observers::{ObserversTuple, ValueObserver}, @@ -59,7 +59,7 @@ impl Named for ValueBloomFeedback<'_, T> { } } -impl StateInitializer for ValueBloomFeedback<'_, T> { +impl StateInitializer for ValueBloomFeedback<'_, T> { fn init_state(&mut self, state: &mut S) -> Result<(), Error> { let _ = state.named_metadata_or_insert_with::(&self.name, || { @@ -71,7 +71,7 @@ impl StateInitializer for ValueBloomFeedback<'_, T } } -impl, S: HasNamedMetadataMut, T: Hash> Feedback +impl, S: HasNamedMetadata, T: Hash> Feedback for ValueBloomFeedback<'_, T> { fn is_interesting( diff --git a/crates/libafl/src/mutators/gramatron.rs b/crates/libafl/src/mutators/gramatron.rs index ce28c544aaa..76d1f43d75a 100644 --- a/crates/libafl/src/mutators/gramatron.rs +++ b/crates/libafl/src/mutators/gramatron.rs @@ -12,7 +12,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, IsTestcaseMetadataCell}, generators::GramatronGenerator, inputs::{GramatronInput, Terminal}, @@ -119,7 +119,7 @@ pub struct GramatronSpliceMutator; impl Mutator for GramatronSpliceMutator where - S: HasRand + HasCorpus + HasMetadataMut, + S: HasRand + HasCorpus + HasMetadata, { fn mutate( &mut self, diff --git a/crates/libafl/src/mutators/mopt_mutator.rs b/crates/libafl/src/mutators/mopt_mutator.rs index febd164a3be..897f1a23d89 100644 --- a/crates/libafl/src/mutators/mopt_mutator.rs +++ b/crates/libafl/src/mutators/mopt_mutator.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use super::MutationId; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId}, mutators::{ComposedByMutations, MutationResult, Mutator, MutatorsTuple, ScheduledMutator}, state::{HasCorpus, HasRand, HasSolutions}, @@ -371,7 +371,7 @@ pub struct StdMOptMutator { impl Mutator for StdMOptMutator where MT: MutatorsTuple, - S: HasRand + HasMetadataMut + HasCorpus + HasSolutions, + S: HasRand + HasMetadata + HasCorpus + HasSolutions, { #[inline] fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { @@ -501,7 +501,7 @@ impl StdMOptMutator { swarm_num: usize, ) -> Result where - S: HasMetadataMut + HasRand, + S: HasMetadata + HasRand, MT: NamedTuple, { if !state.has_metadata::() { @@ -519,7 +519,7 @@ impl StdMOptMutator { } fn core_mutate(&mut self, state: &mut S, input: &mut I) -> Result where - S: HasMetadataMut + HasRand + HasSolutions + HasCorpus, + S: HasMetadata + HasRand + HasSolutions + HasCorpus, MT: MutatorsTuple, { let mut r = MutationResult::Skipped; @@ -546,7 +546,7 @@ impl StdMOptMutator { fn pilot_mutate(&mut self, state: &mut S, input: &mut I) -> Result where - S: HasMetadataMut + HasRand + HasSolutions + HasCorpus, + S: HasMetadata + HasRand + HasSolutions + HasCorpus, MT: MutatorsTuple, { let mut r = MutationResult::Skipped; @@ -604,7 +604,7 @@ impl Named for StdMOptMutator { impl ScheduledMutator for StdMOptMutator where MT: MutatorsTuple, - S: HasRand + HasMetadataMut + HasCorpus + HasSolutions, + S: HasRand + HasMetadata + HasCorpus + HasSolutions, { /// Compute the number of iterations used to apply stacked mutations fn iterations(&self, state: &mut S, _: &I) -> u64 { diff --git a/crates/libafl/src/mutators/scheduled.rs b/crates/libafl/src/mutators/scheduled.rs index fe8d67aef6f..db2332e6c6e 100644 --- a/crates/libafl/src/mutators/scheduled.rs +++ b/crates/libafl/src/mutators/scheduled.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use super::MutationId; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, mutators::{ MutationResult, Mutator, MutatorsTuple, diff --git a/crates/libafl/src/mutators/token_mutations.rs b/crates/libafl/src/mutators/token_mutations.rs index 500d509a206..528a0bcb113 100644 --- a/crates/libafl/src/mutators/token_mutations.rs +++ b/crates/libafl/src/mutators/token_mutations.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "std")] use crate::mutators::str_decode; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{CorpusId, HasCurrentCorpusId}, inputs::{HasMutatorBytes, ResizableMutator}, mutators::{ @@ -1325,7 +1325,7 @@ impl AflppRedQueen { impl MultiMutator for AflppRedQueen where - S: HasMetadataMut + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusId, + S: HasMetadata + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusId, I: ResizableMutator + From> + HasMutatorBytes, { #[expect(clippy::needless_range_loop, clippy::too_many_lines)] diff --git a/crates/libafl/src/mutators/tuneable.rs b/crates/libafl/src/mutators/tuneable.rs index 4af018ce840..10e66443092 100644 --- a/crates/libafl/src/mutators/tuneable.rs +++ b/crates/libafl/src/mutators/tuneable.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; pub use crate::mutators::{mutations::*, token_mutations::*}; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, mutators::{ ComposedByMutations, MutationId, MutationResult, Mutator, MutatorsTuple, ScheduledMutator, }, @@ -69,7 +69,7 @@ impl TuneableScheduledMutatorMetadata { } /// Gets the stored metadata, used to alter the [`TuneableScheduledMutator`] behavior, mut - pub fn get_mut(state: &mut S) -> Result<&mut Self, Error> { + pub fn get_mut(state: &mut S) -> Result<&mut Self, Error> { state .metadata_map_mut() .get_mut::() @@ -89,7 +89,7 @@ pub struct TuneableScheduledMutator { impl Mutator for TuneableScheduledMutator where MT: MutatorsTuple, - S: HasRand + HasMetadataMut, + S: HasRand + HasMetadata, { #[inline] fn mutate(&mut self, state: &mut S, input: &mut I) -> Result { @@ -129,7 +129,7 @@ impl Named for TuneableScheduledMutator { impl ScheduledMutator for TuneableScheduledMutator where MT: MutatorsTuple, - S: HasRand + HasMetadataMut, + S: HasRand + HasMetadata, { /// Compute the number of iterations used to apply stacked mutations fn iterations(&self, state: &mut S, _: &I) -> u64 { @@ -212,7 +212,7 @@ impl TuneableScheduledMutator { pub fn new(state: &mut S, mutations: MT) -> Self where MT: NamedTuple, - S: HasRand + HasMetadataMut, + S: HasRand + HasMetadata, { if !state.has_metadata::() { state.add_metadata(TuneableScheduledMutatorMetadata::default()); @@ -233,7 +233,7 @@ impl TuneableScheduledMutator { /// as it internally only needs a single metadata lookup pub fn set_iters(&self, state: &mut S, iters: u64) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); @@ -256,7 +256,7 @@ impl TuneableScheduledMutator { mut iter_probabilities_pow: Vec, ) -> Result<(), Error> where - S: HasMetadataMut, + S: HasMetadata, { if iter_probabilities_pow.len() >= 32 { return Err(Error::illegal_argument( @@ -276,7 +276,7 @@ impl TuneableScheduledMutator { /// Gets the set amount of iterations pub fn get_iters(&self, state: &S) -> Option where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get(state).unwrap(); metadata.iters @@ -285,7 +285,7 @@ impl TuneableScheduledMutator { /// Sets the mutation ids pub fn set_mutation_ids(&self, state: &mut S, mutations: Vec) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids = mutations; @@ -302,7 +302,7 @@ impl TuneableScheduledMutator { mut mutation_probabilities: Vec, ) -> Result<(), Error> where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids.clear(); @@ -321,7 +321,7 @@ impl TuneableScheduledMutator { mutations: Vec, iters: u64, ) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids = mutations; @@ -332,7 +332,7 @@ impl TuneableScheduledMutator { /// Appends a mutation id to the end of the mutations pub fn push_mutation_id(state: &mut S, mutation_id: MutationId) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = TuneableScheduledMutatorMetadata::get_mut(state).unwrap(); metadata.mutation_ids.push(mutation_id); @@ -341,7 +341,7 @@ impl TuneableScheduledMutator { /// Resets this to a randomic mutational stage pub fn reset(self, state: &mut S) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = state .metadata_map_mut() diff --git a/crates/libafl/src/mutators/unicode/mod.rs b/crates/libafl/src/mutators/unicode/mod.rs index 32d14d504ce..b3a86ced4e4 100644 --- a/crates/libafl/src/mutators/unicode/mod.rs +++ b/crates/libafl/src/mutators/unicode/mod.rs @@ -10,7 +10,7 @@ use core::{ use libafl_bolts::{Error, HasLen, Named, rands::Rand}; use crate::{ - HasMetadata, HasMetadataMut, + HasMetadata, corpus::{CorpusId, HasTestcase, IsTestcaseMetadataCell, Testcase}, inputs::{BytesInput, HasMutatorBytes, ResizableMutator}, mutators::{MutationResult, Mutator, Tokens, rand_range}, diff --git a/crates/libafl/src/observers/cmp.rs b/crates/libafl/src/observers/cmp.rs index d941c885e0e..f753f1b4a4a 100644 --- a/crates/libafl/src/observers/cmp.rs +++ b/crates/libafl/src/observers/cmp.rs @@ -11,7 +11,7 @@ use hashbrown::HashMap; use libafl_bolts::{AsSlice, HasLen, Named, ownedref::OwnedRefMut}; use serde::{Deserialize, Serialize}; -use crate::{Error, HasMetadataMut, executors::ExitKind, observers::Observer}; +use crate::{Error, HasMetadata, executors::ExitKind, observers::Observer}; /// A bytes string for cmplog with up to 32 elements. #[derive(Debug, Copy, Clone, Serialize, Deserialize, Eq, PartialEq)] @@ -253,7 +253,7 @@ where impl Observer for StdCmpObserver<'_, CM> where CM: Serialize + CmpMap + HasLen, - S: HasMetadataMut, + S: HasMetadata, { fn pre_exec(&mut self, _state: &mut S, _input: &I) -> Result<(), Error> { self.cmp_map.as_mut().reset()?; diff --git a/crates/libafl/src/schedulers/accounting.rs b/crates/libafl/src/schedulers/accounting.rs index e1cda6c7091..29e0a48d5b2 100644 --- a/crates/libafl/src/schedulers/accounting.rs +++ b/crates/libafl/src/schedulers/accounting.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use super::IndexesLenTimeMinimizerScheduler; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, observers::CanTrack, schedulers::{ @@ -113,7 +113,7 @@ pub struct CoverageAccountingScheduler<'a, CS, I, O> { impl Scheduler for CoverageAccountingScheduler<'_, CS, I, O> where CS: Scheduler, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, I: HasLen, O: CanTrack, { @@ -170,7 +170,7 @@ where #[expect(clippy::cast_possible_wrap)] pub fn update_accounting_score(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { let mut indexes = vec![]; let mut new_favoreds = vec![]; @@ -258,7 +258,7 @@ where /// Cull the `Corpus` pub fn accounting_cull(&self, state: &S) -> Result<(), Error> where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { let Some(top_rated) = state.metadata_map().get::() else { return Ok(()); @@ -284,7 +284,7 @@ where /// Provide the observer responsible for determining new indexes. pub fn new(observer: &O, state: &mut S, base: CS, accounting_map: &'a [u32]) -> Self where - S: HasMetadataMut, + S: HasMetadata, { match state.metadata_map().get::() { Some(meta) => { @@ -315,7 +315,7 @@ where accounting_map: &'a [u32], ) -> Self where - S: HasMetadataMut, + S: HasMetadata, { match state.metadata_map().get::() { Some(meta) => { diff --git a/crates/libafl/src/schedulers/minimizer.rs b/crates/libafl/src/schedulers/minimizer.rs index b9eaf7f1333..0a96124a042 100644 --- a/crates/libafl/src/schedulers/minimizer.rs +++ b/crates/libafl/src/schedulers/minimizer.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use super::HasQueueCycles; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, feedbacks::MapIndexesMetadata, observers::CanTrack, @@ -84,7 +84,7 @@ where CS: RemovableScheduler + Scheduler, F: TestcasePenalty, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, { /// Removes an entry from the corpus fn on_remove(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { @@ -190,7 +190,7 @@ where CS: Scheduler, F: TestcasePenalty, M: for<'a> AsIter<'a, Item = usize> + SerdeAny + HasRefCnt, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { @@ -243,7 +243,7 @@ where pub fn update_score(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where F: TestcasePenalty, - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { // Create a new top rated meta if not existing if state.metadata_map().get::().is_none() { diff --git a/crates/libafl/src/schedulers/mod.rs b/crates/libafl/src/schedulers/mod.rs index b4e78288689..6d9c5fc6f62 100644 --- a/crates/libafl/src/schedulers/mod.rs +++ b/crates/libafl/src/schedulers/mod.rs @@ -10,7 +10,7 @@ use libafl_bolts::{ }; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, HasTestcase, IsTestcaseMetadataCell, SchedulerTestcaseMetadata}, random_corpus_id, state::{HasCorpus, HasRand}, @@ -72,7 +72,7 @@ pub fn on_add_metadata_default( ) -> Result<(), Error> where CS: AflScheduler, - S: HasCorpus + HasTestcase + HasMetadataMut, + S: HasCorpus + HasTestcase + HasMetadata, { let current_id = *state.corpus().current(); @@ -109,7 +109,7 @@ pub fn on_evaluation_metadata_default( where CS: AflScheduler, CS::ObserverRef: AsRef, - S: HasMetadataMut, + S: HasMetadata, O: Hash, OT: MatchName, { diff --git a/crates/libafl/src/schedulers/powersched.rs b/crates/libafl/src/schedulers/powersched.rs index e61e2b87484..ef43c30cd1a 100644 --- a/crates/libafl/src/schedulers/powersched.rs +++ b/crates/libafl/src/schedulers/powersched.rs @@ -10,7 +10,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, HasTestcase}, schedulers::{ AflScheduler, HasQueueCycles, RemovableScheduler, Scheduler, on_add_metadata_default, @@ -306,7 +306,7 @@ impl HasQueueCycles for PowerQueueScheduler { impl Scheduler for PowerQueueScheduler where - for<'a> S: HasCorpus + HasMetadataMut + HasTestcase, + for<'a> S: HasCorpus + HasMetadata + HasTestcase, O: Hash, C: AsRef, { @@ -369,7 +369,7 @@ where #[must_use] pub fn new(state: &mut S, observer: &C, strat: PowerSchedule) -> Self where - S: HasMetadata + HasMetadataMut, + S: HasMetadata + HasMetadata, { if !state.has_metadata::() { state.add_metadata::(SchedulerMetadata::new(Some(strat))); diff --git a/crates/libafl/src/schedulers/probabilistic_sampling.rs b/crates/libafl/src/schedulers/probabilistic_sampling.rs index c4196529615..45a748de038 100644 --- a/crates/libafl/src/schedulers/probabilistic_sampling.rs +++ b/crates/libafl/src/schedulers/probabilistic_sampling.rs @@ -9,7 +9,7 @@ use libafl_bolts::rands::Rand; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, schedulers::{RemovableScheduler, Scheduler, TestcaseScore}, state::{HasCorpus, HasRand}, @@ -66,7 +66,7 @@ impl ProbabilitySamplingScheduler { pub fn store_probability(&self, state: &mut S, id: CorpusId) -> Result<(), Error> where F: TestcaseScore, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, { let prob = F::compute(state, id)?; debug_assert!( @@ -86,7 +86,7 @@ impl ProbabilitySamplingScheduler { impl RemovableScheduler for ProbabilitySamplingScheduler where F: TestcaseScore, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, { fn on_remove(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { let meta = state @@ -120,7 +120,7 @@ where impl Scheduler for ProbabilitySamplingScheduler where F: TestcaseScore, - S: HasCorpus + HasMetadataMut + HasRand, + S: HasCorpus + HasMetadata + HasRand, { fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { let current_id = *state.corpus().current(); diff --git a/crates/libafl/src/schedulers/queue.rs b/crates/libafl/src/schedulers/queue.rs index e9ed6c5153a..2fff7fbb6bb 100644 --- a/crates/libafl/src/schedulers/queue.rs +++ b/crates/libafl/src/schedulers/queue.rs @@ -3,7 +3,7 @@ use alloc::borrow::ToOwned; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell}, schedulers::{HasQueueCycles, RemovableScheduler, Scheduler}, state::HasCorpus, @@ -20,7 +20,7 @@ impl RemovableScheduler for QueueScheduler where S: HasCorpus {} impl Scheduler for QueueScheduler where - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { // Set parent id diff --git a/crates/libafl/src/schedulers/tuneable.rs b/crates/libafl/src/schedulers/tuneable.rs index 0999f90a93f..0588e3e9d02 100644 --- a/crates/libafl/src/schedulers/tuneable.rs +++ b/crates/libafl/src/schedulers/tuneable.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use super::RemovableScheduler; use crate::{ - Error, HasMetadata, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId}, schedulers::Scheduler, state::HasCorpus, @@ -36,7 +36,7 @@ impl TuneableScheduler { #[must_use] pub fn new(state: &mut S) -> Self where - S: HasMetadataMut, + S: HasMetadata, { if !state.has_metadata::() { state.add_metadata(TuneableSchedulerMetadata::default()); @@ -46,7 +46,7 @@ impl TuneableScheduler { fn metadata_mut(state: &mut S) -> &mut TuneableSchedulerMetadata where - S: HasMetadataMut, + S: HasMetadata, { state .metadata_map_mut() @@ -67,7 +67,7 @@ impl TuneableScheduler { /// Sets the next corpus id to be used pub fn set_next(state: &mut S, next: CorpusId) where - S: HasMetadataMut, + S: HasMetadata, { Self::metadata_mut(state).next = Some(next); } @@ -83,7 +83,7 @@ impl TuneableScheduler { /// Resets this to a queue scheduler pub fn reset(state: &mut S) where - S: HasMetadataMut, + S: HasMetadata, { let metadata = Self::metadata_mut(state); metadata.next = None; diff --git a/crates/libafl/src/schedulers/weighted.rs b/crates/libafl/src/schedulers/weighted.rs index a258d1a1201..50360127b23 100644 --- a/crates/libafl/src/schedulers/weighted.rs +++ b/crates/libafl/src/schedulers/weighted.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadataMut, + Error, HasMetadata, corpus::{Corpus, CorpusId, HasTestcase}, random_corpus_id, schedulers::{ @@ -115,7 +115,7 @@ where #[must_use] pub fn new(state: &mut S, observer: &C) -> Self where - S: HasMetadataMut, + S: HasMetadata, { Self::with_schedule(state, observer, None) } @@ -124,7 +124,7 @@ where #[must_use] pub fn with_schedule(state: &mut S, observer: &C, strat: Option) -> Self where - S: HasMetadataMut, + S: HasMetadata, { let _ = state.metadata_or_insert_with(|| SchedulerMetadata::new(strat)); let _ = state.metadata_or_insert_with(WeightedScheduleMetadata::new); @@ -158,7 +158,7 @@ where pub fn create_alias_table(&self, state: &mut S) -> Result<(), Error> where F: TestcaseScore, - S: HasCorpus + HasMetadataMut, + S: HasCorpus + HasMetadata, { let n = state.corpus().count(); @@ -305,7 +305,7 @@ where C: AsRef + Named, F: TestcaseScore, O: Hash, - S: HasCorpus + HasMetadataMut + HasRand + HasTestcase, + S: HasCorpus + HasMetadata + HasRand + HasTestcase, { /// Called when a [`Testcase`] is added to the corpus fn on_add(&mut self, state: &mut S, id: CorpusId) -> Result<(), Error> { diff --git a/crates/libafl/src/stages/calibrate.rs b/crates/libafl/src/stages/calibrate.rs index fb32f763d61..536f3bb2b5f 100644 --- a/crates/libafl/src/stages/calibrate.rs +++ b/crates/libafl/src/stages/calibrate.rs @@ -13,7 +13,7 @@ use num_traits::Bounded; use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, HasScheduler, + Error, HasMetadata, HasNamedMetadata, HasScheduler, corpus::{Corpus, HasCurrentCorpusId, IsTestcaseMetadataCell, SchedulerTestcaseMetadata}, events::{Event, EventFirer, EventWithStats, LogSeverity}, executors::{Executor, ExitKind, HasObservers}, @@ -138,8 +138,8 @@ where Serialize + Deserialize<'de> + 'static + Default + Debug + Bounded, OT: ObserversTuple, S: HasCorpus - + HasMetadataMut - + HasNamedMetadataMut + + HasMetadata + + HasNamedMetadata + HasExecutions + HasCurrentTestcase + HasCurrentCorpusId, @@ -372,7 +372,7 @@ where impl Restartable for CalibrationStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // Calibration stage disallow restarts diff --git a/crates/libafl/src/stages/colorization.rs b/crates/libafl/src/stages/colorization.rs index c1b5ff16475..f7ca8a24745 100644 --- a/crates/libafl/src/stages/colorization.rs +++ b/crates/libafl/src/stages/colorization.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::HasCurrentCorpusId, events::EventFirer, executors::{Executor, HasObservers}, @@ -78,7 +78,7 @@ impl Stage for ColorizationStage, E: HasObservers + Executor, - S: HasCorpus + HasMetadataMut + HasRand + HasNamedMetadata + HasCurrentCorpusId, + S: HasCorpus + HasMetadata + HasRand + HasNamedMetadata + HasCurrentCorpusId, E::Observers: ObserversTuple, I: ResizableMutator + HasMutatorBytes + Clone, O: Hash, @@ -101,7 +101,7 @@ where impl Restartable for ColorizationStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // This is a deterministic stage @@ -161,7 +161,7 @@ where C: AsRef + Named, E: HasObservers + Executor, E::Observers: ObserversTuple, - S: HasCorpus + HasMetadataMut + HasRand + HasCurrentCorpusId + HasCurrentTestcase, + S: HasCorpus + HasMetadata + HasRand + HasCurrentCorpusId + HasCurrentTestcase, I: ResizableMutator + HasMutatorBytes + Clone, { #[inline] diff --git a/crates/libafl/src/stages/concolic.rs b/crates/libafl/src/stages/concolic.rs index e38d80ace12..41a17064c06 100644 --- a/crates/libafl/src/stages/concolic.rs +++ b/crates/libafl/src/stages/concolic.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ #[cfg(all(feature = "concolic_mutation", feature = "introspection"))] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::{HasCurrentCorpusId, IsTestcaseMetadataCell}, executors::{Executor, HasObservers}, observers::{ObserversTuple, concolic::ConcolicObserver}, @@ -53,7 +53,7 @@ where TE::Observers: ObserversTuple, S: HasExecutions + HasCorpus - + HasMetadataMut + + HasMetadata + HasCurrentTestcase + HasCurrentCorpusId + MaybeHasClientPerfMonitor, @@ -81,7 +81,7 @@ where impl Restartable for ConcolicTracingStage<'_, EM, I, TE, S, Z> where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // This is a deterministic stage @@ -377,7 +377,7 @@ where S: HasExecutions + HasCorpus + HasMetadata - + HasNamedMetadataMut + + HasNamedMetadata + HasCurrentTestcase + MaybeHasClientPerfMonitor + HasCurrentCorpusId, @@ -420,7 +420,7 @@ where #[cfg(feature = "concolic_mutation")] impl Restartable for SimpleConcolicMutationalStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/dump.rs b/crates/libafl/src/stages/dump.rs index 2be35e4c49e..d5b58ac9752 100644 --- a/crates/libafl/src/stages/dump.rs +++ b/crates/libafl/src/stages/dump.rs @@ -14,13 +14,7 @@ use std::{ use libafl_bolts::impl_serdeany; use serde::{Deserialize, Serialize}; -use crate::{ - Error, HasMetadataMut, - corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase, TestcaseMetadata}, - inputs::Input, - stages::{Restartable, Stage}, - state::{HasCorpus, HasRand, HasSolutions}, -}; +use crate::{Error, corpus::{Corpus, CorpusId, IsTestcaseMetadataCell, Testcase, TestcaseMetadata}, inputs::Input, stages::{Restartable, Stage}, state::{HasCorpus, HasRand, HasSolutions}, HasMetadata}; /// Metadata used to store information about disk dump indexes for names #[cfg_attr( @@ -49,7 +43,7 @@ impl Stage for DumpToDiskStage Vec, CB2: FnMut(&I, &TestcaseMetadata, &CorpusId) -> P, - S: HasCorpus + HasSolutions + HasRand + HasMetadataMut, + S: HasCorpus + HasSolutions + HasRand + HasMetadata, P: AsRef, { #[inline] @@ -100,7 +94,7 @@ impl Z, > where - S: HasCorpus + HasSolutions + HasRand + HasMetadataMut, + S: HasCorpus + HasSolutions + HasRand + HasMetadata, I: Input, { /// Create a new [`DumpToDiskStage`] with a default `generate_filename` function. @@ -142,7 +136,7 @@ where impl DumpToDiskStage where - S: HasMetadataMut + HasSolutions, + S: HasMetadata + HasSolutions, { /// Create a new [`DumpToDiskStage`] with a custom `generate_filename` function. pub fn new_with_custom_filenames( diff --git a/crates/libafl/src/stages/generalization.rs b/crates/libafl/src/stages/generalization.rs index d09ef199da9..27a6bac3670 100644 --- a/crates/libafl/src/stages/generalization.rs +++ b/crates/libafl/src/stages/generalization.rs @@ -14,7 +14,7 @@ use libafl_bolts::{ #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::{Corpus, HasCurrentCorpusId, IsTestcaseMetadataCell}, executors::{Executor, HasObservers}, feedbacks::map::MapNoveltiesMetadata, @@ -62,7 +62,7 @@ impl Named for GeneralizationStage impl Restartable for GeneralizationStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/logics.rs b/crates/libafl/src/stages/logics.rs index e82811a3109..a5f0c22f638 100644 --- a/crates/libafl/src/stages/logics.rs +++ b/crates/libafl/src/stages/logics.rs @@ -310,7 +310,7 @@ mod test { #[cfg(any(not(feature = "serdeany_autoreg"), miri))] use crate::stages::RetryCountRestartHelper; use crate::{ - HasMetadataMut, NopFuzzer, + HasMetadata, NopFuzzer, events::NopEventManager, executors::nop::NopExecutor, stages::{ @@ -340,7 +340,7 @@ mod test { #[expect(clippy::unnecessary_wraps)] fn should_restart(state: &mut S, _stage: &ST) -> Result where - S: HasMetadataMut, + S: HasMetadata, { // check if we're resuming let _metadata = state.metadata_or_insert_with(|| Self { count: 0 }); @@ -349,7 +349,7 @@ mod test { fn clear_progress(state: &mut S, _stage: &ST) -> Result<(), Error> where - S: HasMetadataMut, + S: HasMetadata, { if state.remove_metadata::().is_none() { return Err(Error::illegal_state( @@ -362,7 +362,7 @@ mod test { impl Stage for ResumeSucceededStage where - S: HasMetadataMut, + S: HasMetadata, { fn perform( &mut self, @@ -384,7 +384,7 @@ mod test { impl Restartable for ResumeSucceededStage where - S: HasMetadataMut, + S: HasMetadata, { fn should_restart(&mut self, state: &mut S) -> Result { TestProgress::should_restart(state, self) @@ -397,7 +397,7 @@ mod test { impl Stage for ResumeFailedStage where - S: HasMetadataMut, + S: HasMetadata, { fn perform( &mut self, @@ -422,7 +422,7 @@ mod test { impl Restartable for ResumeFailedStage where - S: HasMetadataMut, + S: HasMetadata, { fn should_restart(&mut self, state: &mut S) -> Result { TestProgress::should_restart(state, self) diff --git a/crates/libafl/src/stages/mod.rs b/crates/libafl/src/stages/mod.rs index c6fb97a11aa..d26c10bb9c4 100644 --- a/crates/libafl/src/stages/mod.rs +++ b/crates/libafl/src/stages/mod.rs @@ -46,7 +46,7 @@ pub use unicode::*; pub use verify_timeouts::{TimeoutsToVerify, VerifyTimeoutsStage}; use crate::{ - Error, HasNamedMetadata, HasNamedMetadataMut, + Error, HasNamedMetadata, corpus::{CorpusId, HasCurrentCorpusId}, events::SendExiting, state::{HasCurrentStageId, HasExecutions, MaybeHasClientPerfMonitor, Stoppable}, @@ -337,7 +337,7 @@ where impl Restartable for ClosureStage where - S: HasNamedMetadataMut + HasCurrentCorpusId, + S: HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { @@ -385,7 +385,7 @@ impl RetryCountRestartHelper { /// Don't allow restart pub fn no_retry(state: &mut S, name: &str) -> Result where - S: HasNamedMetadataMut + HasCurrentCorpusId, + S: HasNamedMetadata + HasCurrentCorpusId, { Self::should_restart(state, name, 1) } @@ -395,7 +395,7 @@ impl RetryCountRestartHelper { /// Returns `true` if the stage should run pub fn should_restart(state: &mut S, name: &str, max_retries: usize) -> Result where - S: HasNamedMetadataMut + HasCurrentCorpusId, + S: HasNamedMetadata + HasCurrentCorpusId, { let corpus_id = state.current_corpus_id()?.ok_or_else(|| { Error::illegal_state( @@ -434,7 +434,7 @@ impl RetryCountRestartHelper { /// Clears the progress pub fn clear_progress(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { state.named_metadata_mut::(name)?.tries_remaining = None; Ok(()) @@ -508,7 +508,7 @@ impl ExecutionCountRestartHelper { /// Initialize progress for the stage this wrapper wraps. pub fn should_restart(&mut self, state: &mut S, name: &str) -> Result where - S: HasNamedMetadataMut + HasExecutions, + S: HasNamedMetadata + HasExecutions, { let executions = *state.executions(); let metadata = @@ -522,7 +522,7 @@ impl ExecutionCountRestartHelper { /// Clear progress for the stage this wrapper wraps. pub fn clear_progress(&mut self, state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { self.started_at_execs = None; let _metadata = state.remove_named_metadata::(name); diff --git a/crates/libafl/src/stages/mutational.rs b/crates/libafl/src/stages/mutational.rs index d6d62a58291..073d6a27bc7 100644 --- a/crates/libafl/src/stages/mutational.rs +++ b/crates/libafl/src/stages/mutational.rs @@ -12,7 +12,7 @@ use libafl_bolts::{Named, rands::Rand}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::{CorpusId, HasCurrentCorpusId, Testcase, testcase::IsTestcaseMetadataCell}, fuzzer::Evaluator, inputs::Input, @@ -173,7 +173,7 @@ where impl Restartable for StdMutationalStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::should_restart(state, &self.name, 3) @@ -349,7 +349,7 @@ where impl Restartable for MultiMutationalStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/power.rs b/crates/libafl/src/stages/power.rs index 2fe2baca203..16a25c597ed 100644 --- a/crates/libafl/src/stages/power.rs +++ b/crates/libafl/src/stages/power.rs @@ -11,7 +11,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers}, fuzzer::Evaluator, @@ -105,7 +105,7 @@ where impl Restartable for PowerMutationalStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { // Make sure we don't get stuck crashing on a single testcase diff --git a/crates/libafl/src/stages/push/mod.rs b/crates/libafl/src/stages/push/mod.rs index c5ff4786feb..9e4ac7f727c 100644 --- a/crates/libafl/src/stages/push/mod.rs +++ b/crates/libafl/src/stages/push/mod.rs @@ -21,9 +21,8 @@ use libafl_bolts::Named; pub use mutational::StdMutationalPushStage; use crate::{ - Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasMetadata, HasNamedMetadataMut, + Error, EvaluatorObservers, ExecutesInput, ExecutionProcessor, HasMetadata, HasNamedMetadata, HasScheduler, - common::HasNamedMetadata, corpus::{CorpusId, HasCurrentCorpusId}, events::{EventFirer, EventRestarter, HasEventManagerId, ProgressReporter}, executors::{Executor, ExitKind, HasObservers}, @@ -245,7 +244,7 @@ impl Named for PushStageAdapter { impl Restartable for PushStageAdapter where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { diff --git a/crates/libafl/src/stages/replay.rs b/crates/libafl/src/stages/replay.rs index 8204f154e7f..88060d0270b 100644 --- a/crates/libafl/src/stages/replay.rs +++ b/crates/libafl/src/stages/replay.rs @@ -12,7 +12,7 @@ use libafl_bolts::{Named, impl_serdeany}; use serde::{Deserialize, Serialize}; use crate::{ - Error, Evaluator, HasMetadataMut, + Error, Evaluator, HasMetadata, corpus::{Corpus, CorpusId}, stages::{Restartable, Stage}, state::{HasCorpus, HasSolutions}, @@ -108,7 +108,7 @@ impl ReplayStage { impl Stage for ReplayStage where - S: HasCorpus + HasSolutions + HasMetadataMut, + S: HasCorpus + HasSolutions + HasMetadata, Z: Evaluator, I: Clone, { @@ -163,7 +163,7 @@ where impl Restartable for ReplayStage where - S: HasMetadataMut, + S: HasMetadata, { fn should_restart(&mut self, state: &mut S) -> Result { state.metadata_or_insert_with(ReplayRestarterMetadata::default); diff --git a/crates/libafl/src/stages/shadow.rs b/crates/libafl/src/stages/shadow.rs index 3ceec60e099..7bfdd68b4c9 100644 --- a/crates/libafl/src/stages/shadow.rs +++ b/crates/libafl/src/stages/shadow.rs @@ -12,7 +12,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasNamedMetadata, HasNamedMetadataMut, + Error, HasNamedMetadata, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers, ShadowExecutor}, mark_feature_time, @@ -104,7 +104,7 @@ where impl Restartable for ShadowTracingStage where - S: HasNamedMetadataMut + HasCurrentCorpusId, + S: HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::no_retry(state, &self.name) diff --git a/crates/libafl/src/stages/sync.rs b/crates/libafl/src/stages/sync.rs index a7ac2654095..0ee0c28a892 100644 --- a/crates/libafl/src/stages/sync.rs +++ b/crates/libafl/src/stages/sync.rs @@ -15,7 +15,7 @@ use libafl_bolts::{ use serde::{Deserialize, Serialize}; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::{Corpus, CorpusId, HasCurrentCorpusId}, events::{Event, EventConfig, EventFirer, EventWithStats, llmp::LlmpEventConverter}, executors::{Executor, ExitKind, HasObservers}, @@ -80,7 +80,7 @@ where Z: Evaluator, S: HasCorpus + HasRand - + HasMetadataMut + + HasMetadata + HasNamedMetadata + HasCurrentCorpusId + MaybeHasClientPerfMonitor, @@ -155,7 +155,7 @@ where impl Restartable for SyncFromDiskStage where - S: HasMetadata + HasNamedMetadataMut + HasCurrentCorpusId, + S: HasMetadata + HasNamedMetadata + HasCurrentCorpusId, { #[inline] fn should_restart(&mut self, state: &mut S) -> Result { @@ -253,7 +253,7 @@ where ICB: InputConverter, S: HasExecutions + HasRand - + HasMetadataMut + + HasMetadata + HasSolutions + HasCurrentTestcase + Stoppable diff --git a/crates/libafl/src/stages/time_tracker.rs b/crates/libafl/src/stages/time_tracker.rs index b01447322f4..b4945e59265 100644 --- a/crates/libafl/src/stages/time_tracker.rs +++ b/crates/libafl/src/stages/time_tracker.rs @@ -5,7 +5,7 @@ use core::{marker::PhantomData, time::Duration}; use libafl_bolts::{Error, current_time}; use crate::{ - HasMetadataMut, + HasMetadata, stages::{Restartable, Stage}, }; @@ -30,7 +30,7 @@ impl TimeTrackingStageWrapper { impl Stage for TimeTrackingStageWrapper where - S: HasMetadataMut, + S: HasMetadata, ST: Stage, T: libafl_bolts::serdeany::SerdeAny + From, { diff --git a/crates/libafl/src/stages/tmin.rs b/crates/libafl/src/stages/tmin.rs index 0e410975b27..72549f85c5d 100644 --- a/crates/libafl/src/stages/tmin.rs +++ b/crates/libafl/src/stages/tmin.rs @@ -19,7 +19,7 @@ use crate::feedbacks::premature_last_result_err; use crate::monitors::stats::PerfFeature; use crate::{ Error, ExecutesInput, ExecutionProcessor, HasFeedback, HasMetadata, HasNamedMetadata, - HasNamedMetadataMut, HasScheduler, + HasScheduler, corpus::{Corpus, HasCurrentCorpusId, testcase::TestcaseMetadata}, events::EventFirer, executors::{ExitKind, HasObservers}, @@ -96,7 +96,7 @@ where impl Restartable for StdTMinMutationalStage where - S: HasNamedMetadataMut + HasExecutions, + S: HasNamedMetadata + HasExecutions, { fn should_restart(&mut self, state: &mut S) -> Result { self.restart_helper.should_restart(state, &self.name) diff --git a/crates/libafl/src/stages/tracing.rs b/crates/libafl/src/stages/tracing.rs index 02239972984..496d2ba4eb6 100644 --- a/crates/libafl/src/stages/tracing.rs +++ b/crates/libafl/src/stages/tracing.rs @@ -11,7 +11,7 @@ use libafl_bolts::Named; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, HasNamedMetadata, HasNamedMetadataMut, + Error, HasNamedMetadata, corpus::HasCurrentCorpusId, executors::{Executor, HasObservers}, inputs::Input, @@ -94,7 +94,7 @@ where impl Restartable for TracingStage where - S: HasNamedMetadataMut + HasCurrentCorpusId, + S: HasNamedMetadata + HasCurrentCorpusId, { fn should_restart(&mut self, state: &mut S) -> Result { RetryCountRestartHelper::no_retry(state, &self.name) diff --git a/crates/libafl/src/stages/tuneable.rs b/crates/libafl/src/stages/tuneable.rs index 6404b6e7bb2..3a912b4baa5 100644 --- a/crates/libafl/src/stages/tuneable.rs +++ b/crates/libafl/src/stages/tuneable.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "introspection")] use crate::monitors::stats::PerfFeature; use crate::{ - Error, Evaluator, HasMetadata, HasNamedMetadata, HasNamedMetadataMut, mark_feature_time, + Error, Evaluator, HasMetadata, HasNamedMetadata, mark_feature_time, mutators::{MutationResult, Mutator}, nonzero, stages::{ @@ -38,7 +38,7 @@ pub const STD_TUNEABLE_MUTATIONAL_STAGE_NAME: &str = "TuneableMutationalStage"; /// Set the number of iterations to be used by this mutational stage by name pub fn set_iters_by_name(state: &mut S, iters: u64, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { let metadata = state .named_metadata_map_mut() @@ -52,7 +52,7 @@ where /// Set the number of iterations to be used by this mutational stage with a default name pub fn set_iters_std(state: &mut S, iters: u64) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -84,7 +84,7 @@ pub fn set_seed_fuzz_time_by_name( name: &str, ) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { let metadata = state .named_metadata_map_mut() @@ -98,7 +98,7 @@ where /// Set the time for a single seed to be used by this mutational stage with a default name pub fn set_seed_fuzz_time_std(state: &mut S, fuzz_time: Duration) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -126,7 +126,7 @@ where /// Reset this to a normal, randomized, stage by name pub fn reset_by_name(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { state .named_metadata_map_mut() @@ -141,7 +141,7 @@ where /// Reset this to a normal, randomized, stage with a default name pub fn reset_std(state: &mut S) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -215,7 +215,7 @@ where impl Restartable for TuneableMutationalStage where - S: HasNamedMetadataMut + HasExecutions, + S: HasNamedMetadata + HasExecutions, { fn should_restart(&mut self, state: &mut S) -> Result { self.restart_helper.should_restart(state, &self.name) @@ -309,7 +309,7 @@ where #[must_use] pub fn new(state: &mut S, mutator: M) -> Self where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { Self::transforming(state, mutator, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -317,7 +317,7 @@ where /// Crates a new tuneable mutational stage with the given name pub fn with_name(state: &mut S, mutator: M, name: &str) -> Self where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { Self::transforming(state, mutator, name) } @@ -325,7 +325,7 @@ where /// Set the number of iterations to be used by this [`TuneableMutationalStage`] pub fn set_iters(&self, state: &mut S, iters: u64) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_iters_by_name(state, iters, &self.name) } @@ -333,7 +333,7 @@ where /// Set the number of iterations to be used by the std [`TuneableMutationalStage`] pub fn set_iters_std(state: &mut S, iters: u64) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -341,7 +341,7 @@ where /// Set the number of iterations to be used by the [`TuneableMutationalStage`] with the given name pub fn set_iters_by_name(state: &mut S, iters: u64, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_iters_by_name(state, iters, name) } @@ -370,7 +370,7 @@ where /// Set the time to mutate a single input in this [`TuneableMutationalStage`] pub fn set_seed_fuzz_time(&self, state: &mut S, fuzz_time: Duration) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_seed_fuzz_time_by_name(state, fuzz_time, &self.name) } @@ -378,7 +378,7 @@ where /// Set the time to mutate a single input in the std [`TuneableMutationalStage`] pub fn set_seed_fuzz_time_std(state: &mut S, fuzz_time: Duration) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -390,7 +390,7 @@ where name: &str, ) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { set_seed_fuzz_time_by_name(state, fuzz_time, name) } @@ -419,7 +419,7 @@ where /// Reset this to a normal, randomized, stage with pub fn reset(&self, state: &mut S) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { reset_by_name(state, &self.name) } @@ -427,7 +427,7 @@ where /// Reset the std stage to a normal, randomized, stage pub fn reset_std(state: &mut S) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME) } @@ -435,7 +435,7 @@ where /// Reset this to a normal, randomized, stage by name pub fn reset_by_name(state: &mut S, name: &str) -> Result<(), Error> where - S: HasNamedMetadataMut, + S: HasNamedMetadata, { reset_by_name(state, name) } @@ -474,7 +474,7 @@ impl TuneableMutationalStage where M: Mutator, Z: Evaluator, - S: HasRand + HasNamedMetadataMut, + S: HasRand + HasNamedMetadata, { /// Creates a new transforming mutational stage #[must_use] diff --git a/crates/libafl/src/stages/unicode.rs b/crates/libafl/src/stages/unicode.rs index 41aca09e394..475348632a3 100644 --- a/crates/libafl/src/stages/unicode.rs +++ b/crates/libafl/src/stages/unicode.rs @@ -8,7 +8,7 @@ use libafl_bolts::{Error, impl_serdeany}; use serde::{Deserialize, Serialize}; use crate::{ - HasMetadataMut, + HasMetadata, corpus::IsTestcaseMetadataCell, inputs::{BytesInput, HasTargetBytes}, stages::{Restartable, Stage}, diff --git a/crates/libafl/src/stages/verify_timeouts.rs b/crates/libafl/src/stages/verify_timeouts.rs index 8385cc1d793..d28e2f678d6 100644 --- a/crates/libafl/src/stages/verify_timeouts.rs +++ b/crates/libafl/src/stages/verify_timeouts.rs @@ -10,7 +10,7 @@ use libafl_bolts::Error; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{ - Evaluator, HasMetadataMut, + Evaluator, HasMetadata, executors::{Executor, HasObservers, HasTimeout}, inputs::BytesInput, observers::ObserversTuple, @@ -84,7 +84,7 @@ where E::Observers: ObserversTuple, E: Executor + HasObservers + HasTimeout, Z: Evaluator, - S: HasMetadataMut, + S: HasMetadata, I: Debug + Serialize + DeserializeOwned + Default + 'static + Clone, { fn perform( diff --git a/crates/libafl/src/state/mod.rs b/crates/libafl/src/state/mod.rs index 76d39065da3..6c2d2e7b459 100644 --- a/crates/libafl/src/state/mod.rs +++ b/crates/libafl/src/state/mod.rs @@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; #[cfg(feature = "introspection")] use crate::monitors::stats::ClientPerfStats; use crate::{ - Error, HasMetadata, HasMetadataMut, HasNamedMetadata, HasNamedMetadataMut, + Error, HasMetadata, HasNamedMetadata, corpus::{Corpus, CorpusId, HasCurrentCorpusId, HasTestcase, InMemoryCorpus, Testcase}, events::{Event, EventFirer, EventWithStats, LogSeverity}, feedbacks::StateInitializer, @@ -333,9 +333,7 @@ impl HasMetadata for StdState { fn metadata_map(&self) -> &SerdeAnyMap { &self.metadata } -} -impl HasMetadataMut for StdState { /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) #[inline] fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { @@ -349,9 +347,7 @@ impl HasNamedMetadata for StdState { fn named_metadata_map(&self) -> &NamedSerdeAnyMap { &self.named_metadata } -} -impl HasNamedMetadataMut for StdState { /// Get all the metadata into an [`hashbrown::HashMap`] (mutable) #[inline] fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap { @@ -1279,9 +1275,7 @@ impl HasMetadata for NopState { fn metadata_map(&self) -> &SerdeAnyMap { &self.metadata } -} -impl HasMetadataMut for NopState { fn metadata_map_mut(&mut self) -> &mut SerdeAnyMap { &mut self.metadata } @@ -1291,9 +1285,7 @@ impl HasNamedMetadata for NopState { fn named_metadata_map(&self) -> &NamedSerdeAnyMap { &self.named_metadata } -} -impl HasNamedMetadataMut for NopState { fn named_metadata_map_mut(&mut self) -> &mut NamedSerdeAnyMap { &mut self.named_metadata } diff --git a/crates/libafl_frida/src/asan/errors.rs b/crates/libafl_frida/src/asan/errors.rs index d642cc6d3e8..3c86c572e20 100644 --- a/crates/libafl_frida/src/asan/errors.rs +++ b/crates/libafl_frida/src/asan/errors.rs @@ -13,7 +13,7 @@ use frida_gum::interceptor::Interceptor; use frida_gum::{Gum, Process}; use libafl::{ Error, HasMetadata, - corpus::{Testcase, TestcaseMetadata}, + corpus::TestcaseMetadata, executors::ExitKind, feedbacks::{Feedback, StateInitializer}, observers::Observer, diff --git a/crates/libafl_frida/src/lib.rs b/crates/libafl_frida/src/lib.rs index 629183c58ef..6a550863a24 100644 --- a/crates/libafl_frida/src/lib.rs +++ b/crates/libafl_frida/src/lib.rs @@ -350,7 +350,7 @@ mod tests { use frida_gum::Gum; use libafl::{ Fuzzer, StdFuzzer, - corpus::{Corpus, InMemoryCorpus, Testcase}, + corpus::{Corpus, InMemoryCorpus}, events::NopEventManager, executors::{ExitKind, InProcessExecutor}, feedback_and_fast, feedback_or_fast, @@ -463,8 +463,7 @@ mod tests { let mut corpus = InMemoryCorpus::::new(); //TODO - make sure we use the right one - let testcase = Testcase::new(vec![0; 4].into()); - corpus.add(testcase).unwrap(); + corpus.add(vec![0; 4].into()).unwrap(); let rand = StdRand::with_seed(0);