feat(profiling): add ProfilesDictionary (#1349)

feat(profiling): parallel set and string set

test: exercise thread safety

feat(profiling): add ProfilesDictionary

Merge branch 'main' into levi/profiles_dictionary

docs: improve ProfilesDictionary and related types

Merge branch 'main' into levi/profiles_dictionary

test: v1 and v2 have compat reprs

Merge branch 'main' into levi/profiles_dictionary

refactor: move transmute into From

style: remove unused import

Co-authored-by: levi.morrison <[email protected]>

Author: morrisonlevi

libdd-profiling/src/profiles/collections/parallel/string_set.rs

@@ -87,7 +87,7 @@ impl ParallelStringSet {
     }
 
     /// # Safety
-    /// The caller must ensure that the StringId is valid for this set.
+    /// The caller must ensure that the StringRef is valid for this set.
     pub unsafe fn get(&self, id: StringRef) -> &str {
         // SAFETY: safe as long as caller respects this function's safety.
         unsafe { core::mem::transmute::<&str, &str>(id.0.deref()) }

libdd-profiling/src/profiles/datatypes/function.rs

@@ -0,0 +1,120 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::profiles::collections::{SetId, StringRef};
+use crate::profiles::datatypes::StringId2;
+
+/// A representation of a function that is an intersection of the Otel and
+/// Pprof representations. Omits the start line to save space because Datadog
+/// doesn't use this in any way.
+///
+/// This representation is used internally by the `ProfilesDictionary`, and
+/// utilizes the fact that `StringRef`s don't have null values.
+#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Hash)]
+#[repr(C)]
+pub struct Function {
+    pub name: StringRef,
+    pub system_name: StringRef,
+    pub file_name: StringRef,
+}
+
+/// An FFI-safe version of the Function which allows null.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+pub struct Function2 {
+    pub name: StringId2,
+    pub system_name: StringId2,
+    pub file_name: StringId2,
+}
+
+impl From<Function> for Function2 {
+    fn from(f: Function) -> Function2 {
+        Function2 {
+            name: f.name.into(),
+            system_name: f.system_name.into(),
+            file_name: f.file_name.into(),
+        }
+    }
+}
+
+impl From<Function2> for Function {
+    fn from(f2: Function2) -> Function {
+        Function {
+            name: f2.name.into(),
+            system_name: f2.system_name.into(),
+            file_name: f2.file_name.into(),
+        }
+    }
+}
+
+/// An FFI-safe representation of a "handle" to a function which has been
+/// stored in the `ProfilesDictionary`. The representation is ensured to be a
+/// pointer for ABI stability, but callers should not generally dereference
+/// this pointer. When using the id, the caller needs to be sure that the
+/// `ProfilesDictionary` it refers to is the same one that the operations are
+/// performed on; it is not generally guaranteed that ids from one dictionary
+/// can be used in another dictionary, even if it happens to work by
+/// implementation detail.
+#[derive(Clone, Copy, Debug)]
+#[repr(transparent)]
+pub struct FunctionId2(pub(crate) *mut Function2);
+
+// todo: when MSRV is 1.88.0+, derive Default
+impl Default for FunctionId2 {
+    fn default() -> Self {
+        Self(core::ptr::null_mut())
+    }
+}
+
+impl FunctionId2 {
+    pub fn is_empty(self) -> bool {
+        self.0.is_null()
+    }
+
+    /// Converts the `FunctionId2` into an `Option<Function2>` where an empty
+    /// `FunctionId2` converts to a `None`.
+    ///
+    /// # Safety
+    /// The pointer object must still be alive. In general this means the
+    /// profiles dictionary it came from must be alive.
+    pub unsafe fn read(self) -> Option<Function2> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(self.0.read())
+        }
+    }
+}
+
+impl From<SetId<Function>> for FunctionId2 {
+    fn from(id: SetId<Function>) -> FunctionId2 {
+        // SAFETY: the function that SetId points to is layout compatible with
+        // the one that FunctionId2 points to. The reverse is not true for the
+        // null StringId cases.
+        unsafe { core::mem::transmute::<SetId<Function>, FunctionId2>(id) }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::mem::offset_of;
+
+    #[test]
+    fn v1_and_v2_have_compatible_representations() {
+        // Begin with basic size and alignment.
+        assert_eq!(size_of::<Function>(), size_of::<Function2>());
+        assert_eq!(align_of::<Function>(), align_of::<Function2>());
+
+        // Then check members.
+        assert_eq!(offset_of!(Function, name), offset_of!(Function2, name));
+        assert_eq!(
+            offset_of!(Function, system_name),
+            offset_of!(Function2, system_name)
+        );
+        assert_eq!(
+            offset_of!(Function, file_name),
+            offset_of!(Function2, file_name)
+        );
+    }
+}

libdd-profiling/src/profiles/datatypes/mapping.rs

@@ -0,0 +1,138 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::profiles::collections::{SetId, StringRef};
+use crate::profiles::datatypes::StringId2;
+
+/// A representation of a mapping that is an intersection of the Otel and Pprof
+/// representations. Omits boolean attributes because Datadog doesn't use them
+/// in any way.
+///
+/// This representation is used internally by the `ProfilesDictionary`, and
+/// utilizes the fact that `StringRef`s don't have null values.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Hash)]
+pub struct Mapping {
+    pub memory_start: u64,
+    pub memory_limit: u64,
+    pub file_offset: u64,
+    pub filename: StringRef,
+    pub build_id: StringRef, // missing in Otel, is it made into an attribute?
+}
+
+/// An FFI-safe version of the Mapping which allows null.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+pub struct Mapping2 {
+    pub memory_start: u64,
+    pub memory_limit: u64,
+    pub file_offset: u64,
+    pub filename: StringId2,
+    pub build_id: StringId2, // missing in Otel, is it made into an attribute?
+}
+
+impl From<Mapping2> for Mapping {
+    fn from(m2: Mapping2) -> Self {
+        Self {
+            memory_start: m2.memory_start,
+            memory_limit: m2.memory_limit,
+            file_offset: m2.file_offset,
+            filename: m2.filename.into(),
+            build_id: m2.build_id.into(),
+        }
+    }
+}
+
+impl From<Mapping> for Mapping2 {
+    fn from(m: Mapping) -> Self {
+        Self {
+            memory_start: m.memory_start,
+            memory_limit: m.memory_limit,
+            file_offset: m.file_offset,
+            filename: m.filename.into(),
+            build_id: m.build_id.into(),
+        }
+    }
+}
+
+/// An FFI-safe representation of a "handle" to a mapping which has been
+/// stored in the `ProfilesDictionary`. The representation is ensured to be a
+/// pointer for ABI stability, but callers should not generally dereference
+/// this pointer. When using the id, the caller needs to be sure that the
+/// `ProfilesDictionary` it refers to is the same one that the operations are
+/// performed on; it is not generally guaranteed that ids from one dictionary
+/// can be used in another dictionary, even if it happens to work by
+/// implementation detail.
+#[derive(Clone, Copy, Debug)]
+#[repr(transparent)]
+pub struct MappingId2(pub(crate) *mut Mapping2);
+
+// todo: when MSRV is 1.88.0+, derive Default
+impl Default for MappingId2 {
+    fn default() -> Self {
+        Self(core::ptr::null_mut())
+    }
+}
+
+impl MappingId2 {
+    pub fn is_empty(self) -> bool {
+        self.0.is_null()
+    }
+
+    /// Converts the `MappingId2` into an `Option<Mapping2>` where an empty
+    /// `MappingId2` converts to a `None`.
+    ///
+    /// # Safety
+    /// The pointer object must still be alive. In general this means the
+    /// profiles dictionary it came from must be alive.
+    pub unsafe fn read(self) -> Option<Mapping2> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(self.0.read())
+        }
+    }
+}
+
+impl From<SetId<Mapping>> for MappingId2 {
+    fn from(id: SetId<Mapping>) -> MappingId2 {
+        // SAFETY: the mapping that SetId points to is layout compatible with
+        // the one that MappingId2 points to. The reverse is not true for the
+        // null StringId cases.
+        unsafe { core::mem::transmute::<SetId<Mapping>, MappingId2>(id) }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::mem::offset_of;
+    #[test]
+    fn v1_and_v2_have_compatible_representations() {
+        // Begin with basic size and alignment.
+        assert_eq!(size_of::<Mapping>(), size_of::<Mapping2>());
+        assert_eq!(align_of::<Mapping>(), align_of::<Mapping2>());
+
+        // Then check members.
+        assert_eq!(
+            offset_of!(Mapping, memory_start),
+            offset_of!(Mapping2, memory_start)
+        );
+        assert_eq!(
+            offset_of!(Mapping, memory_limit),
+            offset_of!(Mapping2, memory_limit)
+        );
+        assert_eq!(
+            offset_of!(Mapping, file_offset),
+            offset_of!(Mapping2, file_offset)
+        );
+        assert_eq!(
+            offset_of!(Mapping, filename),
+            offset_of!(Mapping2, filename)
+        );
+        assert_eq!(
+            offset_of!(Mapping, build_id),
+            offset_of!(Mapping2, build_id)
+        );
+    }
+}

libdd-profiling/src/profiles/datatypes/mod.rs

@@ -0,0 +1,12 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+mod function;
+mod mapping;
+mod profiles_dictionary;
+mod string;
+
+pub use function::*;
+pub use mapping::*;
+pub use profiles_dictionary::*;
+pub use string::*;