feat(profiling): Profile::{try_new2,try_add_sample2} (#1351)

feat(profiling): parallel set and string set

test: exercise thread safety

feat(profiling): add ProfilesDictionary

Merge branch 'main' into levi/profiles_dictionary

feat(profiling): Profile::try_add_sample2

test: new functions

test: write the test in a more obvious way

docs: improve ProfilesDictionary and related types

Merge branch 'main' into levi/profiles_dictionary

Merge branch 'levi/profiles_dictionary' into levi/try_add_sample2

refactor: try_new2 -> try_new_with_dictionary

fix: bench too

test: v1 and v2 have compat reprs

refactor: move transmute into From

Merge branch 'levi/profiles_dictionary' into levi/try_add_sample2

refactor: more cautious around transmutes

docs: repr(C) Function/Mapping

Merge branch 'main' into levi/profiles_dictionary

Merge branch 'levi/profiles_dictionary' into levi/try_add_sample2

refactor: move transmute into From

Merge branch 'levi/profiles_dictionary' into levi/try_add_sample2

refactor: expand unsafe and better document it

docs: try_reserve_exact optimization for Box<[]>

style: remove unused import

Merge branch 'levi/profiles_dictionary' into levi/try_add_sample2

style: remove unused import

refactor: impl Default for internal::Function

bench: fix filename assignment

Merge branch 'main' into levi/try_add_sample2

Merge branch 'main' into levi/try_add_sample2

Co-authored-by: levi.morrison <[email protected]>

Author: morrisonlevi

libdd-profiling-ffi/src/lib.rs

@@ -7,13 +7,13 @@
 #![cfg_attr(not(test), deny(clippy::todo))]
 #![cfg_attr(not(test), deny(clippy::unimplemented))]
 
-#[cfg(all(feature = "symbolizer", not(target_os = "windows")))]
-pub use symbolizer_ffi::*;
-
 mod exporter;
 mod profiles;
 mod string_storage;
 
+#[cfg(all(feature = "symbolizer", not(target_os = "windows")))]
+pub use symbolizer_ffi::*;
+
 // re-export crashtracker ffi
 #[cfg(feature = "crashtracker-ffi")]
 pub use libdd_crashtracker_ffi::*;

libdd-profiling-ffi/src/profiles/datatypes.rs

@@ -6,8 +6,7 @@ use anyhow::Context;
 use function_name::named;
 use libdd_common_ffi::slice::{AsBytes, ByteSlice, CharSlice, Slice};
 use libdd_common_ffi::{wrap_with_ffi_result, Error, Handle, Timespec, ToInner};
-use libdd_profiling::api;
-use libdd_profiling::api::ManagedStringId;
+use libdd_profiling::api::{self, ManagedStringId};
 use libdd_profiling::internal;
 use std::num::NonZeroI64;
 use std::str::Utf8Error;

libdd-profiling/benches/add_samples.rs

@@ -0,0 +1,161 @@
+// Copyright 2025-Present Datadog, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use criterion::*;
+use libdd_profiling::api2::Location2;
+use libdd_profiling::profiles::datatypes::{Function, FunctionId2, MappingId2};
+use libdd_profiling::{self as profiling, api, api2};
+
+fn make_sample_types() -> Vec<api::ValueType<'static>> {
+    vec![api::ValueType::new("samples", "count")]
+}
+
+fn make_stack_api(frames: &[Frame]) -> (Vec<api::Location<'static>>, Vec<i64>) {
+    // No mappings in Ruby, but the v1 API requires it.
+    let mapping = api::Mapping::default();
+    let mut locations = Vec::with_capacity(frames.len());
+    for frame in frames {
+        locations.push(api::Location {
+            mapping,
+            function: api::Function {
+                name: frame.function_name,
+                filename: frame.file_name,
+                ..Default::default()
+            },
+            line: frame.line_number as i64,
+            ..Default::default()
+        });
+    }
+    let values = vec![1i64];
+    (locations, values)
+}
+
+fn make_stack_api2(frames: &[Frame2]) -> (Vec<Location2>, Vec<i64>) {
+    let mut locations = Vec::with_capacity(frames.len());
+
+    for frame in frames {
+        locations.push(Location2 {
+            mapping: MappingId2::default(),
+            function: frame.function,
+            address: 0,
+            line: frame.line_number as i64,
+        });
+    }
+
+    let values = vec![1i64];
+    (locations, values)
+}
+
+#[derive(Clone, Copy)]
+struct Frame {
+    file_name: &'static str,
+    line_number: u32,
+    function_name: &'static str,
+}
+
+impl Frame {
+    pub const fn new(
+        file_name: &'static str,
+        line_number: u32,
+        function_name: &'static str,
+    ) -> Self {
+        Self {
+            file_name,
+            line_number,
+            function_name,
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Frame2 {
+    function: FunctionId2,
+    line_number: u32,
+}
+
+pub fn bench_add_sample_vs_add2(c: &mut Criterion) {
+    let sample_types = make_sample_types();
+    let dict = profiling::profiles::datatypes::ProfilesDictionary::try_new().unwrap();
+
+    // This is root-to-leaf, instead of leaf-to-root. We'll reverse it below.
+    // Taken from a Ruby app, everything here is source-available.
+    let mut frames = [
+        Frame::new("/usr/local/bundle/gems/logging-2.4.0/lib/logging/diagnostic_context.rb", 474, "create_with_logging_context"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/thread_pool.rb", 155, "spawn_thread"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/server.rb", 245, "run"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/server.rb", 464, "process_client"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/request.rb", 99, "handle_request"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/thread_pool.rb", 378, "with_force_shutdown"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/request.rb", 100, "handle_request"),
+        Frame::new("/usr/local/bundle/gems/puma-6.4.3/lib/puma/configuration.rb", 272, "call"),
+        Frame::new("/usr/local/bundle/gems/railties-7.0.8.7/lib/rails/engine.rb", 530, "call"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/tracing/contrib/rack/middlewares.rb", 474, "call"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/tracing/contrib/rack/trace_proxy_middleware.rb", 17, "call"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/tracing/contrib/rack/middlewares.rb", 70, "call"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/appsec/contrib/rack/request_middleware.rb", 82, "call"),
+        Frame::new("/usr/local/lib/libruby.so.3.3", 0, "catch"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/appsec/contrib/rack/request_middleware.rb", 85, "catch"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/appsec/instrumentation/gateway.rb", 41, "push"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/appsec/instrumentation/gateway.rb", 37, "push"),
+        Frame::new("/usr/local/bundle/gems/datadog-2.18.0/lib/datadog/appsec/instrumentation/gateway/middleware.rb", 18, "call"),
+    ];
+    frames.reverse();
+
+    let strings = dict.strings();
+    let functions = dict.functions();
+
+    let frames2 = frames.map(|f| {
+        let set_id = functions
+            .try_insert(Function {
+                name: strings.try_insert(f.file_name).unwrap(),
+                system_name: Default::default(),
+                file_name: strings.try_insert(f.file_name).unwrap(),
+            })
+            .unwrap();
+        Frame2 {
+            function: FunctionId2::from(set_id),
+            line_number: f.line_number,
+        }
+    });
+    let dict = profiling::profiles::collections::Arc::try_new(dict).unwrap();
+
+    c.bench_function("profile_add_sample_frames_x1000", |b| {
+        b.iter(|| {
+            let mut profile = profiling::internal::Profile::try_new(&sample_types, None).unwrap();
+            let (locations, values) = make_stack_api(frames.as_slice());
+            for _ in 0..1000 {
+                let sample = api::Sample {
+                    locations: locations.clone(),
+                    values: &values,
+                    labels: vec![],
+                };
+                black_box(profile.try_add_sample(sample, None)).unwrap();
+            }
+            black_box(profile.only_for_testing_num_aggregated_samples())
+        })
+    });
+
+    c.bench_function("profile_add_sample2_frames_x1000", |b| {
+        b.iter(|| {
+            let mut profile = profiling::internal::Profile::try_new_with_dictionary(
+                &sample_types,
+                None,
+                dict.try_clone().unwrap(),
+            )
+            .unwrap();
+            let (locations, values) = make_stack_api2(frames2.as_slice());
+            for _ in 0..1000 {
+                // Provide an empty iterator for labels conversion path
+                let labels_iter = std::iter::empty::<anyhow::Result<api2::Label>>();
+                // SAFETY: all ids come from the profile's dictionary.
+                black_box(unsafe {
+                    profile.try_add_sample2(&locations, &values, labels_iter, None)
+                })
+                .unwrap();
+            }
+            black_box(profile.only_for_testing_num_aggregated_samples())
+        })
+    });
+}
+
+criterion_group!(benches, bench_add_sample_vs_add2);

libdd-profiling/benches/main.rs

@@ -3,6 +3,7 @@
 
 use criterion::criterion_main;
 
+mod add_samples;
 mod interning_strings;
 
-criterion_main!(interning_strings::benches);
+criterion_main!(interning_strings::benches, add_samples::benches);

libdd-profiling/src/api2.rs

@@ -0,0 +1,57 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::profiles::datatypes::{FunctionId2, MappingId2, StringId2};
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(C)]
+pub struct Location2 {
+    pub mapping: MappingId2,
+    pub function: FunctionId2,
+
+    /// The instruction address for this location, if available.  It
+    /// should be within [Mapping.memory_start...Mapping.memory_limit]
+    /// for the corresponding mapping. A non-leaf address may be in the
+    /// middle of a call instruction. It is up to display tools to find
+    /// the beginning of the instruction if necessary.
+    pub address: u64,
+    pub line: i64,
+}
+
+#[derive(Copy, Clone, Debug, Default)]
+pub struct Label<'a> {
+    pub key: StringId2,
+
+    /// At most one of `.str` and `.num` should not be empty.
+    pub str: &'a str,
+    pub num: i64,
+
+    /// Should only be present when num is present.
+    /// Specifies the units of num.
+    /// Use arbitrary string (for example, "requests") as a custom count unit.
+    /// If no unit is specified, consumer may apply heuristic to deduce the unit.
+    /// Consumers may also  interpret units like "bytes" and "kilobytes" as memory
+    /// units and units like "seconds" and "nanoseconds" as time units,
+    /// and apply appropriate unit conversions to these.
+    pub num_unit: &'a str,
+}
+
+impl<'a> Label<'a> {
+    pub const fn str(key: StringId2, str: &'a str) -> Label<'a> {
+        Label {
+            key,
+            str,
+            num: 0,
+            num_unit: "",
+        }
+    }
+
+    pub const fn num(key: StringId2, num: i64, num_unit: &'a str) -> Label<'a> {
+        Label {
+            key,
+            str: "",
+            num,
+            num_unit,
+        }
+    }
+}

libdd-profiling/src/internal/function.rs

@@ -6,7 +6,7 @@ use super::*;
 /// Represents a [pprof::Function] with some space-saving changes:
 ///  - The id is not stored on the struct. It's stored in the container that holds the struct.
 ///  - ids for linked objects use 32-bit numbers instead of 64 bit ones.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Hash, PartialOrd, Ord)]
 pub struct Function {
     pub name: StringId,
     pub system_name: StringId,

libdd-profiling/src/internal/owned_types/mod.rs

@@ -33,6 +33,13 @@ pub struct Period {
     pub value: i64,
 }
 
+impl<'a> From<api::Period<'a>> for Period {
+    #[inline]
+    fn from(period: api::Period<'a>) -> Self {
+        Period::from(&period)
+    }
+}
+
 impl<'a> From<&'a api::Period<'a>> for Period {
     #[inline]
     fn from(period: &'a api::Period<'a>) -> Self {