feat(datadog_agent): support v3 series metrics intake

changelog.d/25790_datadog_agent_v3_series.feature.md

@@ -0,0 +1,3 @@
+The `datadog_agent` source now accepts the v3 series metrics intake format used by default by Datadog Agent 7.81.0 and newer.
+
+authors: arfa79

lib/datadog-proto/build.rs

@@ -9,6 +9,7 @@ use std::{
 fn main() -> Result<()> {
     for path in [
         "proto/datadog/agentpayload.proto",
+        "proto/datadog/metrics/intake_v3.proto",
         "proto/datadog/trace/agent_payload.proto",
         "proto/datadog/trace/tracer_payload.proto",
         "proto/datadog/trace/span.proto",

lib/vector-core/proto/event.proto

@@ -79,6 +79,7 @@ message DatadogOriginMetadata {
   optional uint32 origin_product = 1;
   optional uint32 origin_category = 2;
   optional uint32 origin_service = 3;
+  optional int32 metric_type = 4;
 }
 
 message Secrets {
@@ -106,6 +107,7 @@ message Metadata {
   Secrets secrets = 6;
   bytes source_event_id = 7;
   optional TraceLayout trace_layout = 8;
+  optional string datadog_metric_unit = 9;
 }
 
 message Metric {

lib/vector-core/src/event/metadata.rs

@@ -116,6 +116,10 @@ pub(super) struct Inner {
     #[serde(default)]
     pub(crate) datadog_origin_metadata: Option<DatadogMetricOriginMetadata>,
 
+    /// Datadog metric unit received from an upstream Agent payload.
+    #[serde(default)]
+    pub(crate) datadog_metric_unit: Option<String>,
+
     /// Which legacy trace key layout produced this event. `None` means unmarked.
     /// Meaningful only on traces; always `None` for logs and metrics.
     #[serde(default)]
@@ -129,6 +133,8 @@ pub(super) struct Inner {
 /// Metric Origin metadata for submission to Datadog.
 #[derive(Clone, Default, Debug, Deserialize, PartialEq, Serialize)]
 pub struct DatadogMetricOriginMetadata {
+    /// Datadog origin metric type.
+    metric_type: Option<i32>,
     /// `OriginProduct`
     product: Option<u32>,
     /// `OriginCategory`
@@ -146,12 +152,25 @@ impl DatadogMetricOriginMetadata {
     #[must_use]
     pub fn new(product: Option<u32>, category: Option<u32>, service: Option<u32>) -> Self {
         Self {
+            metric_type: None,
             product,
             category,
             service,
         }
     }
 
+    /// Returns the Datadog origin metric type.
+    pub fn metric_type(&self) -> Option<i32> {
+        self.metric_type
+    }
+
+    /// Sets the Datadog origin metric type.
+    #[must_use]
+    pub fn with_metric_type(mut self, metric_type: Option<i32>) -> Self {
+        self.metric_type = metric_type;
+        self
+    }
+
     /// Returns a reference to the `OriginProduct`.
     pub fn product(&self) -> Option<u32> {
         self.product
@@ -304,6 +323,16 @@ impl EventMetadata {
         self.inner.datadog_origin_metadata.as_ref()
     }
 
+    /// Returns the Datadog metric unit.
+    pub fn datadog_metric_unit(&self) -> Option<&str> {
+        self.inner.datadog_metric_unit.as_deref()
+    }
+
+    /// Sets the Datadog metric unit.
+    pub fn set_datadog_metric_unit(&mut self, unit: String) {
+        self.get_mut().datadog_metric_unit = Some(unit);
+    }
+
     /// Returns a reference to the event id.
     pub fn source_event_id(&self) -> Option<Uuid> {
         self.inner.source_event_id
@@ -333,6 +362,7 @@ impl Default for Inner {
             upstream_id: None,
             dropped_fields: ObjectMap::new(),
             datadog_origin_metadata: None,
+            datadog_metric_unit: None,
             trace_layout: None,
             source_event_id: Some(Uuid::new_v4()),
         }
@@ -415,6 +445,13 @@ impl EventMetadata {
         self
     }
 
+    /// Sets the Datadog metric unit.
+    #[must_use]
+    pub fn with_datadog_metric_unit(mut self, unit: String) -> Self {
+        self.get_mut().datadog_metric_unit = Some(unit);
+        self
+    }
+
     /// Replaces the existing `source_event_id` with the given one.
     #[must_use]
     pub fn with_source_event_id(mut self, source_event_id: Option<Uuid>) -> Self {
@@ -431,6 +468,12 @@ impl EventMetadata {
         let other = other.into_owned();
         inner.finalizers.merge(other.finalizers);
         inner.secrets.merge(other.secrets);
+        if inner.datadog_origin_metadata.is_none() {
+            inner.datadog_origin_metadata = other.datadog_origin_metadata;
+        }
+        if inner.datadog_metric_unit.is_none() {
+            inner.datadog_metric_unit = other.datadog_metric_unit;
+        }
 
         // Update `source_event_id` if necessary.
         if inner.source_event_id.is_none() {

lib/vector-core/src/event/proto.rs

@@ -595,6 +595,7 @@ impl From<Secrets> for super::metadata::Secrets {
 impl From<super::DatadogMetricOriginMetadata> for DatadogOriginMetadata {
     fn from(value: super::DatadogMetricOriginMetadata) -> Self {
         Self {
+            metric_type: value.metric_type(),
             origin_product: value.product(),
             origin_category: value.category(),
             origin_service: value.service(),
@@ -609,6 +610,7 @@ impl From<DatadogOriginMetadata> for super::DatadogMetricOriginMetadata {
             value.origin_category,
             value.origin_service,
         )
+        .with_metric_type(value.metric_type)
     }
 }
 
@@ -636,6 +638,7 @@ impl From<EventMetadata> for Metadata {
             source_type,
             upstream_id,
             datadog_origin_metadata,
+            datadog_metric_unit,
             source_event_id,
             trace_layout,
             ..
@@ -651,6 +654,7 @@ impl From<EventMetadata> for Metadata {
             upstream_id: upstream_id.map(|id| id.as_ref().clone()).map(Into::into),
             secrets,
             source_event_id: source_event_id.map_or(vec![], std::convert::Into::into),
+            datadog_metric_unit,
             trace_layout: encode_trace_layout(trace_layout),
         }
     }
@@ -667,6 +671,7 @@ impl TryFrom<Metadata> for EventMetadata {
             upstream_id,
             secrets,
             datadog_origin_metadata,
+            datadog_metric_unit,
             source_event_id,
             trace_layout,
         } = value;
@@ -707,6 +712,7 @@ impl TryFrom<Metadata> for EventMetadata {
                 schema_definition: default_schema_definition(),
                 dropped_fields: ObjectMap::new(),
                 datadog_origin_metadata,
+                datadog_metric_unit,
                 trace_layout: decode_trace_layout(trace_layout),
                 source_event_id,
             }),

lib/vector-core/src/event/test/serialization.rs

@@ -109,6 +109,28 @@ fn disk_buffer_preserves_unrecognized_trace_layout() {
     );
 }
 
+#[test]
+fn disk_buffer_preserves_datadog_metric_unit() {
+    let mut metric = Metric::new(
+        "test",
+        MetricKind::Absolute,
+        MetricValue::Gauge { value: 1.0 },
+    );
+    metric
+        .metadata_mut()
+        .set_datadog_metric_unit("byte".to_owned());
+    let expected = EventArray::from(Event::Metric(metric));
+
+    let mut buffer = BytesMut::with_capacity(64);
+    encode_value(expected, &mut buffer);
+    let actual = decode_value::<EventArray, _>(buffer);
+
+    let EventArray::Metrics(metrics) = actual else {
+        panic!("expected a metrics array");
+    };
+    assert_eq!(metrics[0].metadata().datadog_metric_unit(), Some("byte"));
+}
+
 #[test]
 fn serialization() {
     let mut event = LogEvent::from("raw log line");

src/common/datadog.rs

@@ -64,6 +64,9 @@ pub struct DatadogSeriesMetric {
     /// device
     #[serde(skip_serializing_if = "Option::is_none")]
     pub device: Option<String>,
+    /// unit
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub unit: Option<String>,
     /// metadata
     #[serde(skip_serializing_if = "Option::is_none")]
     pub metadata: Option<DatadogSeriesMetricMetadata>,

src/sinks/datadog/metrics/encoder.rs

@@ -768,8 +768,10 @@ fn series_to_proto_message(
         tags,
         points,
         r#type: metric_type.into(),
-        // unit is omitted
-        unit: "".to_string(),
+        unit: event_metadata
+            .datadog_metric_unit()
+            .unwrap_or_default()
+            .to_string(),
         source_type_name,
         interval: maybe_interval.unwrap_or(0) as i64,
         metadata,
@@ -879,11 +881,14 @@ pub(super) fn generate_origin_metadata(
     //     - `log_to_metric` transform set the OriginService in the EventMetadata when it creates
     //        the new metric.
     if let Some(pass_through) = maybe_pass_through {
-        Some(DatadogMetricOriginMetadata::new(
-            pass_through.product().or(Some(origin_product_value)),
-            pass_through.category().or(Some(ORIGIN_CATEGORY_VALUE)),
-            pass_through.service().or(Some(no_value)),
-        ))
+        Some(
+            DatadogMetricOriginMetadata::new(
+                pass_through.product().or(Some(origin_product_value)),
+                pass_through.category().or(Some(ORIGIN_CATEGORY_VALUE)),
+                pass_through.service().or(Some(no_value)),
+            )
+            .with_metric_type(pass_through.metric_type()),
+        )
 
     // No metadata has been set upstream
     } else {
@@ -979,6 +984,7 @@ fn generate_series_metrics(
         host,
         source_type_name,
         device,
+        unit: event_metadata.datadog_metric_unit().map(ToOwned::to_owned),
         metadata,
     }])
 }
@@ -1579,9 +1585,12 @@ mod tests {
         let category = 11;
         let service = 9;
 
-        let event_metadata = EventMetadata::default().with_origin_metadata(
-            DatadogMetricOriginMetadata::new(Some(product), Some(category), Some(service)),
-        );
+        let event_metadata = EventMetadata::default()
+            .with_origin_metadata(
+                DatadogMetricOriginMetadata::new(Some(product), Some(category), Some(service))
+                    .with_metric_type(Some(9)),
+            )
+            .with_datadog_metric_unit("byte".to_string());
         let counter = get_simple_counter_with_metadata(event_metadata);
 
         // series v1
@@ -1603,6 +1612,8 @@ mod tests {
             assert_eq!(generated_origin.product().unwrap(), product);
             assert_eq!(generated_origin.category().unwrap(), category);
             assert_eq!(generated_origin.service().unwrap(), service);
+            assert_eq!(generated_origin.metric_type(), Some(9));
+            assert_eq!(actual.unit.as_deref(), Some("byte"));
         }
         // series v2
         {
@@ -1618,6 +1629,7 @@ mod tests {
             assert_eq!(generated_origin.origin_product, product);
             assert_eq!(generated_origin.origin_category, category);
             assert_eq!(generated_origin.origin_service, service);
+            assert_eq!(series_proto.unit, "byte");
         }
     }