feat: support for log_byte and trace_metric_byte

Author: solnic

lib/sentry/client_report/sender.ex

@@ -6,7 +6,19 @@ defmodule Sentry.ClientReport.Sender do
 
   use GenServer
 
-  alias Sentry.{Client, ClientReport, Config, Envelope, Transaction}
+  alias Sentry.{
+    Client,
+    ClientReport,
+    Config,
+    Envelope,
+    LogBatch,
+    LogEvent,
+    Metric,
+    MetricBatch,
+    Transaction
+  }
+
+  alias Sentry.Telemetry.Category
 
   @send_interval 30_000
 
@@ -39,6 +51,10 @@ defmodule Sentry.ClientReport.Sender do
                | Sentry.CheckIn.t()
                | ClientReport.t()
                | Sentry.Event.t()
+               | LogBatch.t()
+               | LogEvent.t()
+               | Metric.t()
+               | MetricBatch.t()
                | Sentry.Transaction.t()
   def record_discarded_events(reason, event_items, genserver)
       when is_list(event_items) do
@@ -65,6 +81,38 @@ defmodule Sentry.ClientReport.Sender do
     [{Envelope.get_data_category(transaction), 1}, {"span", span_count}]
   end
 
+  defp data_categories(%LogEvent{} = log_event) do
+    [
+      {Category.data_category(:log), 1},
+      {Category.byte_data_category(:log), Envelope.log_event_byte_size(log_event)}
+    ]
+  end
+
+  defp data_categories(%Metric{} = metric) do
+    [
+      {Category.data_category(:metric), 1},
+      {Category.byte_data_category(:metric), Envelope.metric_byte_size(metric)}
+    ]
+  end
+
+  defp data_categories(%LogBatch{log_events: log_events}) do
+    bytes = Enum.reduce(log_events, 0, &(Envelope.log_event_byte_size(&1) + &2))
+
+    [
+      {Category.data_category(:log), length(log_events)},
+      {Category.byte_data_category(:log), bytes}
+    ]
+  end
+
+  defp data_categories(%MetricBatch{metrics: metrics}) do
+    bytes = Enum.reduce(metrics, 0, &(Envelope.metric_byte_size(&1) + &2))
+
+    [
+      {Category.data_category(:metric), length(metrics)},
+      {Category.byte_data_category(:metric), bytes}
+    ]
+  end
+
   defp data_categories(item) do
     [{Envelope.get_data_category(item), 1}]
   end

lib/sentry/envelope.ex

@@ -138,6 +138,46 @@ defmodule Sentry.Envelope do
   def get_data_category(%LogBatch{}), do: "log_item"
   def get_data_category(%MetricBatch{}), do: "trace_metric"
 
+  @doc """
+  Approximates the serialized byte size of a single log event.
+
+  The size is computed by encoding the log event with the same serialization used
+  when the event is placed in an envelope item (see `Sentry.LogEvent.to_map/1`),
+  then measuring the byte size of the resulting JSON. Per the client report spec,
+  a serialized-size approximation like this is acceptable for `log_byte` outcomes.
+
+  Returns `0` if the log event cannot be encoded.
+  """
+  @doc since: "13.4.0"
+  @spec log_event_byte_size(LogEvent.t()) :: non_neg_integer()
+  def log_event_byte_size(%LogEvent{} = log_event) do
+    log_event |> LogEvent.to_map() |> item_byte_size()
+  end
+
+  @doc """
+  Approximates the serialized byte size of a single metric.
+
+  The size is computed by encoding the metric with the same serialization used
+  when the metric is placed in an envelope item (see `Sentry.Metric.to_map/1`),
+  then measuring the byte size of the resulting JSON. Per the client report spec,
+  a serialized-size approximation like this is acceptable for `trace_metric_byte`
+  outcomes.
+
+  Returns `0` if the metric cannot be encoded.
+  """
+  @doc since: "13.4.0"
+  @spec metric_byte_size(Metric.t()) :: non_neg_integer()
+  def metric_byte_size(%Metric{} = metric) do
+    metric |> Metric.to_map() |> item_byte_size()
+  end
+
+  defp item_byte_size(map) do
+    case Sentry.JSON.encode(map, Config.json_library()) do
+      {:ok, encoded} -> byte_size(encoded)
+      {:error, _reason} -> 0
+    end
+  end
+
   @doc """
   Returns the total number of payload items in the envelope.
 

lib/sentry/logger_handler/logs_backend.ex

@@ -37,8 +37,8 @@ defmodule Sentry.LoggerHandler.LogsBackend do
     log_event_struct = LogEvent.from_logger_event(log_event, attributes, parameters)
 
     case TelemetryProcessor.add(log_event_struct) do
-      {:ok, {:rate_limited, data_category}} ->
-        Sentry.ClientReport.Sender.record_discarded_events(:ratelimit_backoff, data_category)
+      {:ok, {:rate_limited, _data_category}} ->
+        Sentry.ClientReport.Sender.record_discarded_events(:ratelimit_backoff, [log_event_struct])
 
       :ok ->
         :ok

lib/sentry/metrics.ex

@@ -135,8 +135,8 @@ defmodule Sentry.Metrics do
       metric = Metric.attach_default_attributes(metric)
 
       case TelemetryProcessor.add(metric) do
-        {:ok, {:rate_limited, data_category}} ->
-          ClientReport.Sender.record_discarded_events(:ratelimit_backoff, data_category)
+        {:ok, {:rate_limited, _data_category}} ->
+          ClientReport.Sender.record_discarded_events(:ratelimit_backoff, [metric])
 
         :ok ->
           :ok

lib/sentry/telemetry/buffer.ex

@@ -24,6 +24,7 @@ defmodule Sentry.Telemetry.Buffer do
 
   alias Sentry.ClientReport
   alias Sentry.Telemetry.Category
+  alias Sentry.{LogEvent, Metric}
 
   @enforce_keys [:category, :capacity, :batch_size]
   defstruct [
@@ -178,12 +179,9 @@ defmodule Sentry.Telemetry.Buffer do
 
   defp offer(%Buffer{size: size, capacity: capacity} = state, item)
        when size >= capacity do
-    {{:value, _dropped}, items} = :queue.out(state.items)
+    {{:value, dropped}, items} = :queue.out(state.items)
 
-    ClientReport.Sender.record_discarded_events(
-      :cache_overflow,
-      Category.data_category(state.category)
-    )
+    record_overflow_discard(state.category, dropped)
 
     %{state | items: :queue.in(item, items)}
   end
@@ -192,6 +190,24 @@ defmodule Sentry.Telemetry.Buffer do
     %{state | items: :queue.in(item, state.items), size: state.size + 1}
   end
 
+  # Log and metric structs go through the item-based recorder for their byte
+  # outcome; other categories fall back to the category string because the
+  # buffer is generic and the item-based recorder raises on unknown items.
+  defp record_overflow_discard(:log, %LogEvent{} = dropped) do
+    ClientReport.Sender.record_discarded_events(:cache_overflow, [dropped])
+  end
+
+  defp record_overflow_discard(:metric, %Metric{} = dropped) do
+    ClientReport.Sender.record_discarded_events(:cache_overflow, [dropped])
+  end
+
+  defp record_overflow_discard(category, _dropped) do
+    ClientReport.Sender.record_discarded_events(
+      :cache_overflow,
+      Category.data_category(category)
+    )
+  end
+
   defp poll_batch(state, count), do: poll_batch(state, count, [])
   defp poll_batch(state, 0, acc), do: {Enum.reverse(acc), state}
   defp poll_batch(%{size: 0} = state, _count, acc), do: {Enum.reverse(acc), state}

lib/sentry/telemetry/category.ex

@@ -186,4 +186,75 @@ defmodule Sentry.Telemetry.Category do
   def data_category(:transaction), do: "transaction"
   def data_category(:log), do: "log_item"
   def data_category(:metric), do: "trace_metric"
+
+  @doc """
+  Returns the byte-based Sentry data category string for a given telemetry category.
+
+  Some data categories have a companion "byte" category used to report the total
+  serialized size of dropped items in client reports and to honor byte-based rate
+  limits. Only `:log` and `:metric` currently have such companion categories.
+
+  These strings are used in client reports and rate limiting alongside the
+  count-based category returned by `data_category/1`.
+
+  ## Examples
+
+      iex> Sentry.Telemetry.Category.byte_data_category(:log)
+      "log_byte"
+
+      iex> Sentry.Telemetry.Category.byte_data_category(:metric)
+      "trace_metric_byte"
+
+  """
+  @spec byte_data_category(:log | :metric) :: String.t()
+  def byte_data_category(:log), do: "log_byte"
+  def byte_data_category(:metric), do: "trace_metric_byte"
+
+  @doc """
+  Returns every rate-limit data category that gates the given count-based data
+  category, including any companion byte category.
+
+  Logs and metrics have a companion byte category (`log_byte` /
+  `trace_metric_byte`) that Sentry can rate-limit independently, so an active
+  limit on either the count or the byte category must suppress sending. All
+  other categories gate on themselves only.
+
+  These strings are matched against the limits stored from the
+  `X-Sentry-Rate-Limits` response header.
+
+  ## Examples
+
+      iex> Sentry.Telemetry.Category.rate_limit_categories("log_item")
+      ["log_item", "log_byte"]
+
+      iex> Sentry.Telemetry.Category.rate_limit_categories("trace_metric")
+      ["trace_metric", "trace_metric_byte"]
+
+      iex> Sentry.Telemetry.Category.rate_limit_categories("error")
+      ["error"]
+
+  """
+  @spec rate_limit_categories(String.t()) :: [String.t(), ...]
+  def rate_limit_categories("log_item"), do: ["log_item", "log_byte"]
+  def rate_limit_categories("trace_metric"), do: ["trace_metric", "trace_metric_byte"]
+  def rate_limit_categories(category) when is_binary(category), do: [category]
+
+  @doc """
+  Returns all Sentry data category strings recognized by the SDK.
+
+  This includes the count-based categories returned by `data_category/1` as well
+  as the byte-based categories returned by `byte_data_category/1`. Any category
+  outside this set is unknown to the SDK.
+
+  ## Examples
+
+      iex> categories = Sentry.Telemetry.Category.data_categories()
+      iex> "log_byte" in categories and "trace_metric_byte" in categories
+      true
+
+  """
+  @spec data_categories() :: [String.t(), ...]
+  def data_categories do
+    Enum.map(@categories, &data_category/1) ++ ["log_byte", "trace_metric_byte"]
+  end
 end

lib/sentry/telemetry/scheduler.ex

@@ -531,9 +531,9 @@ defmodule Sentry.Telemetry.Scheduler do
       items == [] ->
         :ok
 
-      # Transactions carry spans, so pass the actual structs through the
-      # list-based recorder to also record the discarded "span" outcomes.
-      category == :transaction ->
+      # These categories expand to paired outcomes (span / log_byte /
+      # trace_metric_byte) that only the struct-based recorder emits.
+      category in [:transaction, :log, :metric] ->
         ClientReport.Sender.record_discarded_events(:ratelimit_backoff, items)
 
       true ->
@@ -551,8 +551,10 @@ defmodule Sentry.Telemetry.Scheduler do
   defp category_rate_limited?(%{on_envelope: cb}, _category) when is_function(cb, 1), do: false
 
   defp category_rate_limited?(_state, category) do
-    data_category = Category.data_category(category)
-    RateLimiter.rate_limited?(data_category)
+    category
+    |> Category.data_category()
+    |> Category.rate_limit_categories()
+    |> Enum.any?(&RateLimiter.rate_limited?/1)
   end
 
   defp default_weights do

lib/sentry/telemetry_processor.ex

@@ -253,7 +253,7 @@ defmodule Sentry.TelemetryProcessor do
   defp add_to_buffer(processor, category, item) when is_atom(processor) do
     data_category = Category.data_category(category)
 
-    if RateLimiter.rate_limited?(data_category) do
+    if data_category_rate_limited?(data_category) do
       {:ok, {:rate_limited, data_category}}
     else
       Buffer.add(buffer_name(processor, category), item)
@@ -265,7 +265,7 @@ defmodule Sentry.TelemetryProcessor do
   defp add_to_buffer(processor, category, item) do
     data_category = Category.data_category(category)
 
-    if RateLimiter.rate_limited?(data_category) do
+    if data_category_rate_limited?(data_category) do
       {:ok, {:rate_limited, data_category}}
     else
       buffer = get_buffer(processor, category)
@@ -276,6 +276,12 @@ defmodule Sentry.TelemetryProcessor do
     end
   end
 
+  defp data_category_rate_limited?(data_category) do
+    data_category
+    |> Category.rate_limit_categories()
+    |> Enum.any?(&RateLimiter.rate_limited?/1)
+  end
+
   defp safe_get_buffer(processor, category) when is_atom(processor) do
     try do
       {:ok,

lib/sentry/transport.ex

@@ -4,6 +4,7 @@ defmodule Sentry.Transport do
   # This module is exclusively responsible for encoding and POSTing envelopes to Sentry.
 
   alias Sentry.{ClientError, ClientReport, Config, Envelope, LoggerUtils}
+  alias Sentry.Telemetry.Category
   alias Sentry.Transport.RateLimiter
 
   @default_retries [1000, 2000, 4000, 8000]
@@ -86,8 +87,10 @@ defmodule Sentry.Transport do
   defp check_rate_limited(envelope_items) do
     rate_limited? =
       Enum.any?(envelope_items, fn item ->
-        category = Envelope.get_data_category(item)
-        RateLimiter.rate_limited?(category)
+        item
+        |> Envelope.get_data_category()
+        |> Category.rate_limit_categories()
+        |> Enum.any?(&RateLimiter.rate_limited?/1)
       end)
 
     if rate_limited?, do: {:error, :rate_limited}, else: :ok

test/envelope_test.exs

@@ -336,4 +336,45 @@ defmodule Sentry.EnvelopeTest do
       assert Envelope.get_data_category(metric_batch) == "trace_metric"
     end
   end
+
+  describe "log_event_byte_size/1" do
+    test "approximates the serialized size of a single log event" do
+      log_event = %LogEvent{
+        timestamp: 1_588_601_261.535_386,
+        level: :info,
+        body: "something happened"
+      }
+
+      assert Envelope.log_event_byte_size(log_event) > 0
+    end
+
+    test "grows with the size of the log body" do
+      base = %LogEvent{timestamp: 1_588_601_261.535_386, level: :info, body: "hi"}
+      large = %LogEvent{base | body: String.duplicate("x", 1_000)}
+
+      assert Envelope.log_event_byte_size(large) >
+               Envelope.log_event_byte_size(base) + 900
+    end
+  end
+
+  describe "metric_byte_size/1" do
+    test "approximates the serialized size of a single metric" do
+      metric = %Metric{
+        type: :counter,
+        name: "test.counter",
+        value: 1,
+        timestamp: 1_588_601_261.535_386
+      }
+
+      assert Envelope.metric_byte_size(metric) > 0
+    end
+
+    test "grows with the size of the metric name" do
+      base = %Metric{type: :counter, name: "m", value: 1, timestamp: 1_588_601_261.535_386}
+      large = %Metric{base | name: String.duplicate("n", 1_000)}
+
+      assert Envelope.metric_byte_size(large) >
+               Envelope.metric_byte_size(base) + 900
+    end
+  end
 end