From c207781c1b4be08e4cad4a6118f85dd31fe8a938 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 14 Jul 2025 14:25:38 -0400 Subject: [PATCH 1/7] Added logic to unwind and surface the potential root cause when an error occurs during the agent's premain initialization phase. --- .../BootstrapInitializationTelemetry.java | 19 ++++++++++++++++++- ...ootstrapInitializationTelemetryTest.groovy | 12 ++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java index 6e09fa0f30a..549429fefdb 100644 --- a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java +++ b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java @@ -6,6 +6,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import java.util.Stack; /** Thread safe telemetry class used to relay information about tracer activation. */ public abstract class BootstrapInitializationTelemetry { @@ -115,7 +116,23 @@ public void onAbort(String reasonCode) { @Override public void onError(Throwable t) { - onPoint("library_entrypoint.error", "error_type:" + t.getClass().getName()); + Stack causes = new Stack<>(); + + Throwable cause = t.getCause(); + if (cause != null) { + while (cause != null) { + causes.push(cause.getClass().getName()); + cause = cause.getCause(); + } + } + causes.push(t.getClass().getName()); + + // Limit the number of causes to avoid overpopulating the JSON payload. + int cnt = Math.min(5, causes.size()); + while (cnt > 0) { + onPoint("library_entrypoint.error", "error_type:" + causes.pop()); + cnt--; + } } @Override diff --git a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy index da0dafa6d4a..f2fc4b80d6b 100644 --- a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy +++ b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy @@ -76,6 +76,18 @@ class BootstrapInitializationTelemetryTest extends Specification { !capture.json().contains("library_entrypoint.complete") } + def "unwind root cause"() { + when: + initTelemetry.initMetaInfo("runtime_name", "java") + initTelemetry.initMetaInfo("runtime_version", "1.8.0_382") + + initTelemetry.onError(new Exception("top cause", new NullPointerException("root cause"))) + initTelemetry.finish() + + then: + capture.json() == '{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382"},"points":[{"name":"library_entrypoint.error","tags":["error_type:java.lang.Exception"]},{"name":"library_entrypoint.error","tags":["error_type:java.lang.NullPointerException"]},{"name":"library_entrypoint.complete"}]}' + } + static class Capture implements BootstrapInitializationTelemetry.JsonSender { String json From 796cbc21b12db389b3b0dbe1a95c00feed099642 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 15 Jul 2025 12:59:04 -0400 Subject: [PATCH 2/7] Refactored points to support many tags. --- .../environment/EnvironmentVariables.java | 17 +++++++ .../environment/EnvironmentVariablesTest.java | 4 ++ .../BootstrapInitializationTelemetry.java | 44 ++++++++++++------- ...ootstrapInitializationTelemetryTest.groovy | 4 +- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java b/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java index 96f60bd6c4e..c1b6a4e4c07 100644 --- a/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java +++ b/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java @@ -38,4 +38,21 @@ public static String getOrDefault(@Nonnull String name, String defaultValue) { return defaultValue; } } + + /** + * Gets an environment variable value, or default value if missing or can't be retrieved. + * + * @param name The environment variable name. + * @param defaultValue The default value to return if the environment variable is missing or can't + * be retrieved. + * @return The environment variable value, {@code defaultValue} if missing or can't be retrieved. + */ + public static int getOrDefault(@Nonnull String name, int defaultValue) { + try { + String value = System.getenv(name); + return value == null ? defaultValue : Integer.parseInt(value); + } catch (SecurityException | NumberFormatException e) { + return defaultValue; + } + } } diff --git a/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java b/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java index 80cb4f601b6..5febb35ca84 100644 --- a/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java +++ b/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java @@ -23,5 +23,9 @@ void testGetOrDefault() { assertNull(EnvironmentVariables.getOrDefault(MISSING_ENV_VAR, null)); assertThrows(NullPointerException.class, () -> EnvironmentVariables.getOrDefault(null, "")); + + assertEquals(42, EnvironmentVariables.getOrDefault(EXISTING_ENV_VAR, 42)); + assertEquals(42, EnvironmentVariables.getOrDefault(MISSING_ENV_VAR, 42)); + assertThrows(NullPointerException.class, () -> EnvironmentVariables.getOrDefault(null, 42)); } } diff --git a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java index 549429fefdb..f1a05f40823 100644 --- a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java +++ b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java @@ -1,12 +1,15 @@ package datadog.trace.bootstrap; import datadog.json.JsonWriter; +import datadog.trace.bootstrap.environment.EnvironmentVariables; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.Closeable; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Stack; +import java.util.Map; /** Thread safe telemetry class used to relay information about tracer activation. */ public abstract class BootstrapInitializationTelemetry { @@ -89,7 +92,7 @@ public static final class JsonBased extends BootstrapInitializationTelemetry { private final JsonSender sender; private final List meta; - private final List points; + private final Map> points; // one way false to true private volatile boolean incomplete = false; @@ -97,7 +100,7 @@ public static final class JsonBased extends BootstrapInitializationTelemetry { JsonBased(JsonSender sender) { this.sender = sender; this.meta = new ArrayList<>(); - this.points = new ArrayList<>(); + this.points = new LinkedHashMap<>(); } @Override @@ -116,23 +119,23 @@ public void onAbort(String reasonCode) { @Override public void onError(Throwable t) { - Stack causes = new Stack<>(); + List causes = new ArrayList<>(); Throwable cause = t.getCause(); if (cause != null) { while (cause != null) { - causes.push(cause.getClass().getName()); + causes.add("error_type:" + cause.getClass().getName()); cause = cause.getCause(); } } - causes.push(t.getClass().getName()); + causes.add("error_type:" + t.getClass().getName()); - // Limit the number of causes to avoid overpopulating the JSON payload. - int cnt = Math.min(5, causes.size()); - while (cnt > 0) { - onPoint("library_entrypoint.error", "error_type:" + causes.pop()); - cnt--; - } + // Limit the number of tags to avoid overpopulating the JSON payload. + int maxTags = EnvironmentVariables.getOrDefault("DD_TELEMETRY_FORWARDER_MAX_TAGS", 5); + int sz = causes.size(); + int cnt = Math.min(maxTags, sz); + + onPoint("library_entrypoint.error", causes.subList(sz - cnt, sz)); } @Override @@ -147,9 +150,12 @@ public void onError(String reasonCode) { } private void onPoint(String name, String tag) { + onPoint(name, Collections.singletonList(tag)); + } + + private void onPoint(String name, List tags) { synchronized (this.points) { - this.points.add(name); - this.points.add(tag); + this.points.put(name, tags); } } @@ -173,10 +179,14 @@ public void finish() { writer.name("points").beginArray(); synchronized (this.points) { - for (int i = 0; i + 1 < this.points.size(); i = i + 2) { + for (Map.Entry> entry : points.entrySet()) { writer.beginObject(); - writer.name("name").value(this.points.get(i)); - writer.name("tags").beginArray().value(this.points.get(i + 1)).endArray(); + writer.name("name").value(entry.getKey()); + writer.name("tags").beginArray(); + for (String tag : entry.getValue()) { + writer.value(tag); + } + writer.endArray(); writer.endObject(); } this.points.clear(); diff --git a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy index f2fc4b80d6b..0e239cfc464 100644 --- a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy +++ b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy @@ -81,11 +81,11 @@ class BootstrapInitializationTelemetryTest extends Specification { initTelemetry.initMetaInfo("runtime_name", "java") initTelemetry.initMetaInfo("runtime_version", "1.8.0_382") - initTelemetry.onError(new Exception("top cause", new NullPointerException("root cause"))) + initTelemetry.onError(new Exception("top cause", new FileNotFoundException("root cause"))) initTelemetry.finish() then: - capture.json() == '{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382"},"points":[{"name":"library_entrypoint.error","tags":["error_type:java.lang.Exception"]},{"name":"library_entrypoint.error","tags":["error_type:java.lang.NullPointerException"]},{"name":"library_entrypoint.complete"}]}' + capture.json() == '{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382"},"points":[{"name":"library_entrypoint.error","tags":["error_type:java.io.FileNotFoundException","error_type:java.lang.Exception"]},{"name":"library_entrypoint.complete"}]}' } static class Capture implements BootstrapInitializationTelemetry.JsonSender { From 47eba9fd4e585c5ca9512e64853c1fb7c9fdc4bf Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Tue, 15 Jul 2025 14:31:02 -0400 Subject: [PATCH 3/7] Fixed failed test coverage. --- components/environment/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/components/environment/build.gradle.kts b/components/environment/build.gradle.kts index c8c280e6b26..70d96b1f0c2 100644 --- a/components/environment/build.gradle.kts +++ b/components/environment/build.gradle.kts @@ -18,6 +18,7 @@ tasks.shadowJar { * Configure test coverage. */ extra.set("minimumInstructionCoverage", 0.7) +extra.set("minimumBranchCoverage", 0.7) val excludedClassesCoverage by extra { listOf( "datadog.environment.JavaVirtualMachine", // depends on OS and JVM vendor From 4ffa71ce2c368e4fe9db76d6ba32b761ee52c92f Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 16 Jul 2025 09:11:47 -0400 Subject: [PATCH 4/7] Updates based on PR review comments. --- components/environment/build.gradle.kts | 1 - .../environment/EnvironmentVariables.java | 17 ----------- .../BootstrapInitializationTelemetry.java | 30 +++++++++++++------ 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/components/environment/build.gradle.kts b/components/environment/build.gradle.kts index 70d96b1f0c2..c8c280e6b26 100644 --- a/components/environment/build.gradle.kts +++ b/components/environment/build.gradle.kts @@ -18,7 +18,6 @@ tasks.shadowJar { * Configure test coverage. */ extra.set("minimumInstructionCoverage", 0.7) -extra.set("minimumBranchCoverage", 0.7) val excludedClassesCoverage by extra { listOf( "datadog.environment.JavaVirtualMachine", // depends on OS and JVM vendor diff --git a/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java b/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java index c1b6a4e4c07..96f60bd6c4e 100644 --- a/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java +++ b/components/environment/src/main/java/datadog/environment/EnvironmentVariables.java @@ -38,21 +38,4 @@ public static String getOrDefault(@Nonnull String name, String defaultValue) { return defaultValue; } } - - /** - * Gets an environment variable value, or default value if missing or can't be retrieved. - * - * @param name The environment variable name. - * @param defaultValue The default value to return if the environment variable is missing or can't - * be retrieved. - * @return The environment variable value, {@code defaultValue} if missing or can't be retrieved. - */ - public static int getOrDefault(@Nonnull String name, int defaultValue) { - try { - String value = System.getenv(name); - return value == null ? defaultValue : Integer.parseInt(value); - } catch (SecurityException | NumberFormatException e) { - return defaultValue; - } - } } diff --git a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java index f1a05f40823..6a666e9bc3d 100644 --- a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java +++ b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java @@ -122,20 +122,32 @@ public void onError(Throwable t) { List causes = new ArrayList<>(); Throwable cause = t.getCause(); - if (cause != null) { - while (cause != null) { - causes.add("error_type:" + cause.getClass().getName()); - cause = cause.getCause(); - } + while (cause != null) { + causes.add("error_type:" + cause.getClass().getName()); + cause = cause.getCause(); } causes.add("error_type:" + t.getClass().getName()); // Limit the number of tags to avoid overpopulating the JSON payload. - int maxTags = EnvironmentVariables.getOrDefault("DD_TELEMETRY_FORWARDER_MAX_TAGS", 5); - int sz = causes.size(); - int cnt = Math.min(maxTags, sz); + int maxTags = maxTags(5); + int numCauses = causes.size(); + if (numCauses > maxTags) { + causes = causes.subList(numCauses - maxTags, numCauses); + } + + onPoint("library_entrypoint.error", causes); + } - onPoint("library_entrypoint.error", causes.subList(sz - cnt, sz)); + private int maxTags(int defaultValue) { + try { + String s = + EnvironmentVariables.getOrDefault( + "DD_TELEMETRY_FORWARDER_MAX_TAGS", String.valueOf(defaultValue)); + + return Integer.parseInt(s); + } catch (Exception e) { + return defaultValue; + } } @Override From 102e18509b152a8ea8a0504c96b0ab3667f525bf Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 16 Jul 2025 09:15:40 -0400 Subject: [PATCH 5/7] Updates based on PR review comments. --- .../java/datadog/environment/EnvironmentVariablesTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java b/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java index 5febb35ca84..80cb4f601b6 100644 --- a/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java +++ b/components/environment/src/test/java/datadog/environment/EnvironmentVariablesTest.java @@ -23,9 +23,5 @@ void testGetOrDefault() { assertNull(EnvironmentVariables.getOrDefault(MISSING_ENV_VAR, null)); assertThrows(NullPointerException.class, () -> EnvironmentVariables.getOrDefault(null, "")); - - assertEquals(42, EnvironmentVariables.getOrDefault(EXISTING_ENV_VAR, 42)); - assertEquals(42, EnvironmentVariables.getOrDefault(MISSING_ENV_VAR, 42)); - assertThrows(NullPointerException.class, () -> EnvironmentVariables.getOrDefault(null, 42)); } } From d0dc9bdf06720f1b0313c390b12338f4726ff130 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Wed, 16 Jul 2025 11:24:11 -0400 Subject: [PATCH 6/7] Updates based on PR review comments. --- .../BootstrapInitializationTelemetry.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java index 6a666e9bc3d..9c2a3ba407e 100644 --- a/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java +++ b/dd-java-agent/src/main/java/datadog/trace/bootstrap/BootstrapInitializationTelemetry.java @@ -13,6 +13,8 @@ /** Thread safe telemetry class used to relay information about tracer activation. */ public abstract class BootstrapInitializationTelemetry { + private static final int DEFAULT_MAX_TAGS = 5; + /** Returns a singleton no op instance of initialization telemetry */ public static BootstrapInitializationTelemetry noOpInstance() { return NoOp.INSTANCE; @@ -129,7 +131,7 @@ public void onError(Throwable t) { causes.add("error_type:" + t.getClass().getName()); // Limit the number of tags to avoid overpopulating the JSON payload. - int maxTags = maxTags(5); + int maxTags = maxTags(); int numCauses = causes.size(); if (numCauses > maxTags) { causes = causes.subList(numCauses - maxTags, numCauses); @@ -138,16 +140,18 @@ public void onError(Throwable t) { onPoint("library_entrypoint.error", causes); } - private int maxTags(int defaultValue) { - try { - String s = - EnvironmentVariables.getOrDefault( - "DD_TELEMETRY_FORWARDER_MAX_TAGS", String.valueOf(defaultValue)); + private int maxTags() { + String maxTags = EnvironmentVariables.get("DD_TELEMETRY_FORWARDER_MAX_TAGS"); - return Integer.parseInt(s); - } catch (Exception e) { - return defaultValue; + if (maxTags != null) { + try { + return Integer.parseInt(maxTags); + } catch (Throwable ignore) { + // Ignore and return default value. + } } + + return DEFAULT_MAX_TAGS; } @Override From bbb59b8726eb937f86adf62089705eb267c5ec71 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Thu, 17 Jul 2025 12:06:52 -0400 Subject: [PATCH 7/7] Fixed test after merge with master. --- .../BootstrapInitializationTelemetryTest.groovy | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy index 8ba0f14ec70..ac62d4c7ff1 100644 --- a/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy +++ b/dd-java-agent/src/test/groovy/datadog/trace/bootstrap/BootstrapInitializationTelemetryTest.groovy @@ -1,5 +1,6 @@ package datadog.trace.bootstrap +import groovy.json.JsonBuilder import spock.lang.Specification import static java.nio.charset.StandardCharsets.UTF_8 @@ -108,18 +109,18 @@ class BootstrapInitializationTelemetryTest extends Specification { def "unwind root cause"() { when: - initTelemetry.initMetaInfo("runtime_name", "java") - initTelemetry.initMetaInfo("runtime_version", "1.8.0_382") - initTelemetry.onError(new Exception("top cause", new FileNotFoundException("root cause"))) initTelemetry.finish() then: - capture.json() == '{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382"},"points":[{"name":"library_entrypoint.error","tags":["error_type:java.io.FileNotFoundException","error_type:java.lang.Exception"]},{"name":"library_entrypoint.complete"}]}' + capture.json() == json("error", "internal_error", "top cause", [ + [name: "library_entrypoint.error", tags: ["error_type:java.io.FileNotFoundException", "error_type:java.lang.Exception"]], + [name: "library_entrypoint.complete"] + ]) } - private String json(String result, String resultClass, String resultReason, List points) { - return """{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382","result":"${result}","result_class":"${resultClass}","result_reason":"${resultReason}"},"points":${new groovy.json.JsonBuilder(points)}}""" + private static String json(String result, String resultClass, String resultReason, List points) { + return """{"metadata":{"runtime_name":"java","runtime_version":"1.8.0_382","result":"${result}","result_class":"${resultClass}","result_reason":"${resultReason}"},"points":${new JsonBuilder(points)}}""" } static class Capture implements BootstrapInitializationTelemetry.JsonSender {