Upgrade to OTEL 0.13.0 (hypertrace#185)

* Upgrade to OTEL 0.13.0

Signed-off-by: Pavol Loffay <[email protected]>

* move opa to agent classloader

Signed-off-by: Pavol Loffay <[email protected]>

Author: pavolloffay

CONTRIBUTING.md

@@ -0,0 +1,14 @@
+# Contributing
+
+## Upgrade OpenTelemetry auto instrumentation
+
+To upgrade OpenTelemetry auto instrumentation (agent) follow these steps:
+
+1. Upgrade OpenTelemetry API and agent versions in [build.gradle.kts](./build.gradle.kts). 
+   The OpenTelemetry API version has to match the version used by the agent.
+2. Make sure shadow replace configuration from [build.gradle.kts](./build.gradle.kts) and [./instrumentation/build.gradle.kts](./instrumentation/build.gradle.kts)
+   is aligned with https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/5ef915879162bcccd0f3727d25bb735816263188/javaagent/javaagent.gradle#L83-L95 
+   and https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/5ef915879162bcccd0f3727d25bb735816263188/instrumentation/instrumentation.gradle#L66-L76
+3. Make sure (./buildSrc) is synced with https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/master/buildSrc. 
+   It should be one to one copy.
+4. Make sure Hypertrace instrumentation names match with OpenTelemetry instrumentation names.

build.gradle.kts

@@ -28,8 +28,8 @@ subprojects {
     description = "Hypertrace OpenTelemetry Javaagent"
 
     extra.set("versions", mapOf(
-            "opentelemetry" to "0.12.0",
-            "opentelemetry_java_agent" to "0.12.1",
+            "opentelemetry" to "0.13.1",
+            "opentelemetry_java_agent" to "0.13.0",
             "byte_buddy" to "1.10.18"
     ))
 

buildSrc/build.gradle.kts

@@ -1,42 +1,42 @@
 plugins {
-    groovy
-    `java-gradle-plugin`
-    id("com.diffplug.gradle.spotless") version "4.3.0"
+  groovy
+  `java-gradle-plugin`
+  id("com.diffplug.gradle.spotless") version "4.3.0"
 }
 
 gradlePlugin {
-    plugins {
-        create("muzzle-plugin") {
-            id = "muzzle"
-            implementationClass = "MuzzlePlugin"
-        }
-        create("auto-instrumentation-plugin") {
-            id = "io.opentelemetry.instrumentation.auto-instrumentation"
-            implementationClass = "io.opentelemetry.instrumentation.gradle.AutoInstrumentationPlugin"
-        }
+  plugins {
+    create("muzzle-plugin") {
+      id = "muzzle"
+      implementationClass = "MuzzlePlugin"
     }
+    create("auto-instrumentation-plugin") {
+      id = "io.opentelemetry.instrumentation.auto-instrumentation"
+      implementationClass = "io.opentelemetry.instrumentation.gradle.AutoInstrumentationPlugin"
+    }
+  }
 }
 
 repositories {
-    mavenLocal()
-    jcenter()
-    mavenCentral()
+  mavenLocal()
+  jcenter()
+  mavenCentral()
 }
 
 dependencies {
-    implementation(gradleApi())
-    implementation(localGroovy())
+  implementation(gradleApi())
+  implementation(localGroovy())
 
-    implementation("org.eclipse.aether", "aether-connector-basic", "1.1.0")
-    implementation("org.eclipse.aether", "aether-transport-http", "1.1.0")
-    implementation("org.apache.maven", "maven-aether-provider", "3.3.9")
+  implementation("org.eclipse.aether", "aether-connector-basic", "1.1.0")
+  implementation("org.eclipse.aether", "aether-transport-http", "1.1.0")
+  implementation("org.apache.maven", "maven-aether-provider", "3.3.9")
 
-    implementation("com.google.guava", "guava", "20.0")
-    implementation("org.ow2.asm", "asm", "7.0-beta")
-    implementation("org.ow2.asm", "asm-tree", "7.0-beta")
-    implementation("org.apache.httpcomponents:httpclient:4.5.10")
-    implementation("net.bytebuddy:byte-buddy-gradle-plugin:1.10.18")
+  implementation("com.google.guava", "guava", "20.0")
+  implementation("org.ow2.asm", "asm", "7.0-beta")
+  implementation("org.ow2.asm", "asm-tree", "7.0-beta")
+  implementation("org.apache.httpcomponents:httpclient:4.5.10")
+  implementation("net.bytebuddy:byte-buddy-gradle-plugin:1.10.18")
 
-    testImplementation("org.spockframework", "spock-core", "1.3-groovy-2.5")
-    testImplementation("org.codehaus.groovy", "groovy-all", "2.5.8")
+  testImplementation("org.spockframework", "spock-core", "1.3-groovy-2.5")
+  testImplementation("org.codehaus.groovy", "groovy-all", "2.5.8")
 }

buildSrc/src/main/groovy/MuzzlePlugin.groovy

@@ -76,7 +76,7 @@ class MuzzlePlugin implements Plugin<Project> {
           ClassLoader userCL = createCompileDepsClassLoader(project, bootstrapProject)
           ClassLoader instrumentationCL = createInstrumentationClassloader(project, toolingProject)
           Method assertionMethod = instrumentationCL.loadClass('io.opentelemetry.javaagent.tooling.muzzle.matcher.MuzzleGradlePluginUtil')
-                  .getMethod('assertInstrumentationMuzzled', ClassLoader.class, ClassLoader.class, boolean.class)
+            .getMethod('assertInstrumentationMuzzled', ClassLoader.class, ClassLoader.class, boolean.class)
           assertionMethod.invoke(null, instrumentationCL, userCL, true)
         }
         println "Muzzle executing for $project"
@@ -88,7 +88,7 @@ class MuzzlePlugin implements Plugin<Project> {
       doLast {
         ClassLoader instrumentationCL = createInstrumentationClassloader(project, toolingProject)
         Method assertionMethod = instrumentationCL.loadClass('io.opentelemetry.javaagent.tooling.muzzle.matcher.MuzzleGradlePluginUtil')
-                .getMethod('printMuzzleReferences', ClassLoader.class)
+          .getMethod('printMuzzleReferences', ClassLoader.class)
         assertionMethod.invoke(null, instrumentationCL)
       }
     }
@@ -311,17 +311,17 @@ class MuzzlePlugin implements Plugin<Project> {
       }
 
       def draftVersion = versionString.contains("rc") ||
-              versionString.contains(".cr") ||
-              versionString.contains("alpha") ||
-              versionString.contains("beta") ||
-              versionString.contains("-b") ||
-              versionString.contains(".m") ||
-              versionString.contains("-m") ||
-              versionString.contains("-dev") ||
-              versionString.contains("-ea") ||
-              versionString.contains("-atlassian-") ||
-              versionString.contains("public_draft") ||
-              versionString.matches(GIT_SHA_PATTERN)
+        versionString.contains(".cr") ||
+        versionString.contains("alpha") ||
+        versionString.contains("beta") ||
+        versionString.contains("-b") ||
+        versionString.contains(".m") ||
+        versionString.contains("-m") ||
+        versionString.contains("-dev") ||
+        versionString.contains("-ea") ||
+        versionString.contains("-atlassian-") ||
+        versionString.contains("public_draft") ||
+        versionString.matches(GIT_SHA_PATTERN)
 
       return !draftVersion
     }
@@ -382,7 +382,7 @@ class MuzzlePlugin implements Plugin<Project> {
         try {
           // find all instrumenters, get muzzle, and assert
           Method assertionMethod = instrumentationCL.loadClass('io.opentelemetry.javaagent.tooling.muzzle.matcher.MuzzleGradlePluginUtil')
-                  .getMethod('assertInstrumentationMuzzled', ClassLoader.class, ClassLoader.class, boolean.class)
+            .getMethod('assertInstrumentationMuzzled', ClassLoader.class, ClassLoader.class, boolean.class)
           assertionMethod.invoke(null, instrumentationCL, userCL, muzzleDirective.assertPass)
         } finally {
           Thread.currentThread().contextClassLoader = ccl

buildSrc/src/main/java/io/opentelemetry/instrumentation/gradle/bytebuddy/ByteBuddyPluginConfigurator.java

@@ -5,6 +5,7 @@
 
 package io.opentelemetry.instrumentation.gradle.bytebuddy;
 
+import com.google.common.collect.ImmutableList;
 import java.io.File;
 import java.util.Arrays;
 import java.util.List;
@@ -46,7 +47,16 @@ public ByteBuddyPluginConfigurator(
     this.project = project;
     this.sourceSet = sourceSet;
     this.pluginClassName = pluginClassName;
-    this.inputClasspath = inputClasspath;
+
+    // add build resources dir to classpath if it's present
+    File resourcesDir = sourceSet.getOutput().getResourcesDir();
+    this.inputClasspath =
+        resourcesDir == null
+            ? inputClasspath
+            : ImmutableList.<File>builder()
+                .addAll(inputClasspath)
+                .add(sourceSet.getOutput().getResourcesDir())
+                .build();
   }
 
   public void configure() {
@@ -58,6 +68,8 @@ public void configure() {
 
       if (compile != null) {
         Task languageTask = createLanguageTask(compile, taskName + language);
+        // We also process resources for SPI classes.
+        languageTask.dependsOn(sourceSet.getProcessResourcesTaskName());
         byteBuddyTask.dependsOn(languageTask);
       }
     }

filter-custom-opa/build.gradle.kts

@@ -3,7 +3,7 @@ plugins {
 }
 
 dependencies {
-    implementation(project(":filter-api"))
+    api(project(":filter-api"))
     implementation("org.slf4j:slf4j-api:1.7.30")
     implementation("com.squareup.okhttp3:okhttp:3.14.9")
     implementation("com.fasterxml.jackson.core:jackson-databind:2.11.3")

instrumentation/build.gradle.kts

@@ -76,5 +76,6 @@ tasks {
         relocate("io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api")
         relocate("io.opentelemetry.spi", "io.opentelemetry.javaagent.shaded.io.opentelemetry.spi")
         relocate("io.opentelemetry.context", "io.opentelemetry.javaagent.shaded.io.opentelemetry.context")
+        relocate ("io.opentelemetry.extension.trace.propagation", "io.opentelemetry.javaagent.shaded.io.opentelemetry.extension.trace.propagation")
     }
 }

javaagent-core/src/main/java/org/hypertrace/agent/core/EnvironmentConfig.java

@@ -45,6 +45,7 @@ private EnvironmentConfig() {}
   private static final String OPA_PREFIX = REPORTING_PREFIX + "opa.";
   static final String OPA_ENDPOINT = OPA_PREFIX + "endpoint";
   static final String OPA_POLL_PERIOD = OPA_PREFIX + "poll.period.seconds";
+  static final String OPA_ENABLED = OPA_PREFIX + "enabled";
 
   private static final String CAPTURE_PREFIX = HT_PREFIX + "data.capture.";
   public static final String CAPTURE_BODY_MAX_SIZE_BYTES = CAPTURE_PREFIX + "body.max.size.bytes";
@@ -103,6 +104,10 @@ private static Opa.Builder applyOpa(Opa.Builder builder) {
       builder.setPollPeriodSeconds(
           Int32Value.newBuilder().setValue(Integer.parseInt(pollPeriod)).build());
     }
+    String enabled = getProperty(OPA_ENABLED);
+    if (enabled != null) {
+      builder.setEnabled(BoolValue.newBuilder().setValue(Boolean.valueOf(enabled)).build());
+    }
     return builder;
   }
 

javaagent-core/src/test/java/org/hypertrace/agent/core/EnvironmentConfigTest.java

@@ -31,6 +31,7 @@ class EnvironmentConfigTest {
   @ClearSystemProperty(key = EnvironmentConfig.REPORTING_SECURE)
   @ClearSystemProperty(key = EnvironmentConfig.OPA_ENDPOINT)
   @ClearSystemProperty(key = EnvironmentConfig.OPA_POLL_PERIOD)
+  @ClearSystemProperty(key = EnvironmentConfig.OPA_ENABLED)
   @ClearSystemProperty(key = EnvironmentConfig.PROPAGATION_FORMATS)
   @ClearSystemProperty(key = EnvironmentConfig.CAPTURE_HTTP_BODY_PREFIX + "request")
   @ClearSystemProperty(key = EnvironmentConfig.CAPTURE_BODY_MAX_SIZE_BYTES)
@@ -42,6 +43,7 @@ public void systemProperties() {
     System.setProperty(EnvironmentConfig.CAPTURE_HTTP_BODY_PREFIX + "request", "true");
     System.setProperty(EnvironmentConfig.OPA_ENDPOINT, "http://azkaban:9090");
     System.setProperty(EnvironmentConfig.OPA_POLL_PERIOD, "10");
+    System.setProperty(EnvironmentConfig.OPA_ENABLED, "true");
     System.setProperty(EnvironmentConfig.PROPAGATION_FORMATS, "B3,TRACECONTEXT");
     System.setProperty(EnvironmentConfig.CAPTURE_BODY_MAX_SIZE_BYTES, "512");
 
@@ -56,6 +58,7 @@ public void systemProperties() {
     Assertions.assertEquals("http://:-)", agentConfig.getReporting().getEndpoint().getValue());
     Assertions.assertEquals(
         "http://azkaban:9090", agentConfig.getReporting().getOpa().getEndpoint().getValue());
+    Assertions.assertEquals(true, agentConfig.getReporting().getOpa().getEnabled().getValue());
     Assertions.assertEquals(
         10, agentConfig.getReporting().getOpa().getPollPeriodSeconds().getValue());
     Assertions.assertEquals(512, agentConfig.getDataCapture().getBodyMaxSizeBytes().getValue());

javaagent/build.gradle.kts

@@ -14,7 +14,6 @@ dependencies {
     implementation("io.opentelemetry.javaagent", "opentelemetry-javaagent", version = "${versions["opentelemetry_java_agent"]}", classifier = "all")
     implementation(project(":javaagent-core"))
     implementation(project(":filter-api"))
-    implementation(project(":filter-custom-opa"))
 }
 
 base.archivesBaseName = "hypertrace-agent"
@@ -45,10 +44,6 @@ tasks {
 
         relocate("com.blogspot.mydailyjava", "io.opentelemetry.javaagent.shaded.org.hypertrace.shaded.com.blogspot.mydailyjava")
 
-        // used by the filter-custom-opa
-        relocate("okhttp3", "io.opentelemetry.javaagent.shaded.org.hypertrace.shaded.okhttp3")
-        relocate("okio", "io.opentelemetry.javaagent.shaded.org.hypertrace.shaded.okio")
-
         dependencies {
             exclude(dependency("org.codehaus.mojo:animal-sniffer-annotations"))
             exclude(dependency("javax.annotation:javax.annotation-api"))
@@ -65,6 +60,7 @@ tasks {
         relocate("io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api")
         relocate("io.opentelemetry.spi", "io.opentelemetry.javaagent.shaded.io.opentelemetry.spi")
         relocate("io.opentelemetry.context", "io.opentelemetry.javaagent.shaded.io.opentelemetry.context")
+        relocate("io.opentelemetry.extension.trace.propagation", "io.opentelemetry.javaagent.shaded.io.opentelemetry.extension.trace.propagation")
 
         mergeServiceFiles {
             include("inst/META-INF/services/*")

otel-extensions/build.gradle.kts

@@ -5,6 +5,8 @@ plugins {
 val versions: Map<String, String> by extra
 
 dependencies {
+    api(project(":filter-custom-opa"))
+
     compileOnly("io.opentelemetry:opentelemetry-sdk:${versions["opentelemetry"]}")
     implementation("io.opentelemetry.javaagent:opentelemetry-javaagent-spi:${versions["opentelemetry_java_agent"]}")
 

otel-extensions/src/main/java/org/hypertrace/agent/otel/extensions/FilterComponentInstaller.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hypertrace.agent.otel.extensions;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.spi.ComponentInstaller;
+import org.hypertrace.agent.filter.FilterRegistry;
+
+@AutoService(ComponentInstaller.class)
+public class FilterComponentInstaller implements ComponentInstaller {
+
+  @Override
+  public void beforeByteBuddyAgent() {
+    // resolves filter via service loader resolution
+    FilterRegistry.getFilter();
+  }
+
+  @Override
+  public void afterByteBuddyAgent() {}
+}

otel-extensions/src/main/java/org/hypertrace/agent/otel/extensions/processor/HypertraceTracerCustomizer.java

@@ -18,7 +18,7 @@
 
 import com.google.auto.service.AutoService;
 import io.opentelemetry.javaagent.spi.TracerCustomizer;
-import io.opentelemetry.sdk.trace.TracerSdkManagement;
+import io.opentelemetry.sdk.trace.SdkTracerManagement;
 
 /**
  * This is a workaround to add container ID tags to spans when Zipkin exporter is used. Zipkin
@@ -32,7 +32,7 @@
 public class HypertraceTracerCustomizer implements TracerCustomizer {
 
   @Override
-  public void configure(TracerSdkManagement tracerManagement) {
+  public void configure(SdkTracerManagement tracerManagement) {
     String exporter = System.getProperty("otel.exporter");
     if (exporter != null && exporter.contains("zipkin")) {
       tracerManagement.addSpanProcessor(new AddTagsSpanProcessor());

testing-common/build.gradle.kts

@@ -14,6 +14,7 @@ dependencies {
     api("io.opentelemetry:opentelemetry-sdk:${versions["opentelemetry"]}")
     api("com.squareup.okhttp3:okhttp:4.9.0")
     implementation("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling:${versions["opentelemetry_java_agent"]}")
+    implementation("io.opentelemetry.javaagent:opentelemetry-javaagent-spi:${versions["opentelemetry_java_agent"]}")
     implementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:${versions["opentelemetry_java_agent"]}")
     implementation("ch.qos.logback:logback-classic:1.2.3")
     implementation("org.slf4j:log4j-over-slf4j:1.7.30")