Prevent duplicate telemetry when using both library and auto instrume…

…ntation (#2661)

* Prevent duplicate telemetry when using both library and auto instrumentation

* Add unit test

* Fix Oshi tests

* Fix couchbase 3.1 tests

docs/contributing/muzzle.md

@@ -21,13 +21,15 @@ The compile-time reference collection and code generation process is implemented
 plugin (called `MuzzleCodeGenerationPlugin`).
 
 For each instrumentation module the ByteBuddy plugin collects symbols referring to both internal and
-third party APIs used by the currently processed module's type instrumentations (`InstrumentationModule#typeInstrumentations()`).
-The reference collection process starts from advice classes (values of the map returned by the
-`TypeInstrumentation#transformers()`method) and traverses the class graph until it encounters
-a reference to a non-instrumentation class (determined by `InstrumentationClassPredicate`).
-Aside from references, the collection process also builds a graph of dependencies between internal
-instrumentation helper classes - this dependency graph is later used to construct a list of helper
-classes that will be injected to the application classloader (`InstrumentationModule#getMuzzleHelperClassNames()`).
+third party APIs used by the currently processed module's type
+instrumentations (`InstrumentationModule#typeInstrumentations()`). The reference collection process
+starts from advice classes (values of the map returned by the
+`TypeInstrumentation#transformers()` method) and traverses the class graph until it encounters a
+reference to a non-instrumentation class (determined by `InstrumentationClassPredicate` and
+the `InstrumentationModule#isLibraryInstrumentationClass(String)` predicate). Aside from references,
+the collection process also builds a graph of dependencies between internal instrumentation helper
+classes - this dependency graph is later used to construct a list of helper classes that will be
+injected to the application classloader (`InstrumentationModule#getMuzzleHelperClassNames()`).
 
 All collected references are then used to create a `ReferenceMatcher` instance. This matcher
 is stored in the instrumentation module class in the method `InstrumentationModule#getMuzzleReferenceMatcher()`

examples/distro/gradle/shadow.gradle

@@ -4,8 +4,8 @@ ext.relocatePackages = { shadowJar ->
   // rewrite dependencies calling Logger.getLogger
   shadowJar.relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // rewrite library instrumentation dependencies
-  shadowJar.relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
+  shadowJar.relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API usage
   shadowJar.relocate "io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api"

gradle/instrumentation.gradle

@@ -79,15 +79,12 @@ shadowJar {
 
   archiveFileName = 'agent-testing.jar'
 
-  // rewrite library instrumentation dependencies
-  relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
-
   // Prevents conflict with other SLF4J instances. Important for premain.
   relocate 'org.slf4j', 'io.opentelemetry.javaagent.slf4j'
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
-  // prevents conflict with library instrumentation
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
   relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
 
   // relocate OpenTelemetry API usage

instrumentation/apache-dubbo/apache-dubbo-2.7/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/apachedubbo/v2_7/DubboInstrumentationModule.java

@@ -32,6 +32,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("org.apache.dubbo.rpc.Filter");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.apachedubbo.v2_7");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new DubboInstrumentation());

instrumentation/armeria-1.3/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/armeria/v1_3/ArmeriaInstrumentationModule.java

@@ -26,6 +26,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("com.linecorp.armeria.server.metric.PrometheusExpositionServiceBuilder");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.armeria.v1_3");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/aws-lambda-1.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awslambda/v1_0/AwsLambdaInstrumentationModule.java

@@ -23,6 +23,11 @@ public String[] additionalHelperClassNames() {
     return new String[] {"io.opentelemetry.extension.aws.AwsXrayPropagator"};
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awslambda.v1_0");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new AwsLambdaRequestHandlerInstrumentation());

instrumentation/aws-sdk/aws-sdk-1.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awssdk/v1_11/AwsSdkInstrumentationModule.java

@@ -23,6 +23,11 @@ public String[] additionalHelperClassNames() {
     return new String[] {"io.opentelemetry.extension.aws.AwsXrayPropagator"};
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awssdk.v1_11");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/aws-sdk/aws-sdk-2.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/awssdk/v2_2/AwsSdkInstrumentationModule.java

@@ -43,6 +43,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("software.amazon.awssdk.core.interceptor.ExecutionInterceptor");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.awssdk.v2_2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new AwsSdkInitializationInstrumentation());

instrumentation/couchbase/couchbase-3.1/javaagent/couchbase-3.1-javaagent.gradle

@@ -21,4 +21,9 @@ dependencies {
   testLibrary group: "com.couchbase.client", name: "java-client", version: "3.1.0"
 
   testImplementation group: "org.testcontainers", name: "couchbase", version: versions.testcontainers
-}
+}
+
+configurations {
+  // exclude lib from test classpath, otherwise the javaagent would be skipped
+  testImplementation.exclude group: 'com.couchbase.client', module: 'tracing-opentelemetry'
+}

instrumentation/couchbase/couchbase-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/couchbase/v3_1/CouchbaseInstrumentationModule.java

@@ -20,11 +20,6 @@ public CouchbaseInstrumentationModule() {
     super("couchbase", "couchbase-3.1");
   }
 
-  @Override
-  public boolean isHelperClass(String className) {
-    return className.startsWith("com.couchbase.client.tracing.opentelemetry");
-  }
-
   @Override
   public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     // New class introduced in 3.1, the minimum version we support.
@@ -33,6 +28,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("com.couchbase.client.core.cnc.TracingIdentifiers");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("com.couchbase.client.tracing.opentelemetry");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new CouchbaseEnvironmentInstrumentation());

instrumentation/grpc-1.5/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/grpc/v1_5/GrpcInstrumentationModule.java

@@ -18,6 +18,11 @@ public GrpcInstrumentationModule() {
     super("grpc", "grpc-1.5");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.grpc.v1_5");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(

instrumentation/instrumentation.gradle

@@ -61,12 +61,12 @@ shadowJar {
     exclude(project(':javaagent-api'))
   }
 
-  // rewrite library instrumentation dependencies
-  relocate "io.opentelemetry.instrumentation", "io.opentelemetry.javaagent.shaded.instrumentation"
-
   // rewrite dependencies calling Logger.getLogger
   relocate 'java.util.logging.Logger', 'io.opentelemetry.javaagent.bootstrap.PatchLogger'
 
+  // prevents conflict with library instrumentation, which uses its own instrumentation-api
+  relocate 'io.opentelemetry.instrumentation.api', 'io.opentelemetry.javaagent.shaded.instrumentation.api'
+
   // relocate OpenTelemetry API usage
   relocate "io.opentelemetry.api", "io.opentelemetry.javaagent.shaded.io.opentelemetry.api"
   relocate "io.opentelemetry.semconv", "io.opentelemetry.javaagent.shaded.io.opentelemetry.semconv"

instrumentation/jaxws/jaxws-2.0-axis2-1.6/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/axis2/Axis2InstrumentationModule.java

@@ -26,6 +26,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("org.apache.axis2.jaxws.api.MessageAccessor");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.axis2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new InvocationListenerRegistryTypeInstrumentation());

instrumentation/jaxws/jaxws-2.0-cxf-3.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/cxf/CxfInstrumentationModule.java

@@ -17,6 +17,11 @@ public CxfInstrumentationModule() {
     super("cxf", "cxf-3.0");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.cxf");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new EndpointImplTypeInstrumentation());

instrumentation/lettuce/lettuce-5.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/lettuce/v5_1/LettuceInstrumentationModule.java

@@ -36,6 +36,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("io.lettuce.core.tracing.Tracing");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.lettuce.v5_1");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new DefaultClientResourcesInstrumentation());

instrumentation/log4j/log4j-2.13.2/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/log4j/v2_13_2/Log4j2InstrumentationModule.java

@@ -37,6 +37,11 @@ public ElementMatcher.Junction<ClassLoader> classLoaderMatcher() {
     return hasClassesNamed("org.apache.logging.log4j.core.util.ContextDataProvider");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.log4j.v2_13_2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Arrays.asList(new BugFixingInstrumentation(), new EmptyTypeInstrumentation());

instrumentation/logback/logback-1.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/logback/v1_0/LogbackInstrumentationModule.java

@@ -26,6 +26,11 @@ public List<TypeInstrumentation> typeInstrumentations() {
     return asList(new LoggerInstrumentation(), new LoggingEventInstrumentation());
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.logback.v1_0");
+  }
+
   @Override
   public Map<String, String> contextStore() {
     return singletonMap("ch.qos.logback.classic.spi.ILoggingEvent", Span.class.getName());

instrumentation/okhttp/okhttp-3.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/okhttp/v3_0/OkHttp3InstrumentationModule.java

@@ -29,6 +29,11 @@ public OkHttp3InstrumentationModule() {
     super("okhttp", "okhttp-3.0");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.okhttp.v3_0");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new OkHttpClientInstrumentation());

instrumentation/oshi/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/oshi/OshiInstrumentationModule.java

@@ -31,6 +31,11 @@ public OshiInstrumentationModule() {
     super("oshi");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.oshi");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new SystemInfoInstrumentation());

instrumentation/oshi/javaagent/src/test/groovy/io/opentelemetry/javaagent/instrumentation/oshi/OshiTest.groovy

@@ -21,13 +21,13 @@ class OshiTest extends AgentInstrumentationSpecification {
     expect:
     platform != null
     // TODO (trask) is this the instrumentation library name we want?
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.disk.io") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.disk.operations") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.memory.usage") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.memory.utilization") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.errors") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.io") != null
-    findMetric("io.opentelemetry.javaagent.shaded.instrumentation.oshi", "system.network.packets") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.disk.io") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.disk.operations") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.memory.usage") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.memory.utilization") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.errors") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.io") != null
+    findMetric("io.opentelemetry.instrumentation.oshi", "system.network.packets") != null
   }
 
   def findMetric(instrumentationName, metricName) {

instrumentation/reactor-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/reactor/ReactorInstrumentationModule.java

@@ -26,6 +26,11 @@ public ReactorInstrumentationModule() {
     super("reactor", "reactor-3.1");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.reactor");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new HooksInstrumentation());

instrumentation/rocketmq-client-4.8/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/rocketmq/RocketMqInstrumentationModule.java

@@ -18,6 +18,11 @@ public RocketMqInstrumentationModule() {
     super("rocketmq-client", "rocketmq-client-4.8");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.rocketmq");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(new RocketMqProducerInstrumentation(), new RocketMqConsumerInstrumentation());

instrumentation/rxjava-2.0/javaagent/src/main/java/io/opentelemetry/instrumentation/rxjava2/RxJava2InstrumentationModule.java

@@ -26,6 +26,11 @@ public RxJava2InstrumentationModule() {
     super("rxjava2");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.rxjava2");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return Collections.singletonList(new PluginInstrumentation());

instrumentation/spring/spring-webflux-5.0/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/spring/webflux/client/WebfluxClientInstrumentationModule.java

@@ -29,6 +29,11 @@ public WebfluxClientInstrumentationModule() {
     super("spring-webflux", "spring-webflux-5.0", "spring-webflux-client");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.spring.webflux.client");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return singletonList(new WebClientBuilderInstrumentation());

instrumentation/spring/spring-webmvc-3.1/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/springwebmvc/SpringWebMvcInstrumentationModule.java

@@ -18,6 +18,11 @@ public SpringWebMvcInstrumentationModule() {
     super("spring-webmvc", "spring-webmvc-3.1");
   }
 
+  @Override
+  public boolean isLibraryInstrumentationClass(String className) {
+    return className.startsWith("io.opentelemetry.instrumentation.spring.webmvc");
+  }
+
   @Override
   public List<TypeInstrumentation> typeInstrumentations() {
     return asList(