Use separate TestEventHandlers per framework in CI Vis instrumentations

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java

@@ -44,24 +44,28 @@ public void testSuiteStarted(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
-          suiteDescriptor,
-          testSuiteName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          Collections.emptyList(),
-          false,
-          TestFrameworkInstrumentation.CUCUMBER,
-          null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteStart(
+              suiteDescriptor,
+              testSuiteName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              Collections.emptyList(),
+              false,
+              TestFrameworkInstrumentation.CUCUMBER,
+              null);
     }
   }
 
   @Override
   public void testSuiteFinished(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor, null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFinish(suiteDescriptor, null);
     }
   }
 
@@ -71,17 +75,19 @@ public void testStarted(final Description description) {
     String testName = CucumberUtils.getTestNameForScenario(description);
     List<String> categories = getCategories(description);
 
-    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart(
-        new TestSuiteDescriptor(testSuiteName, null),
-        CucumberUtils.toTestDescriptor(description),
-        testName,
-        FRAMEWORK_NAME,
-        FRAMEWORK_VERSION,
-        null,
-        categories,
-        TestSourceData.UNKNOWN,
-        null,
-        executionHistories.get(description));
+    TestEventsHandlerHolder.HANDLERS
+        .get(TestFrameworkInstrumentation.CUCUMBER)
+        .onTestStart(
+            new TestSuiteDescriptor(testSuiteName, null),
+            CucumberUtils.toTestDescriptor(description),
+            testName,
+            FRAMEWORK_NAME,
+            FRAMEWORK_VERSION,
+            null,
+            categories,
+            TestSourceData.UNKNOWN,
+            null,
+            executionHistories.get(description));
 
     recordFeatureFileCodeCoverage(description);
   }
@@ -100,8 +106,9 @@ private static void recordFeatureFileCodeCoverage(Description scenarioDescriptio
   public void testFinished(final Description description) {
     TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
     TestExecutionHistory executionHistory = executionHistories.get(description);
-    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(
-        testDescriptor, null, executionHistory);
+    TestEventsHandlerHolder.HANDLERS
+        .get(TestFrameworkInstrumentation.CUCUMBER)
+        .onTestFinish(testDescriptor, null, executionHistory);
   }
 
   // same callback is executed both for test cases and test suites (for setup/teardown errors)
@@ -111,11 +118,15 @@ public void testFailure(final Failure failure) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       Throwable throwable = failure.getException();
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFailure(suiteDescriptor, throwable);
     } else {
       TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
       Throwable throwable = failure.getException();
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestFailure(testDescriptor, throwable);
     }
   }
 
@@ -132,10 +143,14 @@ public void testAssumptionFailure(final Failure failure) {
     Description description = failure.getDescription();
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteSkip(suiteDescriptor, reason);
     } else {
       TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSkip(testDescriptor, reason);
     }
   }
 
@@ -147,32 +162,40 @@ public void testIgnored(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
-          suiteDescriptor,
-          testSuiteName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          Collections.emptyList(),
-          false,
-          TestFrameworkInstrumentation.CUCUMBER,
-          null);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor, null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteStart(
+              suiteDescriptor,
+              testSuiteName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              Collections.emptyList(),
+              false,
+              TestFrameworkInstrumentation.CUCUMBER,
+              null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteSkip(suiteDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFinish(suiteDescriptor, null);
     } else {
       String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
       String testName = CucumberUtils.getTestNameForScenario(description);
       List<String> categories = getCategories(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore(
-          new TestSuiteDescriptor(testSuiteName, null),
-          CucumberUtils.toTestDescriptor(description),
-          testName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          categories,
-          TestSourceData.UNKNOWN,
-          reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestIgnore(
+              new TestSuiteDescriptor(testSuiteName, null),
+              CucumberUtils.toTestDescriptor(description),
+              testName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              categories,
+              TestSourceData.UNKNOWN,
+              reason);
     }
   }
 

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberInstrumentation.java

@@ -8,6 +8,7 @@
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.agent.tooling.muzzle.Reference;
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.InstrumentationContext;
 import java.util.Collections;
 import java.util.List;
@@ -82,6 +83,8 @@ public static void addTracingListener(
         }
       }
 
+      TestEventsHandlerHolder.start(TestFrameworkInstrumentation.CUCUMBER);
+
       replacedNotifier.addListener(
           new CucumberTracingListener(
               InstrumentationContext.get(Description.class, TestExecutionHistory.class), children));

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.civisibility.InstrumentationBridge;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.cucumber.core.gherkin.Pickle;
 import java.util.List;
@@ -83,7 +84,10 @@ public static Boolean run(
         @Advice.Argument(0) RunNotifier notifier) {
 
       TestIdentifier test = CucumberUtils.toTestIdentifier(description);
-      SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(test);
+      SkipReason skipReason =
+          TestEventsHandlerHolder.HANDLERS
+              .get(TestFrameworkInstrumentation.CUCUMBER)
+              .skipReason(test);
       if (skipReason == null) {
         return null;
       }

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/execution/Cucumber4ExecutionInstrumentation.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.InstrumentationContext;
 import datadog.trace.instrumentation.junit4.CucumberUtils;
 import datadog.trace.instrumentation.junit4.JUnit4Utils;
@@ -97,8 +98,9 @@ public static Boolean execute(
       Description description = CucumberUtils.getPickleRunnerDescription(pickleRunner);
       TestIdentifier testIdentifier = CucumberUtils.toTestIdentifier(description);
       TestExecutionPolicy executionPolicy =
-          TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy(
-              testIdentifier, TestSourceData.UNKNOWN);
+          TestEventsHandlerHolder.HANDLERS
+              .get(TestFrameworkInstrumentation.CUCUMBER)
+              .executionPolicy(testIdentifier, TestSourceData.UNKNOWN);
       if (!executionPolicy.applicable()) {
         // retries not applicable, run original method
         return null;

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

@@ -1,6 +1,7 @@
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
 import datadog.trace.instrumentation.junit4.CucumberTracingListener
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
@@ -211,7 +212,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     .map(f -> "classpath:" + f).
     collect(Collectors.joining(",")))
 
-    TestEventsHandlerHolder.start()
+    TestEventsHandlerHolder.start(TestFrameworkInstrumentation.CUCUMBER)
     try {
       def result = runner.run(TestSucceedCucumber)
       if (expectSuccess) {
@@ -224,7 +225,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
         }
       }
     } finally {
-      TestEventsHandlerHolder.stop()
+      TestEventsHandlerHolder.stop(TestFrameworkInstrumentation.CUCUMBER)
     }
   }
 

dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy

@@ -1,4 +1,5 @@
 import datadog.trace.api.DisableTestTrace
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
 import junit.runner.Version
@@ -37,7 +38,7 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
   }
 
   private void runTests(Collection<Class<?>> tests, boolean expectSuccess = true) {
-    TestEventsHandlerHolder.start()
+    TestEventsHandlerHolder.start(TestFrameworkInstrumentation.JUNIT4)
     try {
       Class[] array = tests.toArray(new Class[0])
       def result = runner.run(array)
@@ -51,7 +52,7 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
         }
       }
     } finally {
-      TestEventsHandlerHolder.stop()
+      TestEventsHandlerHolder.stop(TestFrameworkInstrumentation.JUNIT4)
     }
   }
 

dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitInstrumentation.java

@@ -7,6 +7,7 @@
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.InstrumentationContext;
 import java.util.Collections;
 import java.util.List;
@@ -73,6 +74,8 @@ public static void addTracingListener(
         }
       }
 
+      TestEventsHandlerHolder.start(TestFrameworkInstrumentation.MUNIT);
+
       replacedNotifier.addListener(
           new MUnitTracingListener(
               InstrumentationContext.get(Description.class, TestExecutionHistory.class)));