Merge branch 'master' into daniel.mohedano/instrumentation-test-handling-refactor

Author: daniel-mohedano

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 315bc8a32cc888834726397f088336ba8038277f
+default_system_tests_commit: &default_system_tests_commit 0509dbd094c9cbf15f58db96f62276a0adff7efa
 
 parameters:
   nightly:
@@ -857,6 +857,9 @@ jobs:
               APPSEC_REQUEST_BLOCKING
               APPSEC_RASP
               APPSEC_RUNTIME_ACTIVATION
+              APPSEC_API_SECURITY
+              APPSEC_API_SECURITY_RC
+              APPSEC_API_SECURITY_WITH_SAMPLING
               APPSEC_WAF_TELEMETRY
               APPSEC_STANDALONE_V2
               IAST_STANDALONE_V2

.github/CODEOWNERS

@@ -65,3 +65,10 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 # @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring
+
+# @DataDog/data-streams-monitoring
+dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/datastreams  @DataDog/data-streams-monitoring
+dd-trace-core/src/main/java/datadog/trace/core/datastreams                  @DataDog/data-streams-monitoring
+dd-trace-core/src/test/groovy/datadog/trace/core/datastreams                @DataDog/data-streams-monitoring
+internal-api/src/main/java/datadog/trace/api/datastreams                    @DataDog/data-streams-monitoring
+internal-api/src/test/groovy/datadog/trace/api/datastreams                  @DataDog/data-streams-monitoring

.github/workflows/README.md

@@ -87,7 +87,7 @@ _Recovery:_ Check at the milestone for the related issues and update them manual
 
 ### prune-github-container-registry [🔗](prune-github-container-registry.yaml)
 
-_Trigger:_ Every week or manually.
+_Trigger:_ Every day or manually.
 
 _Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
 

.github/workflows/prune-github-container-registry.yaml

@@ -23,5 +23,5 @@ jobs:
           keep-tags: |
             latest_snapshot
           prune-tags-regexes: |
-            ^[a-z0-9]{40}$
+            ^[a-z0-9]{40}(-arm64|-amd64)?$
           prune-untagged: true

.gitlab-ci.yml

@@ -24,6 +24,7 @@ variables:
   BUILD_CACHE_POLICY: pull
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
   JAVA_BUILD_IMAGE_VERSION: "v25.01"
+  REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
 
 default:
   tags: [ "arch:amd64" ]
@@ -116,10 +117,6 @@ test_published_artifacts:
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests
   stage: tests
   needs: [ build ]
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - when: on_success
   variables:
     BUILD_CACHE_TYPE: lib
   script:
@@ -144,10 +141,6 @@ muzzle:
   needs: [ build ]
   stage: tests
   parallel: 8
-  rules:
-    - if: '$POPULATE_CACHE'
-      when: never
-    - when: on_success
   variables:
     BUILD_CACHE_TYPE: test
   script:

dd-java-agent/agent-bootstrap/gradle.lockfile

@@ -133,17 +133,17 @@ org.ow2.asm:asm-analysis:7.2=jacocoAnt
 org.ow2.asm:asm-analysis:9.2=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-analysis:9.4=spotbugs
 org.ow2.asm:asm-commons:7.2=jacocoAnt
-org.ow2.asm:asm-commons:9.2=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-commons:9.4=spotbugs
+org.ow2.asm:asm-commons:9.7.1=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-tree:7.2=jacocoAnt
-org.ow2.asm:asm-tree:9.2=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-tree:9.4=spotbugs
+org.ow2.asm:asm-tree:9.7.1=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-util:9.2=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm-util:9.4=spotbugs
 org.ow2.asm:asm:7.2=jacocoAnt
 org.ow2.asm:asm:9.0=jmh,jmhCompileClasspath
-org.ow2.asm:asm:9.2=jmhRuntimeClasspath,testRuntimeClasspath
 org.ow2.asm:asm:9.4=spotbugs
+org.ow2.asm:asm:9.7.1=jmhRuntimeClasspath,testRuntimeClasspath
 org.pitest:pitest-command-line:1.9.11=pitest
 org.pitest:pitest-entry:1.9.11=pitest
 org.pitest:pitest:1.9.11=pitest

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -33,6 +33,7 @@
 import datadog.trace.api.profiling.ProfilingEnablement;
 import datadog.trace.api.scopemanager.ScopeListener;
 import datadog.trace.bootstrap.benchmark.StaticEventLogger;
+import datadog.trace.bootstrap.config.provider.StableConfigSource;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI;
 import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration;
@@ -82,43 +83,39 @@ public class Agent {
   private static final Logger log;
 
   private enum AgentFeature {
-    TRACING(propertyNameToSystemPropertyName(TraceInstrumentationConfig.TRACE_ENABLED), true),
-    JMXFETCH(propertyNameToSystemPropertyName(JmxFetchConfig.JMX_FETCH_ENABLED), true),
-    STARTUP_LOGS(
-        propertyNameToSystemPropertyName(GeneralConfig.STARTUP_LOGS_ENABLED),
-        DEFAULT_STARTUP_LOGS_ENABLED),
-    PROFILING(propertyNameToSystemPropertyName(ProfilingConfig.PROFILING_ENABLED), false),
-    APPSEC(propertyNameToSystemPropertyName(AppSecConfig.APPSEC_ENABLED), false),
-    IAST(propertyNameToSystemPropertyName(IastConfig.IAST_ENABLED), false),
-    REMOTE_CONFIG(
-        propertyNameToSystemPropertyName(RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED), true),
-    DEPRECATED_REMOTE_CONFIG(
-        propertyNameToSystemPropertyName(RemoteConfigConfig.REMOTE_CONFIG_ENABLED), true),
-    CWS(propertyNameToSystemPropertyName(CwsConfig.CWS_ENABLED), false),
-    CIVISIBILITY(propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_ENABLED), false),
-    CIVISIBILITY_AGENTLESS(
-        propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED), false),
-    USM(propertyNameToSystemPropertyName(UsmConfig.USM_ENABLED), false),
-    TELEMETRY(propertyNameToSystemPropertyName(GeneralConfig.TELEMETRY_ENABLED), true),
-    DEBUGGER(
-        propertyNameToSystemPropertyName(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED), false),
-    EXCEPTION_DEBUGGING(
-        propertyNameToSystemPropertyName(DebuggerConfig.EXCEPTION_REPLAY_ENABLED), false),
-    SPAN_ORIGIN(
-        propertyNameToSystemPropertyName(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED),
-        false),
-    DATA_JOBS(propertyNameToSystemPropertyName(GeneralConfig.DATA_JOBS_ENABLED), false),
-    AGENTLESS_LOG_SUBMISSION(
-        propertyNameToSystemPropertyName(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED), false);
-
+    TRACING(TraceInstrumentationConfig.TRACE_ENABLED, true),
+    JMXFETCH(JmxFetchConfig.JMX_FETCH_ENABLED, true),
+    STARTUP_LOGS(GeneralConfig.STARTUP_LOGS_ENABLED, DEFAULT_STARTUP_LOGS_ENABLED),
+    PROFILING(ProfilingConfig.PROFILING_ENABLED, false),
+    APPSEC(AppSecConfig.APPSEC_ENABLED, false),
+    IAST(IastConfig.IAST_ENABLED, false),
+    REMOTE_CONFIG(RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED, true),
+    DEPRECATED_REMOTE_CONFIG(RemoteConfigConfig.REMOTE_CONFIG_ENABLED, true),
+    CWS(CwsConfig.CWS_ENABLED, false),
+    CIVISIBILITY(CiVisibilityConfig.CIVISIBILITY_ENABLED, false),
+    CIVISIBILITY_AGENTLESS(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED, false),
+    USM(UsmConfig.USM_ENABLED, false),
+    TELEMETRY(GeneralConfig.TELEMETRY_ENABLED, true),
+    DEBUGGER(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED, false),
+    EXCEPTION_DEBUGGING(DebuggerConfig.EXCEPTION_REPLAY_ENABLED, false),
+    SPAN_ORIGIN(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED, false),
+    DATA_JOBS(GeneralConfig.DATA_JOBS_ENABLED, false),
+    AGENTLESS_LOG_SUBMISSION(GeneralConfig.AGENTLESS_LOG_SUBMISSION_ENABLED, false);
+
+    private final String configKey;
     private final String systemProp;
     private final boolean enabledByDefault;
 
-    AgentFeature(final String systemProp, final boolean enabledByDefault) {
-      this.systemProp = systemProp;
+    AgentFeature(final String configKey, final boolean enabledByDefault) {
+      this.configKey = configKey;
+      this.systemProp = propertyNameToSystemPropertyName(configKey);
       this.enabledByDefault = enabledByDefault;
     }
 
+    public String getConfigKey() {
+      return configKey;
+    }
+
     public String getSystemProp() {
       return systemProp;
     }
@@ -1212,10 +1209,17 @@ private static boolean isDebugMode() {
   /** @return {@code true} if the agent feature is enabled */
   private static boolean isFeatureEnabled(AgentFeature feature) {
     // must be kept in sync with logic from Config!
-    final String featureEnabledSysprop = feature.getSystemProp();
-    String featureEnabled = System.getProperty(featureEnabledSysprop);
+    final String featureConfigKey = feature.getConfigKey();
+    final String featureSystemProp = feature.getSystemProp();
+    String featureEnabled = System.getProperty(featureSystemProp);
+    if (featureEnabled == null) {
+      featureEnabled = getStableConfig(StableConfigSource.FLEET, featureConfigKey);
+    }
+    if (featureEnabled == null) {
+      featureEnabled = ddGetEnv(featureSystemProp);
+    }
     if (featureEnabled == null) {
-      featureEnabled = ddGetEnv(featureEnabledSysprop);
+      featureEnabled = getStableConfig(StableConfigSource.LOCAL, featureConfigKey);
     }
 
     if (feature.isEnabledByDefault()) {
@@ -1235,11 +1239,17 @@ private static boolean isFeatureEnabled(AgentFeature feature) {
   /** @see datadog.trace.api.ProductActivation#fromString(String) */
   private static boolean isFullyDisabled(final AgentFeature feature) {
     // must be kept in sync with logic from Config!
-    final String featureEnabledSysprop = feature.systemProp;
-    String settingValue = getNullIfEmpty(System.getProperty(featureEnabledSysprop));
+    final String featureConfigKey = feature.getConfigKey();
+    final String featureSystemProp = feature.getSystemProp();
+    String settingValue = getNullIfEmpty(System.getProperty(featureSystemProp));
     if (settingValue == null) {
-      settingValue = getNullIfEmpty(ddGetEnv(featureEnabledSysprop));
-      settingValue = settingValue != null && settingValue.isEmpty() ? null : settingValue;
+      settingValue = getNullIfEmpty(getStableConfig(StableConfigSource.FLEET, featureConfigKey));
+    }
+    if (settingValue == null) {
+      settingValue = getNullIfEmpty(ddGetEnv(featureSystemProp));
+    }
+    if (settingValue == null) {
+      settingValue = getNullIfEmpty(getStableConfig(StableConfigSource.LOCAL, featureConfigKey));
     }
 
     // defaults to inactive
@@ -1353,6 +1363,11 @@ private static String ddGetProperty(final String sysProp) {
     return value;
   }
 
+  /** Looks for sysProp in the Stable Configuration input */
+  private static String getStableConfig(StableConfigSource source, final String sysProp) {
+    return source.get(sysProp);
+  }
+
   /** Looks for the "DD_" environment variable equivalent of the given "dd." system property. */
   private static String ddGetEnv(final String sysProp) {
     return System.getenv(toEnvVar(sysProp));

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java

@@ -142,6 +142,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
       if (null != prefixRoot) {
         String entryKey = computeEntryKey(prefixRoot.relativize(file));
         if (null != entryKey) {
+          int existingPrefixId = prefixTrie.apply(entryKey);
+          if (-1 != existingPrefixId && prefixId != existingPrefixId) {
+            log.warn(
+                "Detected duplicate content under '{}'. Ensure your content is under a distinct directory.",
+                entryKey);
+          }
           prefixTrie.put(entryKey, prefixId);
           if (entryKey.endsWith("*")) {
             // optimization: wildcard will match everything under here so can skip

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java

@@ -1,7 +1,9 @@
 package datadog.trace.bootstrap.instrumentation.decorator;
 
 import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS;
+import static datadog.trace.api.datastreams.DataStreamsContext.fromTags;
 import static datadog.trace.api.gateway.Events.EVENTS;
+import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext;
 import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig;
 import static datadog.trace.bootstrap.instrumentation.decorator.http.HttpResourceDecorator.HTTP_RESOURCE_DECORATOR;
 
@@ -127,7 +129,7 @@ public AgentSpanContext.Extracted extract(REQUEST_CARRIER carrier) {
     if (null == carrier || null == getter) {
       return null;
     }
-    return tracer().propagate().extract(carrier, getter);
+    return extractContextAndGetSpanContext(carrier, getter);
   }
 
   /** Deprecated. Use {@link #startSpan(String, Object, AgentSpanContext.Extracted)} instead. */
@@ -148,7 +150,7 @@ public AgentSpan startSpan(
     }
     AgentPropagation.ContextVisitor<REQUEST_CARRIER> getter = getter();
     if (null != carrier && null != getter) {
-      tracer().getDataStreamsMonitoring().setCheckpoint(span, SERVER_PATHWAY_EDGE_TAGS, 0, 0);
+      tracer().getDataStreamsMonitoring().setCheckpoint(span, fromTags(SERVER_PATHWAY_EDGE_TAGS));
     }
     return span;
   }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/httpurlconnection/HeadersInjectAdapter.java

@@ -1,11 +1,11 @@
 package datadog.trace.bootstrap.instrumentation.httpurlconnection;
 
-import datadog.trace.bootstrap.instrumentation.api.AgentPropagation;
+import datadog.context.propagation.CarrierSetter;
 import java.net.HttpURLConnection;
 import javax.annotation.ParametersAreNonnullByDefault;
 
 @ParametersAreNonnullByDefault
-public class HeadersInjectAdapter implements AgentPropagation.Setter<HttpURLConnection> {
+public class HeadersInjectAdapter implements CarrierSetter<HttpURLConnection> {
 
   public static final HeadersInjectAdapter SETTER = new HeadersInjectAdapter();