Merge branch 'master' into nicholas.hulston/pass-error-tags-to-lambda-extension

Author: nhulston

.circleci/config.continue.yml.j2

@@ -858,6 +858,9 @@ jobs:
               APPSEC_RASP
               APPSEC_RUNTIME_ACTIVATION
               APPSEC_WAF_TELEMETRY
+              APPSEC_STANDALONE_V2
+              IAST_STANDALONE_V2
+              SCA_STANDALONE_V2
               REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
               "
             fi

.gitlab-ci.yml

@@ -23,15 +23,24 @@ variables:
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
+  JAVA_BUILD_IMAGE_VERSION: "v25.01"
 
 default:
   tags: [ "arch:amd64" ]
 
+.fan_in:
+  stage: tests
+  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204-slim:release
+  script:
+    - echo "done"
+
 .gradle_build: &gradle_build
-  image: ghcr.io/datadog/dd-trace-java-docker-build:v25.01-base
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
+  stage: build
   variables:
-    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
-    GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
+    GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    MAVEN_OPTS: "-Xms64M -Xmx512M"
+    GRADLE_WORKERS: 2
     KUBERNETES_CPU_REQUEST: 8
     KUBERNETES_MEMORY_REQUEST: 6Gi
   cache:
@@ -42,14 +51,15 @@ default:
         - .gradle/caches
         - .gradle/notifications
       policy: $DEPENDENCY_CACHE_POLICY
-    - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
+    - key: $CI_PIPELINE_ID-$BUILD_CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
       paths:
         - .gradle/caches/$GRADLE_VERSION
         - .gradle/$GRADLE_VERSION/executionHistory
         - workspace
       policy: $BUILD_CACHE_POLICY
   before_script:
     - export GRADLE_USER_HOME=`pwd`/.gradle
+    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
     # for weird reasons, gradle will always "chmod 700" the .gradle folder
     # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails
     # This dance is a hack to have .gradle owned by the Gitlab runner user
@@ -61,21 +71,24 @@ default:
 
 build:
   extends: .gradle_build
-  stage: build
   variables:
     BUILD_CACHE_POLICY: push
+    BUILD_CACHE_TYPE: lib
     DEPENDENCY_CACHE_POLICY: pull
   script:
-    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
+    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks; fi
+    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS
     - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
     - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
     - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
   artifacts:
+    when: always
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'
       - 'workspace/dd-trace-api/build/libs/*.jar'
       - 'workspace/dd-trace-ot/build/libs/*.jar'
       - 'upstream.env'
+      - '.gradle/daemon/*/*.out.log'
     reports:
       dotenv: build.env
 
@@ -90,6 +103,94 @@ build_and_populate_dep_cache:
     - when: manual
       allow_failure: true
 
+spotless:
+  extends: .gradle_build
+  stage: tests
+  needs: []
+  script:
+    - export JAVA_HOME=$JAVA_11_HOME
+    - ./gradlew spotlessCheck $GRADLE_ARGS
+
+test_published_artifacts:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests
+  stage: tests
+  needs: [ build ]
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: lib
+  script:
+    - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+    - rm -rf "${mvn_local_repo}/com/datadoghq"
+    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
+    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew publishToMavenLocal $GRADLE_ARGS
+    - cd test-published-dependencies
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+    - ./gradlew check --info $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./check_reports
+
+muzzle:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  parallel: 8
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: on_success
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - echo CI_NODE_INDEX=$CI_NODE_INDEX
+    - echo CI_NODE_TOTAL=CI_NODE_TOTAL
+    - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS
+    - sort workspace/build/muzzleTasks > sortedMuzzleTasks
+    - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit
+    - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1))
+    - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_reports.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+muzzle-dep-report:
+  extends: .gradle_build
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: test
+  script:
+    - export SKIP_BUILDSCAN="true"
+    - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
+  after_script:
+    - .circleci/collect_muzzle_deps.sh
+  artifacts:
+    when: always
+    paths:
+      - ./reports
+      - '.gradle/daemon/*/*.out.log'
+
+required:
+  extends: .fan_in
+  needs:
+    - spotless
+    - muzzle
+    - test_published_artifacts
+
 deploy_to_profiling_backend:
   stage: publish
   needs: [ build ]
@@ -165,6 +266,8 @@ deploy_to_sonatype:
   extends: .gradle_build
   stage: publish
   needs: [ build ]
+  variables:
+    BUILD_CACHE_TYPE: lib
   rules:
     - if: '$POPULATE_CACHE'
       when: never
@@ -180,7 +283,7 @@ deploy_to_sonatype:
     - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
     - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
-    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository $GRADLE_ARGS
+    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
   artifacts:
     paths:
       - 'workspace/dd-java-agent/build/libs/*.jar'

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java

@@ -116,7 +116,7 @@ private static boolean copyOOMEscript(Path scriptPath) {
   private static class ScriptCleanupVisitor implements FileVisitor<Path> {
     private static final Pattern PID_PATTERN = Pattern.compile(".*?" + PID_PREFIX + "(\\d+)");
 
-    private final Set<String> pidSet = PidHelper.getJavaPids();
+    private Set<String> pidSet;
 
     static void run(Path dir) {
       try {
@@ -145,9 +145,14 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
       Matcher matcher = PID_PATTERN.matcher(fileName);
       if (matcher.find()) {
         String pid = matcher.group(1);
-        if (pid != null && !pid.equals(PidHelper.getPid()) && !this.pidSet.contains(pid)) {
-          LOG.debug("Cleaning process specific file {}", file);
-          Files.delete(file);
+        if (pid != null && !pid.equals(PidHelper.getPid())) {
+          if (this.pidSet == null) {
+            this.pidSet = PidHelper.getJavaPids(); // only fork jps when required
+          }
+          if (!this.pidSet.contains(pid)) {
+            LOG.debug("Cleaning process specific file {}", file);
+            Files.delete(file);
+          }
         }
       }
       return CONTINUE;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java

@@ -701,15 +701,19 @@ public void commit(CapturedContext lineContext, int line) {
       shouldCommit = true;
     }
     if (shouldCommit) {
-      if (isCaptureSnapshot()) {
-        // freeze context just before commit because line probes have only one context
-        Duration timeout =
-            Duration.of(Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
-        lineContext.freeze(new TimeoutChecker(timeout));
-        snapshot.addLine(lineContext, line);
+      incrementBudget();
+      if (inBudget()) {
+        if (isCaptureSnapshot()) {
+          // freeze context just before commit because line probes have only one context
+          Duration timeout =
+              Duration.of(
+                  Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
+          lineContext.freeze(new TimeoutChecker(timeout));
+          snapshot.addLine(lineContext, line);
+        }
+        commitSnapshot(snapshot, sink);
+        return;
       }
-      commitSnapshot(snapshot, sink);
-      return;
     }
     sink.skipSnapshot(id, DebuggerContext.SkipCause.CONDITION);
   }

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/probe/LogProbeTest.java

@@ -102,7 +102,34 @@ public void budgets() {
     assertEquals(runs * LogProbe.NON_CAPTURING_PROBE_BUDGET, sink.highRate);
   }
 
+  @Test
+  public void budgetsOnLineProbes() {
+    BudgetSink sink = new BudgetSink(getConfig(), mock(ProbeStatusSink.class));
+    DebuggerAgentHelper.injectSink(sink);
+
+    TracerAPI tracer =
+        CoreTracer.builder().idGenerationStrategy(IdGenerationStrategy.fromName("random")).build();
+    AgentTracer.registerIfAbsent(tracer);
+    int runs = 100;
+    for (int i = 0; i < runs; i++) {
+      runTrace(tracer, true, 100);
+    }
+    assertEquals(runs * LogProbe.CAPTURING_PROBE_BUDGET, sink.captures);
+
+    sink = new BudgetSink(getConfig(), mock(ProbeStatusSink.class));
+    DebuggerAgentHelper.injectSink(sink);
+    runs = 1010;
+    for (int i = 0; i < runs; i++) {
+      runTrace(tracer, false, 100);
+    }
+    assertEquals(runs * LogProbe.NON_CAPTURING_PROBE_BUDGET, sink.highRate);
+  }
+
   private void runTrace(TracerAPI tracer, boolean captureSnapshot) {
+    runTrace(tracer, captureSnapshot, null);
+  }
+
+  private void runTrace(TracerAPI tracer, boolean captureSnapshot, Integer line) {
     AgentSpan span = tracer.startSpan("budget testing", "test span");
     span.setTag(Tags.PROPAGATED_DEBUG, "12345:1");
     try (AgentScope scope = tracer.activateSpan(span, ScopeSource.MANUAL)) {
@@ -125,7 +152,11 @@ private void runTrace(TracerAPI tracer, boolean captureSnapshot) {
       int runs = budget + 20;
 
       for (int i = 0; i < runs; i++) {
-        logProbe.commit(entryContext, exitContext, emptyList());
+        if (line == null) {
+          logProbe.commit(entryContext, exitContext, emptyList());
+        } else {
+          logProbe.commit(entryContext, line);
+        }
       }
       assertEquals(runs, span.getLocalRootSpan().getTag(format("_dd.ld.probe_id.%s", logProbe.id)));
     }

dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/TempLocationManager.java

@@ -73,7 +73,7 @@ default void onCleanupStart(boolean selfCleanup, long timeout, TimeUnit unit) {}
   private final class CleanupVisitor implements FileVisitor<Path> {
     private boolean shouldClean;
 
-    private final Set<String> pidSet = PidHelper.getJavaPids();
+    private Set<String> pidSet;
 
     private final boolean cleanSelf;
     private final Instant cutoff;
@@ -120,7 +120,14 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
       // the JFR repository directories are under <basedir>/pid_<pid>
       String pid = fileName.startsWith(TEMPDIR_PREFIX) ? fileName.substring(4) : null;
       boolean isSelfPid = pid != null && pid.equals(PidHelper.getPid());
-      shouldClean |= cleanSelf ? isSelfPid : !isSelfPid && !pidSet.contains(pid);
+      if (cleanSelf) {
+        shouldClean |= isSelfPid;
+      } else if (!isSelfPid) {
+        if (pidSet == null) {
+          pidSet = PidHelper.getJavaPids(); // only fork jps when required
+        }
+        shouldClean |= !pidSet.contains(pid);
+      }
       if (shouldClean) {
         log.debug("Cleaning temporary location {}", dir);
       }
@@ -294,7 +301,7 @@ private TempLocationManager() {
     tempDir = baseTempDir.resolve(TEMPDIR_PREFIX + pid);
     if (runStartupCleanup) {
       // do not execute the background cleanup task when running in tests
-      AgentTaskScheduler.INSTANCE.execute(() -> cleanup(false));
+      AgentTaskScheduler.INSTANCE.execute(cleanupTask);
     }
 
     Thread selfCleanup =

dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/TempLocationManagerTest.java

@@ -239,7 +239,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
     Files.createFile(otherTempdir.resolve("dummy"));
     boolean rslt =
         instance.cleanup(
-            selfCleanup, (long) (timeoutMs * (shouldSucceed ? 10 : 0.5d)), TimeUnit.MILLISECONDS);
+            selfCleanup, (long) (timeoutMs * (shouldSucceed ? 20 : 0.5d)), TimeUnit.MILLISECONDS);
     assertEquals(shouldSucceed, rslt);
   }
 

dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/util/JfrCliHelper.java

@@ -2,6 +2,7 @@
 
 import static datadog.trace.util.AgentThreadFactory.AgentThread.PROFILER_HTTP_DISPATCHER;
 
+import datadog.trace.api.Platform;
 import datadog.trace.api.profiling.RecordingData;
 import datadog.trace.relocate.api.IOLogger;
 import datadog.trace.util.AgentThreadFactory;
@@ -44,7 +45,7 @@ public static void invokeOn(final RecordingData data, final IOLogger ioLogger) {
     File tmp = null;
     try {
       Path jfr = Paths.get(System.getProperty("java.home"), "bin", "jfr");
-      if (!Files.exists(jfr)) {
+      if (Platform.isJ9() || !Files.exists(jfr)) {
         ioLogger.error("Failed to gather information on recording, can't find `jfr`");
         return;
       }

dd-java-agent/agent-profiling/profiling-uploader/src/test/java/com/datadog/profiling/uploader/ProfileUploaderTest.java

@@ -42,6 +42,7 @@
 import datadog.common.version.VersionInfo;
 import datadog.trace.api.Config;
 import datadog.trace.api.DDTags;
+import datadog.trace.api.Platform;
 import datadog.trace.api.profiling.ProfilingSnapshot;
 import datadog.trace.api.profiling.RecordingData;
 import datadog.trace.api.profiling.RecordingInputStream;
@@ -561,7 +562,10 @@ public void test413Response() throws Exception {
 
     verify(recording).release();
 
-    if (Files.exists(Paths.get(System.getProperty("java.home"), "bin", "jfr"))) {
+    // J9 has 'almost' implemented JFR, but not really
+    // we need to skip this part for J9
+    if (!Platform.isJ9()
+        && Files.exists(Paths.get(System.getProperty("java.home"), "bin", "jfr"))) {
       verify(ioLogger)
           .error(
               eq("Failed to upload profile, it's too big. Dumping information about the profile"));