Merge branch 'master' into andrea.marziali/complete-process-tags

Author: amarziali

.github/workflows/README.md

@@ -106,7 +106,7 @@ _Recovery:_ Manually trigger the action again.
 
 ### analyze-changes [🔗](analyze-changes.yaml)
 
-_Trigger:_ When pushing commits to `master` or any pull request targeting `master`.
+_Trigger:_ When pushing commits to `master`.
 
 _Action:_
 

.github/workflows/analyze-changes.yaml

@@ -61,6 +61,8 @@ jobs:
 
   trivy:
     name: Analyze changes with Trivy
+    # Don’t run on PR, only when pushing to master
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     permissions:
       actions: read

.gitlab-ci.yml

@@ -24,7 +24,7 @@ variables:
   BUILD_JOB_NAME: "build"
   DEPENDENCY_CACHE_POLICY: pull
   BUILD_CACHE_POLICY: pull
-  GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
+  GRADLE_VERSION: "8.5" # must match gradle-wrapper.properties
   MAVEN_REPOSITORY_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/maven-central/"
   GRADLE_PLUGIN_PROXY: "http://artifactual.artifactual.all-clusters.local-dc.fabric.dog:8081/repository/gradle-plugin-portal-proxy/"
   JAVA_BUILD_IMAGE_VERSION: "v25.05"

.gitlab/benchmarks.yml

@@ -14,7 +14,6 @@
     - when: on_success
   script:
     - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
-    - export CIRCLE_CI_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.circleci_token --with-decryption --query "Parameter.Value" --out text)
     - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
     - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
   artifacts:

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/sink/SymbolSink.java

@@ -105,16 +105,37 @@ public void flush() {
     String json =
         SERVICE_VERSION_ADAPTER.toJson(
             new ServiceVersion(serviceName, env, version, "JAVA", scopesToSerialize));
-    if (json.length() > maxPayloadSize) {
-      LOGGER.debug(
-          "Upload split is required for {} scopes: {}/{}",
-          scopesToSerialize.size(),
-          json.length(),
-          maxPayloadSize);
-      splitAndSend(scopesToSerialize);
+    updateStats(scopesToSerialize, json);
+    doUpload(scopesToSerialize, json);
+  }
+
+  private void doUpload(List<Scope> scopesToSerialize, String json) {
+    byte[] jsonBytes = json.getBytes(StandardCharsets.UTF_8);
+    byte[] payload = null;
+    if (isCompressed) {
+      payload = compressPayload(jsonBytes);
+    }
+    if (payload == null) {
+      if (json.length() > maxPayloadSize) {
+        LOGGER.warn("Payload is too big: {}/{}", json.length(), maxPayloadSize);
+        splitAndSend(scopesToSerialize);
+        return;
+      }
+      symbolUploader.uploadAsMultipart(
+          "",
+          event,
+          new BatchUploader.MultiPartContent(jsonBytes, "file", "file.json", APPLICATION_JSON));
     } else {
-      LOGGER.debug("Sending {} jar scopes size={}", scopesToSerialize.size(), json.length());
-      doUpload(scopesToSerialize, json);
+      if (payload.length > maxPayloadSize) {
+        LOGGER.warn("Compressed payload is too big: {}/{}", payload.length, maxPayloadSize);
+        splitAndSend(scopesToSerialize);
+        return;
+      }
+      LOGGER.debug("Sending {} jar scopes size={}", scopesToSerialize.size(), payload.length);
+      symbolUploader.uploadAsMultipart(
+          "",
+          event,
+          new BatchUploader.MultiPartContent(payload, "file", "file.gz", APPLICATION_GZIP));
     }
   }
 
@@ -146,16 +167,6 @@ private void splitAndSend(List<Scope> scopesToSerialize) {
               SERVICE_VERSION_ADAPTER.toJson(
                   new ServiceVersion(
                       serviceName, env, version, "JAVA", Collections.singletonList(scope)));
-          if (json.length() > maxPayloadSize) {
-            // this jar scope is still too big, split it by classes
-            LOGGER.debug(
-                "Upload split is required for jar scope {}: {}/{}",
-                scope.getName(),
-                json.length(),
-                maxPayloadSize);
-            splitAndSend(Collections.singletonList(scope));
-            continue;
-          }
           LOGGER.debug("Sending {} jar scope size={}", scope.getName(), json.length());
           doUpload(Collections.singletonList(scope), json);
         }
@@ -168,22 +179,10 @@ private void splitAndSend(List<Scope> scopesToSerialize) {
         String jsonFirstHalf =
             SERVICE_VERSION_ADAPTER.toJson(
                 new ServiceVersion(serviceName, env, version, "JAVA", firstHalf));
-        if (jsonFirstHalf.length() > maxPayloadSize) {
-          LOGGER.warn(
-              "Cannot split jar scope list in 2, first half is too big: {}",
-              jsonFirstHalf.length());
-          return;
-        }
         doUpload(firstHalf, jsonFirstHalf);
         String jsonSecondHalf =
             SERVICE_VERSION_ADAPTER.toJson(
                 new ServiceVersion(serviceName, env, version, "JAVA", secondHalf));
-        if (jsonSecondHalf.length() > maxPayloadSize) {
-          LOGGER.warn(
-              "Cannot split jar scope list in 2, second half is too big: {}",
-              jsonSecondHalf.length());
-          return;
-        }
         doUpload(secondHalf, jsonSecondHalf);
       }
     } else {
@@ -213,31 +212,6 @@ private void splitAndSend(List<Scope> scopesToSerialize) {
     }
   }
 
-  private void doUpload(List<Scope> scopes, String json) {
-    updateStats(scopes, json);
-    byte[] jsonBytes = json.getBytes(StandardCharsets.UTF_8);
-    byte[] payload = null;
-    if (isCompressed) {
-      payload = compressPayload(jsonBytes);
-    }
-    if (payload == null) {
-      if (jsonBytes.length > maxPayloadSize) {
-        LOGGER.warn("Compressed payload is too big: {}/{}", payload.length, maxPayloadSize);
-        splitAndSend(scopes);
-        return;
-      }
-      symbolUploader.uploadAsMultipart(
-          "",
-          event,
-          new BatchUploader.MultiPartContent(jsonBytes, "file", "file.json", APPLICATION_JSON));
-    } else {
-      symbolUploader.uploadAsMultipart(
-          "",
-          event,
-          new BatchUploader.MultiPartContent(payload, "file", "file.gz", APPLICATION_GZIP));
-    }
-  }
-
   private static Scope createJarScope(String jarName, List<Scope> classScopes) {
     return Scope.builder(ScopeType.JAR, jarName, 0, 0).name(jarName).scopes(classScopes).build();
   }

dd-java-agent/agent-debugger/src/main/resources/third_party_libraries.json

@@ -14,6 +14,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import okhttp3.MediaType;
 import org.junit.jupiter.api.Test;
 
 class SymbolSinkTest {
@@ -202,6 +203,31 @@ public void splitByClassScopesImpossible() {
     assertTrue(symbolUploaderMock.multiPartContents.isEmpty());
   }
 
+  @Test
+  public void maxCompressedAndSplit() {
+    SymbolUploaderMock symbolUploaderMock = new SymbolUploaderMock();
+    Config config = mock(Config.class);
+    when(config.getServiceName()).thenReturn("service1");
+    when(config.isSymbolDatabaseCompressed()).thenReturn(true);
+    SymbolSink symbolSink = new SymbolSink(config, symbolUploaderMock, 512);
+    final int NUM_JAR_SCOPES = 100;
+    for (int i = 0; i < NUM_JAR_SCOPES; i++) {
+      symbolSink.addScope(
+          Scope.builder(ScopeType.JAR, "jar" + i + ".jar", 0, 0)
+              .scopes(singletonList(Scope.builder(ScopeType.CLASS, "class" + i, 0, 0).build()))
+              .build());
+    }
+    symbolSink.flush();
+    assertEquals(4, symbolUploaderMock.multiPartContents.size());
+    for (int i = 0; i < 4; i += 2) {
+      BatchUploader.MultiPartContent eventContent = symbolUploaderMock.multiPartContents.get(i);
+      assertEquals("event", eventContent.getPartName());
+      BatchUploader.MultiPartContent symbolContent =
+          symbolUploaderMock.multiPartContents.get(i + 1);
+      assertEquals(MediaType.get("application/gzip"), symbolContent.getMediaType());
+    }
+  }
+
   private static String assertMultipartContent(SymbolUploaderMock symbolUploaderMock, int index) {
     BatchUploader.MultiPartContent eventContent = symbolUploaderMock.multiPartContents.get(index);
     assertEquals("event", eventContent.getPartName());

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/sink/SymbolSinkTest.java

@@ -27,7 +27,17 @@ final testTasks = scalaVersions.collect { scalaLibrary ->
   def (major, minor) = version.split('_').collect(Integer.&valueOf)
   final javaConcatenation = major > 2 || minor > 11 // after 2.11 scala uses java.lang.StringBuilder to perform concatenation
 
-  final configuration = configurations.create("${version}Implementation")
+  final configuration = configurations.create("${version}Implementation") {
+    canBeConsumed = false
+    canBeResolved = false
+    canBeDeclared = true
+  }
+  final classPathConfiguration = configurations.create("${version}CompileClasspath") {
+    canBeConsumed = false
+    canBeResolved = true
+    canBeDeclared = false
+    extendsFrom(configuration)
+  }
 
   dependencies { handler ->
     handler.add(configuration.name, scalaLibrary)
@@ -40,7 +50,7 @@ final testTasks = scalaVersions.collect { scalaLibrary ->
   final customSourceSet = sourceSets.create("${version}") {
     scala {
       srcDirs = ['src/test/scala']
-      compileClasspath += configuration
+      compileClasspath += classPathConfiguration
     }
   }
 
@@ -49,7 +59,7 @@ final testTasks = scalaVersions.collect { scalaLibrary ->
       .filter { !it.toString().contains('scala-library') }   // exclude default scala-library
       .minus(files(sourceSets.test.scala.classesDirectory))  // exclude default /build/classes/scala/test folder
       .plus(customSourceSet.output.classesDirs)              // add /build/classes/scala/${version} folder
-      .plus(configuration)                                   // add new scala-library configuration
+      .plus(classPathConfiguration)                                   // add new scala-library configuration
     systemProperty('uses.java.concat', javaConcatenation)
     dependsOn(tasks.named("compile${version.capitalize()}Scala"))
     group = 'verification'

dd-java-agent/instrumentation/scala/build.gradle

@@ -37,9 +37,10 @@ dependencies {
   baseTestImplementation group: 'org.apache.spark', name: "spark-core_2.12", version: "2.4.0"
   baseTestImplementation group: 'org.apache.spark', name: "spark-sql_2.12", version: "2.4.0"
 
-  latest212DepTestImplementation group: 'org.apache.spark', name: "spark-core_2.12", version: '3.+'
-  latest212DepTestImplementation group: 'org.apache.spark', name: "spark-sql_2.12", version: '3.+'
+  // FIXME: 3.6.0 seems missing from central
+  latest212DepTestImplementation group: 'org.apache.spark', name: "spark-core_2.12", version: '3.5.5'
+  latest212DepTestImplementation group: 'org.apache.spark', name: "spark-sql_2.12", version: '3.5.5'
 
-  latest213DepTestImplementation group: 'org.apache.spark', name: "spark-core_2.13", version: '3.+'
-  latest213DepTestImplementation group: 'org.apache.spark', name: "spark-sql_2.13", version: '3.+'
+  latest213DepTestImplementation group: 'org.apache.spark', name: "spark-core_2.13", version: '3.5.5'
+  latest213DepTestImplementation group: 'org.apache.spark', name: "spark-sql_2.13", version: '3.5.5'
 }

dd-java-agent/instrumentation/spark-executor/build.gradle

@@ -52,9 +52,10 @@ dependencies {
   test_spark32Implementation group: 'org.apache.spark', name: "spark-yarn_$scalaVersion", version: "3.2.4"
 
   // FIXME: Currently not working on Spark 4.0.0 preview releases.
-  latestDepTestImplementation group: 'org.apache.spark', name: "spark-core_$scalaVersion", version: '3.+'
-  latestDepTestImplementation group: 'org.apache.spark', name: "spark-sql_$scalaVersion", version: '3.+'
-  latestDepTestImplementation group: 'org.apache.spark', name: "spark-yarn_$scalaVersion", version: '3.+'
+  // FIXME: 3.6.0 seems missing from central
+  latestDepTestImplementation group: 'org.apache.spark', name: "spark-core_$scalaVersion", version: '3.5.5'
+  latestDepTestImplementation group: 'org.apache.spark', name: "spark-sql_$scalaVersion", version: '3.5.5'
+  latestDepTestImplementation group: 'org.apache.spark', name: "spark-yarn_$scalaVersion", version: '3.5.5'
 }
 
 tasks.named("test").configure {