Fix Jackson nodes introspection for request/response schema extraction

dd-java-agent/appsec/build.gradle

@@ -24,6 +24,7 @@ dependencies {
   testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.2'
   testImplementation group: 'com.flipkart.zjsonpatch', name: 'zjsonpatch', version: '0.4.11'
   testImplementation libs.logback.classic
+  testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.16.0'
 
   testFixturesApi project(':dd-java-agent:testing')
 }

dd-java-agent/appsec/src/main/java/com/datadog/appsec/event/data/ObjectIntrospection.java

@@ -3,13 +3,16 @@
 import com.datadog.appsec.gateway.AppSecRequestContext;
 import datadog.trace.api.Platform;
 import datadog.trace.api.telemetry.WafMetricCollector;
+import datadog.trace.util.MethodHandles;
+import java.lang.invoke.MethodHandle;
 import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import org.slf4j.Logger;
@@ -178,6 +181,19 @@ private static Object doConversion(Object obj, int depth, State state) {
       return obj.toString();
     }
 
+    // Jackson databind nodes (via reflection)
+    Class<?> clazz = obj.getClass();
+    if (clazz.getName().startsWith("com.fasterxml.jackson.databind.node.")) {
+      try {
+        return doConversionJacksonNode(
+            new JacksonContext(clazz.getClassLoader()), obj, depth, state);
+      } catch (Throwable e) {
+        // in case of failure let default conversion run
+        log.debug("Error handling jackson node {}", clazz, e);
+        return null;
+      }
+    }
+
     // maps
     if (obj instanceof Map) {
       Map<Object, Object> newMap = new HashMap<>((int) Math.ceil(((Map) obj).size() / .75));
@@ -212,7 +228,6 @@ private static Object doConversion(Object obj, int depth, State state) {
     }
 
     // arrays
-    Class<?> clazz = obj.getClass();
     if (clazz.isArray()) {
       int length = Array.getLength(obj);
       List<Object> newList = new ArrayList<>(length);
@@ -305,4 +320,139 @@ private static String checkStringLength(final String str, final State state) {
     }
     return str;
   }
+
+  /**
+   * Converts Jackson databind JsonNode objects to WAF-compatible data structures using reflection.
+   *
+   * <p>Jackson databind objects ({@link com.fasterxml.jackson.databind.JsonNode}) implement
+   * iterable interfaces which interferes with the standard object introspection logic. This method
+   * bypasses that by using reflection to directly access JsonNode internals and convert them to
+   * appropriate data types.
+   *
+   * <p>Supported JsonNode types and their conversions:
+   *
+   * <ul>
+   *   <li>{@code OBJECT} - Converted to {@link HashMap} with string keys and recursively converted
+   *       values
+   *   <li>{@code ARRAY} - Converted to {@link ArrayList} with recursively converted elements
+   *   <li>{@code STRING} - Extracted as {@link String}, subject to length truncation
+   *   <li>{@code NUMBER} - Extracted as the appropriate {@link Number} subtype (Integer, Long,
+   *       Double, etc.)
+   *   <li>{@code BOOLEAN} - Extracted as {@link Boolean}
+   *   <li>{@code NULL}, {@code MISSING}, {@code BINARY}, {@code POJO} - Converted to {@code null}
+   * </ul>
+   *
+   * <p>The method applies the same truncation limits as the main conversion logic:
+   */
+  private static Object doConversionJacksonNode(
+      final JacksonContext ctx, final Object node, final int depth, final State state)
+      throws Throwable {
+    if (node == null) {
+      return null;
+    }
+    state.elemsLeft--;
+    if (state.elemsLeft <= 0) {
+      state.listMapTooLarge = true;
+      return null;
+    }
+    if (depth > MAX_DEPTH) {
+      state.objectTooDeep = true;
+      return null;
+    }
+    final String type = ctx.getNodeType(node);
+    if (type == null) {
+      return null;
+    }
+    switch (type) {
+      case "OBJECT":
+        final Map<Object, Object> newMap = new HashMap<>(ctx.getSize(node));
+        for (Iterator<String> names = ctx.getFieldNames(node); names.hasNext(); ) {
+          final String key = names.next();
+          final Object newKey = keyConversion(key, state);
+          if (newKey == null && key != null) {
+            // probably we're out of elements anyway
+            continue;
+          }
+          final Object value = ctx.getField(node, key);
+          newMap.put(newKey, doConversionJacksonNode(ctx, value, depth + 1, state));
+        }
+        return newMap;
+      case "ARRAY":
+        final List<Object> newList = new ArrayList<>(ctx.getSize(node));
+        for (Object o : ((Iterable<?>) node)) {
+          if (state.elemsLeft <= 0) {
+            state.listMapTooLarge = true;
+            break;
+          }
+          newList.add(doConversionJacksonNode(ctx, o, depth + 1, state));
+        }
+        return newList;
+      case "BOOLEAN":
+        return ctx.getBooleanValue(node);
+      case "NUMBER":
+        return ctx.getNumberValue(node);
+      case "STRING":
+        return checkStringLength(ctx.getTextValue(node), state);
+      default:
+        // return null for the rest
+        return null;
+    }
+  }
+
+  /**
+   * Context class used to cache method resolutions while converting a top level json node class.
+   */
+  private static class JacksonContext {
+    private final MethodHandles handles;
+    private final Class<?> jsonNode;
+    private MethodHandle nodeType;
+    private MethodHandle size;
+    private MethodHandle fieldNames;
+    private MethodHandle fieldValue;
+    private MethodHandle textValue;
+    private MethodHandle booleanValue;
+    private MethodHandle numberValue;
+
+    private JacksonContext(final ClassLoader cl) throws ClassNotFoundException {
+      handles = new MethodHandles(cl);
+      jsonNode = cl.loadClass("com.fasterxml.jackson.databind.JsonNode");
+    }
+
+    private String getNodeType(final Object node) throws Throwable {
+      nodeType = nodeType == null ? handles.method(jsonNode, "getNodeType") : nodeType;
+      final Enum<?> type = (Enum<?>) nodeType.invoke(node);
+      return type == null ? null : type.name();
+    }
+
+    private int getSize(final Object node) throws Throwable {
+      size = size == null ? handles.method(jsonNode, "size") : size;
+      return (int) size.invoke(node);
+    }
+
+    @SuppressWarnings("unchecked")
+    private Iterator<String> getFieldNames(final Object node) throws Throwable {
+      fieldNames = fieldNames == null ? handles.method(jsonNode, "fieldNames") : fieldNames;
+      return (Iterator<String>) fieldNames.invoke(node);
+    }
+
+    private Object getField(final Object node, final String name) throws Throwable {
+      fieldValue = fieldValue == null ? handles.method(jsonNode, "get", String.class) : fieldValue;
+      return fieldValue.invoke(node, name);
+    }
+
+    private String getTextValue(final Object node) throws Throwable {
+      textValue = textValue == null ? handles.method(jsonNode, "textValue") : textValue;
+      return (String) textValue.invoke(node);
+    }
+
+    private Number getNumberValue(final Object node) throws Throwable {
+      numberValue = numberValue == null ? handles.method(jsonNode, "numberValue") : numberValue;
+      return (Number) numberValue.invoke(node);
+    }
+
+    private Boolean getBooleanValue(final Object node) throws Throwable {
+      booleanValue = booleanValue == null ? handles.method(jsonNode, "booleanValue") : booleanValue;
+      return (Boolean) booleanValue.invoke(node);
+    }
+  }
 }

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/event/data/ObjectIntrospectionSpecification.groovy

@@ -1,8 +1,14 @@
 package com.datadog.appsec.event.data
 
 import com.datadog.appsec.gateway.AppSecRequestContext
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.node.ArrayNode
+import com.fasterxml.jackson.databind.node.ObjectNode
 import datadog.trace.api.telemetry.WafMetricCollector
 import datadog.trace.test.util.DDSpecification
+import groovy.json.JsonBuilder
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
 import spock.lang.Shared
 
 import java.nio.CharBuffer
@@ -14,6 +20,9 @@ class ObjectIntrospectionSpecification extends DDSpecification {
   @Shared
   protected static final ORIGINAL_METRIC_COLLECTOR = WafMetricCollector.get()
 
+  @Shared
+  protected static final MAPPER = new ObjectMapper()
+
   AppSecRequestContext ctx = Mock(AppSecRequestContext)
 
   WafMetricCollector wafMetricCollector = Mock(WafMetricCollector)
@@ -318,4 +327,152 @@ class ObjectIntrospectionSpecification extends DDSpecification {
     1 * wafMetricCollector.wafInputTruncated(true, false, false)
     1 * listener.onTruncation()
   }
+
+  void 'jackson node types comprehensive coverage'() {
+    when:
+    final result = convert(input, ctx)
+
+    then:
+    result == expected
+
+    where:
+    input                                          || expected
+    MAPPER.readTree('null')                        || null
+    MAPPER.readTree('true')                        || true
+    MAPPER.readTree('false')                       || false
+    MAPPER.readTree('42')                          || 42
+    MAPPER.readTree('3.14')                        || 3.14
+    MAPPER.readTree('"hello"')                     || 'hello'
+    MAPPER.readTree('[]')                          || []
+    MAPPER.readTree('{}')                          || [:]
+    MAPPER.readTree('[1, 2, 3]')                   || [1, 2, 3]
+    MAPPER.readTree('{"key": "value"}')            || [key: 'value']
+  }
+
+  void 'jackson nested structures'() {
+    when:
+    final result = convert(input, ctx)
+
+    then:
+    result == expected
+
+    where:
+    input                                          || expected
+    MAPPER.readTree('{"a": {"b": {"c": 123}}}')    || [a: [b: [c: 123]]]
+    MAPPER.readTree('[[[1, 2]], [[3, 4]]]')        || [[[1, 2]], [[3, 4]]]
+    MAPPER.readTree('{"arr": [1, null, true]}')    || [arr: [1, null, true]]
+    MAPPER.readTree('[{"x": 1}, {"y": 2}]')        || [[x: 1], [y: 2]]
+  }
+
+  void 'jackson edge cases'() {
+    when:
+    final result = convert(input, ctx)
+
+    then:
+    result == expected
+
+    where:
+    input                                          || expected
+    MAPPER.readTree('""')                          || ''
+    MAPPER.readTree('0')                           || 0
+    MAPPER.readTree('-1')                          || -1
+    MAPPER.readTree('9223372036854775807')         || 9223372036854775807L  // Long.MAX_VALUE
+    MAPPER.readTree('1.7976931348623157E308')      || 1.7976931348623157E308d  // Double.MAX_VALUE
+    MAPPER.readTree('{"": "empty_key"}')           || ['': 'empty_key']
+    MAPPER.readTree('{"null_value": null}')        || [null_value: null]
+  }
+
+  void 'jackson string truncation'() {
+    setup:
+    final longString = 'A' * (ObjectIntrospection.MAX_STRING_LENGTH + 1)
+    final jsonInput = '{"long": "' + longString + '"}'
+
+    when:
+    final result = convert(MAPPER.readTree(jsonInput), ctx)
+
+    then:
+    1 * ctx.setWafTruncated()
+    1 * wafMetricCollector.wafInputTruncated(true, false, false)
+    result["long"].length() <= ObjectIntrospection.MAX_STRING_LENGTH
+  }
+
+  void 'jackson with deep nesting triggers depth limit'() {
+    setup:
+    // Create deeply nested JSON
+    final json = JsonOutput.toJson(
+    (1..(ObjectIntrospection.MAX_DEPTH + 1)).inject([:], { result, i -> [("child_$i".toString()) : result] })
+    )
+
+    when:
+    final result = convert(MAPPER.readTree(json), ctx)
+
+    then:
+    // Should truncate at max depth and set truncation flag
+    1 * ctx.setWafTruncated()
+    1 * wafMetricCollector.wafInputTruncated(false, false, true)
+    countNesting(result as Map, 0) <= ObjectIntrospection.MAX_DEPTH
+  }
+
+  void 'jackson with large arrays triggers element limit'() {
+    setup:
+    // Create large array
+    final largeArray = (1..(ObjectIntrospection.MAX_ELEMENTS + 1)).toList()
+    final json = new JsonBuilder(largeArray).toString()
+
+    when:
+    final result = convert(MAPPER.readTree(json), ctx) as List
+
+    then:
+    // Should truncate and set truncation flag
+    1 * ctx.setWafTruncated()
+    1 * wafMetricCollector.wafInputTruncated(false, true, false)
+    result.size() <= ObjectIntrospection.MAX_ELEMENTS
+  }
+
+  void 'jackson number type variations'() {
+    when:
+    final result = convert(input, ctx)
+
+    then:
+    result == expected
+
+    where:
+    input                                          || expected
+    MAPPER.readTree('0')                           || 0
+    MAPPER.readTree('1')                           || 1
+    MAPPER.readTree('-1')                          || -1
+    MAPPER.readTree('1.0')                         || 1.0
+    MAPPER.readTree('1.5')                         || 1.5
+    MAPPER.readTree('-1.5')                        || -1.5
+    MAPPER.readTree('1e10')                        || 1e10
+    MAPPER.readTree('1.23e-4')                     || 1.23e-4
+  }
+
+  void 'jackson special string values'() {
+    when:
+    final result = convert(input, ctx)
+
+    then:
+    result == expected
+
+    where:
+    input                                          || expected
+    MAPPER.readTree('"\\n"')                       || '\n'
+    MAPPER.readTree('"\\t"')                       || '\t'
+    MAPPER.readTree('"\\r"')                       || '\r'
+    MAPPER.readTree('"\\\\"')                      || '\\'
+    MAPPER.readTree('"\\"quotes\\""')              || '"quotes"'
+    MAPPER.readTree('"unicode: \\u0041"')          || 'unicode: A'
+  }
+
+  private static int countNesting(final Map<String, Object>object, final int levels) {
+    if (object.isEmpty()) {
+      return levels
+    }
+    final child = object.values().first()
+    if (child == null) {
+      return levels
+    }
+    return countNesting(object.values().first() as Map, levels + 1)
+  }
 }

dd-smoke-tests/appsec/springboot/src/main/java/datadog/smoketest/appsec/springboot/controller/WebController.java

@@ -1,5 +1,6 @@
 package datadog.smoketest.appsec.springboot.controller;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.squareup.okhttp.OkHttpClient;
 import com.squareup.okhttp.Request;
 import datadog.smoketest.appsec.springboot.service.AsyncService;
@@ -18,6 +19,7 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -211,6 +213,11 @@ public ResponseEntity<String> apiSecuritySampling(@PathVariable("status_code") i
     return ResponseEntity.status(statusCode).body("EXECUTED");
   }
 
+  @PostMapping(value = "/api_security/jackson", consumes = MediaType.APPLICATION_JSON_VALUE)
+  public ResponseEntity<JsonNode> apiSecurityJackson(@RequestBody final JsonNode body) {
+    return ResponseEntity.status(200).body(body);
+  }
+
   @GetMapping("/custom-headers")
   public ResponseEntity<String> customHeaders() {
     HttpHeaders headers = new HttpHeaders();