From 2346ba4f5f79c510437ab6f8f69ff8736b99077f Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Thu, 28 Nov 2024 15:57:34 +0100 Subject: [PATCH 01/20] improved extraction of user-data bytes for JSON content type --- .../internal/serde/InternalSerdeImpl.java | 2 ++ .../internal/serde/UserDataDeserializer.java | 27 ++++++++++++++++++- .../java/com/arangodb/ArangoDatabaseTest.java | 21 +++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index c51e50ab6..d24aa30df 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -9,6 +9,7 @@ import com.arangodb.util.RawJson; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; @@ -38,6 +39,7 @@ final class InternalSerdeImpl implements InternalSerde { this.userSerde = userSerde; mapper.deactivateDefaultTyping(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.enable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION); mapper.registerModule(InternalModule.INSTANCE.get()); if (protocolModule != null) { mapper.registerModule(protocolModule); diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java index 91220088b..e8b9641dc 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java @@ -1,6 +1,7 @@ package com.arangodb.internal.serde; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JavaType; @@ -10,6 +11,7 @@ import java.io.IOException; import java.lang.reflect.Type; +import java.util.Arrays; import static com.arangodb.internal.serde.SerdeUtils.convertToType; @@ -29,7 +31,7 @@ private UserDataDeserializer(final JavaType targetType, final InternalSerde serd @Override public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return serde.deserializeUserData(p.readValueAsTree(), targetType); + return serde.deserializeUserData(extractBytes(p), targetType); } @Override @@ -41,4 +43,27 @@ public Object deserializeWithType(JsonParser p, DeserializationContext ctxt, Typ public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) { return new UserDataDeserializer(ctxt.getContextualType(), serde); } + + private byte[] extractBytes(JsonParser parser) throws IOException { + JsonToken t = parser.currentToken(); + if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { + throw new RuntimeException("Unexpected token: " + t); + } + byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); + int start = (int) parser.currentTokenLocation().getByteOffset(); + if (t.isStructStart()) { + int open = 1; + while (open > 0) { + t = parser.nextToken(); + if (t.isStructStart()) { + open++; + } else if (t.isStructEnd()) { + open--; + } + } + } + parser.finishToken(); + int end = (int) parser.currentLocation().getByteOffset(); + return Arrays.copyOfRange(data, start, end); + } } diff --git a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java index 0cfd5b63c..c8ee786e3 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java @@ -768,6 +768,27 @@ void queryRawBytes(ArangoDatabase db) { assertThat(data.get("value").numberValue()).isEqualTo(1); } + @ParameterizedTest + @MethodSource("dbs") + void queryUserDataScalar(ArangoDatabase db) { + List docs = Arrays.asList("a", "b", "c"); + ArangoCursor res = db.query("FOR d IN @docs RETURN d", String.class, + Collections.singletonMap("docs", docs), new AqlQueryOptions().batchSize(1)); + assertThat((Iterable) res).contains("a", "b", "c"); + } + + @ParameterizedTest + @MethodSource("dbs") + void queryUserDataStruct(ArangoDatabase db) { + RawJson a = RawJson.of("\"foo\""); + RawJson b = RawJson.of("{\"key\":\"value\"}"); + RawJson c = RawJson.of("[1,null,true,\"bla\",{},[],\"\"]"); + RawJson docs = RawJson.of("[" + a.get() + "," + b.get() + "," + c.get() + "]"); + ArangoCursor res = db.query("FOR d IN @docs RETURN d", RawJson.class, + Collections.singletonMap("docs", docs), new AqlQueryOptions().batchSize(1)); + assertThat((Iterable) res).containsExactly(a, b, c); + } + @ParameterizedTest @MethodSource("dbs") void changeQueryCache(ArangoDatabase db) { From d3ef4744e76ce7cb34ab94a326922c09d52abdb8 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Thu, 28 Nov 2024 22:30:05 +0100 Subject: [PATCH 02/20] improved extraction of user-data bytes for VPACK content type --- pom.xml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index efcc9fef4..ab863dd16 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ com.arangodb jackson-dataformat-velocypack - 4.4.0 + 4.5.0-SNAPSHOT com.arangodb @@ -325,6 +325,19 @@ + + + oss.sonatype.org-snapshot + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + + + + https://github.com/arangodb/arangodb-java-driver scm:git:git://github.com/arangodb/arangodb-java-driver.git From a2d88a06169a8382f7cfd769a9ccfe377484b097 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Fri, 29 Nov 2024 12:08:51 +0100 Subject: [PATCH 03/20] improved deserialization of managed classes by avoid extracting bytes when not needed --- .../internal/serde/InternalDeserializers.java | 17 +++++- .../internal/serde/InternalModule.java | 2 + .../internal/serde/InternalSerdeImpl.java | 27 +--------- .../arangodb/internal/serde/SerdeUtils.java | 52 +++++++++++++++++++ .../internal/serde/UserDataDeserializer.java | 31 +++-------- .../java/com/arangodb/ArangoDatabaseTest.java | 14 ++++- 6 files changed, 90 insertions(+), 53 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java index d56807af9..c72663c2b 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java @@ -6,8 +6,10 @@ import com.arangodb.entity.ReplicationFactor; import com.arangodb.entity.arangosearch.CollectionLink; import com.arangodb.entity.arangosearch.FieldLink; +import com.arangodb.util.RawBytes; import com.arangodb.util.RawJson; import com.arangodb.internal.InternalResponse; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.DeserializationContext; @@ -16,6 +18,7 @@ import com.fasterxml.jackson.databind.node.*; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -26,7 +29,18 @@ public final class InternalDeserializers { static final JsonDeserializer RAW_JSON_DESERIALIZER = new JsonDeserializer() { @Override public RawJson deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return RawJson.of(SerdeUtils.INSTANCE.writeJson(p.readValueAsTree())); + if (JsonFactory.FORMAT_NAME_JSON.equals(p.getCodec().getFactory().getFormatName())) { + return RawJson.of(new String(SerdeUtils.extractBytes(p), StandardCharsets.UTF_8)); + } else { + return RawJson.of(SerdeUtils.INSTANCE.writeJson(p.readValueAsTree())); + } + } + }; + + static final JsonDeserializer RAW_BYTES_DESERIALIZER = new JsonDeserializer() { + @Override + public RawBytes deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return RawBytes.of(SerdeUtils.extractBytes(p)); } }; @@ -134,5 +148,4 @@ public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx } } - } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalModule.java b/core/src/main/java/com/arangodb/internal/serde/InternalModule.java index eefa65759..26133ad2f 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalModule.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalModule.java @@ -4,6 +4,7 @@ import com.arangodb.entity.CollectionType; import com.arangodb.entity.InvertedIndexPrimarySort; import com.arangodb.entity.ReplicationFactor; +import com.arangodb.util.RawBytes; import com.arangodb.util.RawJson; import com.arangodb.internal.InternalRequest; import com.arangodb.internal.InternalResponse; @@ -25,6 +26,7 @@ enum InternalModule implements Supplier { module.addSerializer(CollectionType.class, InternalSerializers.COLLECTION_TYPE); module.addDeserializer(RawJson.class, InternalDeserializers.RAW_JSON_DESERIALIZER); + module.addDeserializer(RawBytes.class, InternalDeserializers.RAW_BYTES_DESERIALIZER); module.addDeserializer(CollectionStatus.class, InternalDeserializers.COLLECTION_STATUS); module.addDeserializer(CollectionType.class, InternalDeserializers.COLLECTION_TYPE); module.addDeserializer(ReplicationFactor.class, InternalDeserializers.REPLICATION_FACTOR); diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index d24aa30df..26d5c9990 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -1,8 +1,6 @@ package com.arangodb.internal.serde; import com.arangodb.ArangoDBException; -import com.arangodb.entity.BaseDocument; -import com.arangodb.entity.BaseEdgeDocument; import com.arangodb.internal.RequestContextHolder; import com.arangodb.serde.ArangoSerde; import com.arangodb.util.RawBytes; @@ -112,7 +110,7 @@ public byte[] serializeUserData(Object value) { return ((RawBytes) value).get(); } else if (RawJson.class.equals(clazz) && JsonFactory.FORMAT_NAME_JSON.equals(mapper.getFactory().getFormatName())) { return ((RawJson) value).get().getBytes(StandardCharsets.UTF_8); - } else if (isManagedClass(clazz)) { + } else if (SerdeUtils.isManagedClass(clazz)) { return serialize(value); } else { return userSerde.serialize(value); @@ -129,13 +127,8 @@ public byte[] serializeCollectionUserData(Iterable value) { } @Override - @SuppressWarnings("unchecked") public T deserializeUserData(byte[] content, Class clazz) { - if (RawBytes.class.equals(clazz)) { - return (T) RawBytes.of(content); - } else if (RawJson.class.equals(clazz) && JsonFactory.FORMAT_NAME_JSON.equals(mapper.getFactory().getFormatName())) { - return (T) RawJson.of(new String(content, StandardCharsets.UTF_8)); - } else if (isManagedClass(clazz)) { + if (SerdeUtils.isManagedClass(clazz)) { return deserialize(content, clazz); } else { return userSerde.deserialize(content, clazz, RequestContextHolder.INSTANCE.getCtx()); @@ -183,20 +176,4 @@ public T deserialize(final byte[] content, final Type type) { } } - private boolean isManagedClass(Class clazz) { - return JsonNode.class.isAssignableFrom(clazz) || - RawJson.class.equals(clazz) || - RawBytes.class.equals(clazz) || - BaseDocument.class.equals(clazz) || - BaseEdgeDocument.class.equals(clazz) || - isEntityClass(clazz); - } - - private boolean isEntityClass(Class clazz) { - Package pkg = clazz.getPackage(); - if (pkg == null) { - return false; - } - return pkg.getName().startsWith("com.arangodb.entity"); - } } diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 299d2ff69..3c6ac427b 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -1,7 +1,13 @@ package com.arangodb.internal.serde; import com.arangodb.ArangoDBException; +import com.arangodb.entity.BaseDocument; +import com.arangodb.entity.BaseEdgeDocument; +import com.arangodb.util.RawBytes; +import com.arangodb.util.RawJson; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -9,6 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Arrays; @@ -81,4 +88,49 @@ public String writeJson(final JsonNode data) { } } + /** + * Extract raw bytes for the current JSON (or VPACK) node + * + * @param parser JsonParser with current token pointing to the node to extract + * @return byte array + */ + public static byte[] extractBytes(JsonParser parser) throws IOException { + JsonToken t = parser.currentToken(); + if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { + throw new RuntimeException("Unexpected token: " + t); + } + byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); + int start = (int) parser.currentTokenLocation().getByteOffset(); + if (t.isStructStart()) { + int open = 1; + while (open > 0) { + t = parser.nextToken(); + if (t.isStructStart()) { + open++; + } else if (t.isStructEnd()) { + open--; + } + } + } + parser.finishToken(); + int end = (int) parser.currentLocation().getByteOffset(); + return Arrays.copyOfRange(data, start, end); + } + + public static boolean isManagedClass(Class clazz) { + return JsonNode.class.isAssignableFrom(clazz) || + RawJson.class.equals(clazz) || + RawBytes.class.equals(clazz) || + BaseDocument.class.equals(clazz) || + BaseEdgeDocument.class.equals(clazz) || + isEntityClass(clazz); + } + + private static boolean isEntityClass(Class clazz) { + Package pkg = clazz.getPackage(); + if (pkg == null) { + return false; + } + return pkg.getName().startsWith("com.arangodb.entity"); + } } diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java index e8b9641dc..56b5e5703 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java @@ -1,7 +1,6 @@ package com.arangodb.internal.serde; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JavaType; @@ -11,7 +10,6 @@ import java.io.IOException; import java.lang.reflect.Type; -import java.util.Arrays; import static com.arangodb.internal.serde.SerdeUtils.convertToType; @@ -31,7 +29,12 @@ private UserDataDeserializer(final JavaType targetType, final InternalSerde serd @Override public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return serde.deserializeUserData(extractBytes(p), targetType); + Class clazz = (Class) targetType; + if (SerdeUtils.isManagedClass(clazz)) { + return p.readValueAs(clazz); + } else { + return serde.deserializeUserData(SerdeUtils.extractBytes(p), targetType); + } } @Override @@ -44,26 +47,4 @@ public JsonDeserializer createContextual(DeserializationContext ctxt, BeanPro return new UserDataDeserializer(ctxt.getContextualType(), serde); } - private byte[] extractBytes(JsonParser parser) throws IOException { - JsonToken t = parser.currentToken(); - if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { - throw new RuntimeException("Unexpected token: " + t); - } - byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); - int start = (int) parser.currentTokenLocation().getByteOffset(); - if (t.isStructStart()) { - int open = 1; - while (open > 0) { - t = parser.nextToken(); - if (t.isStructStart()) { - open++; - } else if (t.isStructEnd()) { - open--; - } - } - } - parser.finishToken(); - int end = (int) parser.currentLocation().getByteOffset(); - return Arrays.copyOfRange(data, start, end); - } } diff --git a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java index c8ee786e3..d86f224b1 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java @@ -779,7 +779,7 @@ void queryUserDataScalar(ArangoDatabase db) { @ParameterizedTest @MethodSource("dbs") - void queryUserDataStruct(ArangoDatabase db) { + void queryUserDataManaged(ArangoDatabase db) { RawJson a = RawJson.of("\"foo\""); RawJson b = RawJson.of("{\"key\":\"value\"}"); RawJson c = RawJson.of("[1,null,true,\"bla\",{},[],\"\"]"); @@ -789,6 +789,18 @@ void queryUserDataStruct(ArangoDatabase db) { assertThat((Iterable) res).containsExactly(a, b, c); } + @ParameterizedTest + @MethodSource("dbs") + void queryUserData(ArangoDatabase db) { + Object a = "foo"; + Object b = Collections.singletonMap("key", "value"); + Object c = Arrays.asList(1, null, true, "bla", Collections.emptyMap(), Collections.emptyList(), ""); + List docs = Arrays.asList(a, b, c); + ArangoCursor res = db.query("FOR d IN @docs RETURN d", Object.class, + Collections.singletonMap("docs", docs), new AqlQueryOptions().batchSize(1)); + assertThat((Iterable) res).containsExactly(a, b, c); + } + @ParameterizedTest @MethodSource("dbs") void changeQueryCache(ArangoDatabase db) { From c3d2a1b632fa7d7e022239e6c99cadf7fffbfa0d Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Fri, 29 Nov 2024 12:28:40 +0100 Subject: [PATCH 04/20] improved serialization of JsonNode by avoid intermediate parsing when not needed --- .../arangodb/internal/serde/UserDataSerializer.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java index d9a6acb30..ab1d5e07a 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; @@ -16,10 +17,14 @@ class UserDataSerializer extends JsonSerializer { @Override public void serialize(Object value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - // TODO: find a way to append raw bytes directly - // see https://github.com/FasterXML/jackson-core/issues/914 - try (JsonParser parser = gen.getCodec().getFactory().createParser(serde.serializeUserData(value))) { - gen.writeTree(parser.readValueAsTree()); + if (value != null && JsonNode.class.isAssignableFrom(value.getClass())) { + gen.writeTree((JsonNode) value); + } else { + // TODO: find a way to append raw bytes directly + // see https://github.com/FasterXML/jackson-core/issues/914 + try (JsonParser parser = gen.getCodec().getFactory().createParser(serde.serializeUserData(value))) { + gen.writeTree(parser.readValueAsTree()); + } } } } From 7b14abdb2748549ac82ccc73227897df107bf02e Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Fri, 29 Nov 2024 21:56:04 +0100 Subject: [PATCH 05/20] improved RawJson deserialization --- .../internal/serde/InternalDeserializers.java | 8 +- test-perf/pom.xml | 2 +- .../test/java/com/arangodb/SerdeBench.java | 95 +------------------ 3 files changed, 13 insertions(+), 92 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java index c72663c2b..f0f4d0eba 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java @@ -10,6 +10,7 @@ import com.arangodb.util.RawJson; import com.arangodb.internal.InternalResponse; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.DeserializationContext; @@ -18,6 +19,7 @@ import com.fasterxml.jackson.databind.node.*; import java.io.IOException; +import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -32,7 +34,11 @@ public RawJson deserialize(JsonParser p, DeserializationContext ctxt) throws IOE if (JsonFactory.FORMAT_NAME_JSON.equals(p.getCodec().getFactory().getFormatName())) { return RawJson.of(new String(SerdeUtils.extractBytes(p), StandardCharsets.UTF_8)); } else { - return RawJson.of(SerdeUtils.INSTANCE.writeJson(p.readValueAsTree())); + StringWriter w = new StringWriter(); + JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().createGenerator(w); + gen.copyCurrentStructure(p); + gen.close(); + return RawJson.of(w.toString()); } } }; diff --git a/test-perf/pom.xml b/test-perf/pom.xml index b6e419ca7..bcf304dd3 100644 --- a/test-perf/pom.xml +++ b/test-perf/pom.xml @@ -7,7 +7,7 @@ ../test-parent com.arangodb test-parent - 7.14.0 + 7.13.1 test-perf diff --git a/test-perf/src/test/java/com/arangodb/SerdeBench.java b/test-perf/src/test/java/com/arangodb/SerdeBench.java index ae3d90bc3..2ff30ac9b 100644 --- a/test-perf/src/test/java/com/arangodb/SerdeBench.java +++ b/test-perf/src/test/java/com/arangodb/SerdeBench.java @@ -1,10 +1,5 @@ package com.arangodb; -import com.arangodb.entity.MultiDocumentEntity; -import com.arangodb.internal.ArangoCollectionImpl; -import com.arangodb.internal.ArangoDatabaseImpl; -import com.arangodb.internal.ArangoExecutor; -import com.arangodb.internal.InternalResponse; import com.arangodb.internal.serde.InternalSerde; import com.arangodb.internal.serde.InternalSerdeProvider; import com.arangodb.jackson.dataformat.velocypack.VPackMapper; @@ -30,7 +25,6 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -45,35 +39,6 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @Fork(1) public class SerdeBench { - public static class MyCol extends ArangoCollectionImpl { - static ArangoDB jsonAdb = new ArangoDB.Builder() - .host("127.0.0.1", 8529) - .protocol(Protocol.HTTP_JSON) - .build(); - - static ArangoDB vpackAdb = new ArangoDB.Builder() - .host("127.0.0.1", 8529) - .protocol(Protocol.HTTP_VPACK) - .build(); - - private MyCol(ArangoDB adb) { - super((ArangoDatabaseImpl) adb.db(), "foo"); - } - - public static MyCol ofJson() { - return new MyCol(jsonAdb); - } - - public static MyCol ofVpack() { - return new MyCol(vpackAdb); - } - - @Override - public ArangoExecutor.ResponseDeserializer> getDocumentsResponseDeserializer(Class type) { - return super.getDocumentsResponseDeserializer(type); - } - } - @State(Scope.Benchmark) public static class Data { public final byte[] vpack; @@ -81,38 +46,26 @@ public static class Data { public final RawBytes rawJsonBytes; public final RawBytes rawVPackBytes; public final RawJson rawJson; - public final MyCol jsonCol = MyCol.ofJson(); - public final MyCol vpackCol = MyCol.ofVpack(); - public final InternalResponse jsonResp = new InternalResponse(); - public final InternalResponse vpackResp = new InternalResponse(); public Data() { ObjectMapper jsonMapper = new ObjectMapper(); VPackMapper vpackMapper = new VPackMapper(); try { - JsonNode jn = readFile("/api-docs.json", jsonMapper); + String str = new String(Files.readAllBytes( + Paths.get(SerdeBench.class.getResource("/api-docs.json").toURI()))); + JsonNode jn = jsonMapper.readTree(str); + json = jsonMapper.writeValueAsBytes(jn); vpack = vpackMapper.writeValueAsBytes(jn); rawJsonBytes = RawBytes.of(json); rawVPackBytes = RawBytes.of(vpack); - rawJson = RawJson.of(jsonMapper.writeValueAsString(jsonMapper.readTree(json))); + rawJson = RawJson.of(jsonMapper.writeValueAsString(json)); - JsonNode docs = readFile("/multi-docs.json", jsonMapper); - jsonResp.setResponseCode(200); - jsonResp.setBody(jsonMapper.writeValueAsBytes(docs)); - vpackResp.setResponseCode(200); - vpackResp.setBody(vpackMapper.writeValueAsBytes(docs)); } catch (Exception e) { throw new RuntimeException(e); } } - - private JsonNode readFile(String filename, ObjectMapper mapper) throws IOException, URISyntaxException { - String str = new String(Files.readAllBytes( - Paths.get(SerdeBench.class.getResource(filename).toURI()))); - return mapper.readTree(str); - } } public static void main(String[] args) throws RunnerException, IOException { @@ -145,42 +98,4 @@ public void rawJsonDeser(Data data, Blackhole bh) { ); } - @Benchmark - public void rawJsonSer(Data data, Blackhole bh) { - InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); - bh.consume( - serde.serialize(data.rawJson) - ); - } - - @Benchmark - public void extractBytesVPack(Data data, Blackhole bh) { - InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); - bh.consume( - serde.extract(data.vpack, "/definitions/put_api_simple_remove_by_example_opts") - ); - } - - @Benchmark - public void extractBytesJson(Data data, Blackhole bh) { - InternalSerde serde = new InternalSerdeProvider(ContentType.JSON).create(); - bh.consume( - serde.extract(data.json, "/definitions/put_api_simple_remove_by_example_opts") - ); - } - - @Benchmark - public void deserializeDocsJson(Data data, Blackhole bh) { - bh.consume( - data.jsonCol.getDocumentsResponseDeserializer(RawBytes.class).deserialize(data.jsonResp) - ); - } - - @Benchmark - public void deserializeDocsVPack(Data data, Blackhole bh) { - bh.consume( - data.vpackCol.getDocumentsResponseDeserializer(RawBytes.class).deserialize(data.vpackResp) - ); - } - } From 8a68fbf38e27a9cecc8f336da25f1e063e47a0ad Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Fri, 29 Nov 2024 22:07:41 +0100 Subject: [PATCH 06/20] improved RawJson serialization --- .../internal/serde/InternalSerializers.java | 6 +++++- .../src/test/java/com/arangodb/SerdeBench.java | 14 +++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java index 31c87ae77..59f226de3 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java @@ -7,6 +7,7 @@ import com.arangodb.util.RawJson; import com.arangodb.internal.InternalRequest; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; @@ -21,7 +22,10 @@ public final class InternalSerializers { static final JsonSerializer RAW_JSON_SERIALIZER = new JsonSerializer() { @Override public void serialize(RawJson value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeTree(SerdeUtils.INSTANCE.parseJson(value.get())); + try (JsonParser parser = SerdeUtils.INSTANCE.getJsonMapper().createParser(value.get())) { + parser.nextToken(); + gen.copyCurrentStructure(parser); + } } }; static final JsonSerializer REQUEST = new JsonSerializer() { diff --git a/test-perf/src/test/java/com/arangodb/SerdeBench.java b/test-perf/src/test/java/com/arangodb/SerdeBench.java index 2ff30ac9b..52891f5b1 100644 --- a/test-perf/src/test/java/com/arangodb/SerdeBench.java +++ b/test-perf/src/test/java/com/arangodb/SerdeBench.java @@ -60,7 +60,7 @@ public Data() { vpack = vpackMapper.writeValueAsBytes(jn); rawJsonBytes = RawBytes.of(json); rawVPackBytes = RawBytes.of(vpack); - rawJson = RawJson.of(jsonMapper.writeValueAsString(json)); + rawJson = RawJson.of(jsonMapper.writeValueAsString(jsonMapper.readTree(json))); } catch (Exception e) { throw new RuntimeException(e); @@ -90,11 +90,19 @@ public static void main(String[] args) throws RunnerException, IOException { new Runner(opt).run(); } +// @Benchmark +// public void rawJsonDeser(Data data, Blackhole bh) { +// InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); +// bh.consume( +// serde.deserialize(data.vpack, RawJson.class) +// ); +// } + @Benchmark - public void rawJsonDeser(Data data, Blackhole bh) { + public void rawJsonSer(Data data, Blackhole bh) { InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); bh.consume( - serde.deserialize(data.vpack, RawJson.class) + serde.serialize(data.rawJson) ); } From e081ef6cd5cec764eae5e482d8ee6cd7caa0d355 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Mon, 2 Dec 2024 15:16:36 +0100 Subject: [PATCH 07/20] improved extract bytes at json pointer --- .../internal/InternalArangoCollection.java | 5 +++ .../arangodb/internal/net/Communication.java | 6 +-- .../internal/serde/InternalSerde.java | 2 + .../internal/serde/InternalSerdeImpl.java | 43 ++++++++++++++++--- .../arangodb/internal/serde/SerdeUtils.java | 4 ++ .../internal/serde/UserDataSerializer.java | 1 + .../arangodb/internal/util/SerdeUtils.java | 19 -------- .../test/java/com/arangodb/SerdeBench.java | 30 ++++++++++--- 8 files changed, 75 insertions(+), 35 deletions(-) delete mode 100644 core/src/main/java/com/arangodb/internal/util/SerdeUtils.java diff --git a/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java b/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java index 6f054cf5e..b2546565a 100644 --- a/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java +++ b/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java @@ -109,6 +109,7 @@ private InternalRequest createInsertDocumentRequest(final DocumentCreateOptions return request; } + // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> insertDocumentsResponseDeserializer(Class userDataClass) { return (response) -> { final MultiDocumentEntity> multiDocument = new MultiDocumentEntity<>(); @@ -184,6 +185,7 @@ protected InternalRequest getDocumentsRequest(final Iterable keys, final return request; } + // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer> getDocumentsResponseDeserializer( final Class type) { return (response) -> { @@ -247,6 +249,7 @@ private InternalRequest createReplaceDocumentRequest(final DocumentReplaceOption return request; } + // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> replaceDocumentsResponseDeserializer( final Class returnType) { return (response) -> { @@ -310,6 +313,7 @@ private InternalRequest createUpdateDocumentRequest(final DocumentUpdateOptions return request; } + // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> updateDocumentsResponseDeserializer( final Class returnType) { return (response) -> { @@ -367,6 +371,7 @@ private InternalRequest createDeleteDocumentRequest(final DocumentDeleteOptions return request; } + // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> deleteDocumentsResponseDeserializer( final Class userDataClass) { return (response) -> { diff --git a/core/src/main/java/com/arangodb/internal/net/Communication.java b/core/src/main/java/com/arangodb/internal/net/Communication.java index b390ecee1..0309cd04c 100644 --- a/core/src/main/java/com/arangodb/internal/net/Communication.java +++ b/core/src/main/java/com/arangodb/internal/net/Communication.java @@ -23,8 +23,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import static com.arangodb.internal.util.SerdeUtils.toJsonString; - @UsedInApi public abstract class Communication implements Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(Communication.class); @@ -59,7 +57,7 @@ private CompletableFuture doExecuteAsync( final InternalRequest request, final HostHandle hostHandle, final Host host, final int attemptCount, Connection connection, long reqId ) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Send Request [id={}]: {} {}", reqId, request, toJsonString(serde, request.getBody())); + LOGGER.debug("Send Request [id={}]: {} {}", reqId, request, serde.toJsonString(request.getBody())); } final CompletableFuture rfuture = new CompletableFuture<>(); try { @@ -85,7 +83,7 @@ private CompletableFuture doExecuteAsync( handleException(isSafe(request), e, hostHandle, request, host, reqId, attemptCount, rfuture); } else { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Received Response [id={}]: {} {}", reqId, response, toJsonString(serde, response.getBody())); + LOGGER.debug("Received Response [id={}]: {} {}", reqId, response, serde.toJsonString(response.getBody())); } ArangoDBException errorEntityEx = ResponseUtils.translateError(serde, response); if (errorEntityEx instanceof ArangoDBRedirectException) { diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java index 758cc550a..0146669fb 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java @@ -15,6 +15,7 @@ public interface InternalSerde extends ArangoSerde { * * @param content byte array * @return JSON string + * @implSpec return {@code "[Unparsable data]"} in case of parsing exception */ String toJsonString(byte[] content); @@ -97,6 +98,7 @@ default T deserialize(byte[] content, String jsonPointer, Class clazz) { * @param type target data type * @return deserialized object */ + // TODO: avoid serialization-deserialization round-trip default T deserialize(byte[] content, String jsonPointer, Type type) { return deserialize(parse(content, jsonPointer), type); } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index 26d5c9990..8e40e52d0 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; @@ -22,6 +23,7 @@ import java.util.stream.StreamSupport; import static com.arangodb.internal.serde.SerdeUtils.checkSupportedJacksonVersion; +import static com.arangodb.internal.serde.SerdeUtils.extractBytes; final class InternalSerdeImpl implements InternalSerde { @@ -65,18 +67,48 @@ public T deserialize(byte[] content, Class clazz) { @Override public String toJsonString(final byte[] content) { + if (content == null) { + return ""; + } try { return SerdeUtils.INSTANCE.writeJson(mapper.readTree(content)); - } catch (IOException e) { - throw ArangoDBException.of(e); + } catch (Exception e) { + return "[Unparsable data]"; } } @Override public byte[] extract(final byte[] content, final String jsonPointer) { - try { - JsonNode target = parse(content).at(jsonPointer); - return mapper.writeValueAsBytes(target); + if (!jsonPointer.startsWith("/")) { + throw new ArangoDBException("Unsupported JSON pointer: " + jsonPointer); + } + String[] parts = jsonPointer.substring(1).split("/"); + try (JsonParser parser = mapper.createParser(content)) { + int match = 0; + int level = 0; + JsonToken token = parser.nextToken(); + if (token != JsonToken.START_OBJECT) { + throw new ArangoDBException("Unable to parse token: " + token); + } + while (true) { + token = parser.nextToken(); + if (token == JsonToken.START_OBJECT) { + level++; + } + if (token == JsonToken.END_OBJECT) { + level--; + } + if (token == null || level < match) { + throw new ArangoDBException("Unable to parse JSON pointer: " + jsonPointer); + } + if (token == JsonToken.FIELD_NAME && match == level && parts[match].equals(parser.getText())) { + match++; + if (match == parts.length) { + parser.nextToken(); + return extractBytes(parser); + } + } + } } catch (IOException e) { throw ArangoDBException.of(e); } @@ -117,6 +149,7 @@ public byte[] serializeUserData(Object value) { } } + // TODO: review @Override public byte[] serializeCollectionUserData(Iterable value) { List jsonNodeCollection = StreamSupport.stream(value.spliterator(), false) diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 3c6ac427b..c33ad2103 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -62,6 +62,10 @@ static void checkSupportedJacksonVersion() { }); } + public ObjectMapper getJsonMapper() { + return jsonMapper; + } + /** * Parse a JSON string. * diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java index ab1d5e07a..d07c4687c 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java @@ -22,6 +22,7 @@ public void serialize(Object value, JsonGenerator gen, SerializerProvider serial } else { // TODO: find a way to append raw bytes directly // see https://github.com/FasterXML/jackson-core/issues/914 + // TODO: check gen.getOutputContext() try (JsonParser parser = gen.getCodec().getFactory().createParser(serde.serializeUserData(value))) { gen.writeTree(parser.readValueAsTree()); } diff --git a/core/src/main/java/com/arangodb/internal/util/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/util/SerdeUtils.java deleted file mode 100644 index 06e5edbb9..000000000 --- a/core/src/main/java/com/arangodb/internal/util/SerdeUtils.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.arangodb.internal.util; - -import com.arangodb.internal.serde.InternalSerde; - -public class SerdeUtils { - private SerdeUtils() { - } - - public static String toJsonString(InternalSerde serde, byte[] data) { - if (data == null) { - return ""; - } - try { - return serde.toJsonString(data); - } catch (Exception e) { - return "[Unparsable data]"; - } - } -} diff --git a/test-perf/src/test/java/com/arangodb/SerdeBench.java b/test-perf/src/test/java/com/arangodb/SerdeBench.java index 52891f5b1..def0c298a 100644 --- a/test-perf/src/test/java/com/arangodb/SerdeBench.java +++ b/test-perf/src/test/java/com/arangodb/SerdeBench.java @@ -90,13 +90,13 @@ public static void main(String[] args) throws RunnerException, IOException { new Runner(opt).run(); } -// @Benchmark -// public void rawJsonDeser(Data data, Blackhole bh) { -// InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); -// bh.consume( -// serde.deserialize(data.vpack, RawJson.class) -// ); -// } + @Benchmark + public void rawJsonDeser(Data data, Blackhole bh) { + InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); + bh.consume( + serde.deserialize(data.vpack, RawJson.class) + ); + } @Benchmark public void rawJsonSer(Data data, Blackhole bh) { @@ -106,4 +106,20 @@ public void rawJsonSer(Data data, Blackhole bh) { ); } + @Benchmark + public void extractBytesVPack(Data data, Blackhole bh) { + InternalSerde serde = new InternalSerdeProvider(ContentType.VPACK).create(); + bh.consume( + serde.extract(data.vpack, "/definitions/put_api_simple_remove_by_example_opts") + ); + } + + @Benchmark + public void extractBytesJson(Data data, Blackhole bh) { + InternalSerde serde = new InternalSerdeProvider(ContentType.JSON).create(); + bh.consume( + serde.extract(data.json, "/definitions/put_api_simple_remove_by_example_opts") + ); + } + } From 24376b6f2548744a33cb5568d157ce6474edec64 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 4 Dec 2024 13:51:25 +0100 Subject: [PATCH 08/20] improved deserialize bytes at json pointer --- .../main/java/com/arangodb/internal/serde/InternalSerde.java | 3 +-- .../main/java/com/arangodb/internal/serde/SerdeUtils.java | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java index 0146669fb..674f9d30c 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java @@ -98,9 +98,8 @@ default T deserialize(byte[] content, String jsonPointer, Class clazz) { * @param type target data type * @return deserialized object */ - // TODO: avoid serialization-deserialization round-trip default T deserialize(byte[] content, String jsonPointer, Type type) { - return deserialize(parse(content, jsonPointer), type); + return deserialize(extract(content, jsonPointer), type); } /** diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index c33ad2103..4a79f3a25 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -105,6 +105,7 @@ public static byte[] extractBytes(JsonParser parser) throws IOException { } byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); int start = (int) parser.currentTokenLocation().getByteOffset(); + int end = (int) parser.currentLocation().getByteOffset(); if (t.isStructStart()) { int open = 1; while (open > 0) { @@ -117,7 +118,9 @@ public static byte[] extractBytes(JsonParser parser) throws IOException { } } parser.finishToken(); - int end = (int) parser.currentLocation().getByteOffset(); + if ("JSON".equals(parser.getCodec().getFactory().getFormatName())) { + end = (int) parser.currentLocation().getByteOffset(); + } return Arrays.copyOfRange(data, start, end); } From 5da52351cbfe03b6439a961fc1d3b23064f65258 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 4 Dec 2024 15:46:50 +0100 Subject: [PATCH 09/20] improved serialize user data bytes --- .../internal/serde/InternalSerde.java | 20 ---- .../internal/serde/InternalSerdeImpl.java | 6 +- .../internal/serde/RawUserDataValue.java | 92 +++++++++++++++++++ .../internal/serde/UserDataSerializer.java | 8 +- 4 files changed, 94 insertions(+), 32 deletions(-) create mode 100644 core/src/main/java/com/arangodb/internal/serde/RawUserDataValue.java diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java index 674f9d30c..a4b70e2ca 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java @@ -136,26 +136,6 @@ default T deserialize(byte[] content, String jsonPointer, Type type) { */ T deserializeUserData(byte[] content, Type type); - /** - * Deserializes the parsed json node and binds it to the target data type, using the user serde. - * - * @param node parsed json node - * @param clazz class of target data type - * @return deserialized object - */ - default T deserializeUserData(JsonNode node, Class clazz) { - return deserializeUserData(node, (Type) clazz); - } - - /** - * Deserializes the parsed json node and binds it to the target data type, using the user serde. - * - * @param node parsed json node - * @param type target data type - * @return deserialized object - */ - T deserializeUserData(JsonNode node, Type type); - /** * @return the user serde */ diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index 8e40e52d0..b770eea7b 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -114,6 +114,7 @@ public byte[] extract(final byte[] content, final String jsonPointer) { } } + // TODO: remove @Override public JsonNode parse(byte[] content) { try { @@ -178,11 +179,6 @@ public T deserializeUserData(byte[] content, Type type) { } } - @Override - public T deserializeUserData(JsonNode node, Type type) { - return deserializeUserData(serialize(node), type); - } - @Override public ArangoSerde getUserSerde() { return userSerde; diff --git a/core/src/main/java/com/arangodb/internal/serde/RawUserDataValue.java b/core/src/main/java/com/arangodb/internal/serde/RawUserDataValue.java new file mode 100644 index 000000000..4bfde90f2 --- /dev/null +++ b/core/src/main/java/com/arangodb/internal/serde/RawUserDataValue.java @@ -0,0 +1,92 @@ +package com.arangodb.internal.serde; + +import com.fasterxml.jackson.core.SerializableString; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +class RawUserDataValue implements SerializableString { + private final byte[] data; + + RawUserDataValue(byte[] data) { + this.data = data; + } + + @Override + public String getValue() { + throw new UnsupportedOperationException(); + } + + @Override + public int charLength() { + throw new UnsupportedOperationException(); + } + + @Override + public char[] asQuotedChars() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] asUnquotedUTF8() { + return data; + } + + @Override + public byte[] asQuotedUTF8() { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuotedUTF8(byte[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendQuoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int appendUnquotedUTF8(byte[] buffer, int offset) { + final int length = data.length; + if ((offset + length) > buffer.length) { + return -1; + } + System.arraycopy(data, 0, buffer, offset, length); + return length; + } + + @Override + public int appendUnquoted(char[] buffer, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int writeQuotedUTF8(OutputStream out) { + throw new UnsupportedOperationException(); + } + + @Override + public int writeUnquotedUTF8(OutputStream out) throws IOException { + final int length = data.length; + out.write(data, 0, length); + return length; + } + + @Override + public int putQuotedUTF8(ByteBuffer buffer) { + throw new UnsupportedOperationException(); + } + + @Override + public int putUnquotedUTF8(ByteBuffer buffer) { + final int length = data.length; + if (length > buffer.remaining()) { + return -1; + } + buffer.put(data, 0, length); + return length; + } +} diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java index d07c4687c..501998da4 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataSerializer.java @@ -1,7 +1,6 @@ package com.arangodb.internal.serde; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; @@ -20,12 +19,7 @@ public void serialize(Object value, JsonGenerator gen, SerializerProvider serial if (value != null && JsonNode.class.isAssignableFrom(value.getClass())) { gen.writeTree((JsonNode) value); } else { - // TODO: find a way to append raw bytes directly - // see https://github.com/FasterXML/jackson-core/issues/914 - // TODO: check gen.getOutputContext() - try (JsonParser parser = gen.getCodec().getFactory().createParser(serde.serializeUserData(value))) { - gen.writeTree(parser.readValueAsTree()); - } + gen.writeRawValue(new RawUserDataValue(serde.serializeUserData(value))); } } } From 4949fdc5d328357973ec0fe043ef7da3bfddd95c Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 4 Dec 2024 18:10:36 +0100 Subject: [PATCH 10/20] improved serialize list of user data --- .../internal/serde/InternalDeserializers.java | 8 ++++--- .../internal/serde/InternalSerdeImpl.java | 23 +++++++++++-------- .../arangodb/internal/serde/SerdeUtils.java | 2 +- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java index f0f4d0eba..92227f56c 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java @@ -34,10 +34,12 @@ public RawJson deserialize(JsonParser p, DeserializationContext ctxt) throws IOE if (JsonFactory.FORMAT_NAME_JSON.equals(p.getCodec().getFactory().getFormatName())) { return RawJson.of(new String(SerdeUtils.extractBytes(p), StandardCharsets.UTF_8)); } else { + // TODO: compare perfs using ByteArrayOutputStream and StringWriter StringWriter w = new StringWriter(); - JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().createGenerator(w); - gen.copyCurrentStructure(p); - gen.close(); + try (JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().createGenerator(w)) { + gen.copyCurrentStructure(p); + gen.flush(); + } return RawJson.of(w.toString()); } } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index b770eea7b..f01c732d4 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -7,6 +7,7 @@ import com.arangodb.util.RawJson; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; @@ -15,12 +16,10 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import static com.arangodb.internal.serde.SerdeUtils.checkSupportedJacksonVersion; import static com.arangodb.internal.serde.SerdeUtils.extractBytes; @@ -150,14 +149,20 @@ public byte[] serializeUserData(Object value) { } } - // TODO: review @Override public byte[] serializeCollectionUserData(Iterable value) { - List jsonNodeCollection = StreamSupport.stream(value.spliterator(), false) - .map(this::serializeUserData) - .map(this::parse) - .collect(Collectors.toList()); - return serialize(jsonNodeCollection); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try (JsonGenerator gen = mapper.createGenerator(os)) { + gen.writeStartArray(); + for (Object o : value) { + gen.writeRawValue(new RawUserDataValue(serializeUserData(o))); + } + gen.writeEndArray(); + gen.flush(); + } catch (IOException e) { + throw ArangoDBException.of(e); + } + return os.toByteArray(); } @Override diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 4a79f3a25..6da20d535 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -101,7 +101,7 @@ public String writeJson(final JsonNode data) { public static byte[] extractBytes(JsonParser parser) throws IOException { JsonToken t = parser.currentToken(); if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { - throw new RuntimeException("Unexpected token: " + t); + throw new ArangoDBException("Unexpected token: " + t); } byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); int start = (int) parser.currentTokenLocation().getByteOffset(); From 700a59a1005cc29043aed92953bcd16d6c83d88a Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Wed, 4 Dec 2024 23:54:17 +0100 Subject: [PATCH 11/20] improved deserialize of responses of CRUD operations on multiple documents --- .../arangodb/entity/MultiDocumentEntity.java | 7 +- .../internal/InternalArangoCollection.java | 131 ++---------------- .../internal/serde/InternalDeserializers.java | 1 - .../internal/serde/InternalModule.java | 16 +-- .../internal/serde/InternalSerde.java | 20 +-- .../internal/serde/InternalSerdeImpl.java | 26 ++-- .../MultiDocumentEntityDeserializer.java | 85 ++++++++++++ .../arangodb/internal/serde/SerdeUtils.java | 1 + .../com/arangodb/ArangoCollectionTest.java | 3 +- .../com/arangodb/ArangoDatabaseAsyncTest.java | 2 +- .../java/com/arangodb/ArangoDatabaseTest.java | 2 +- 11 files changed, 141 insertions(+), 153 deletions(-) create mode 100644 core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java diff --git a/core/src/main/java/com/arangodb/entity/MultiDocumentEntity.java b/core/src/main/java/com/arangodb/entity/MultiDocumentEntity.java index 95803226f..6bdbe2873 100644 --- a/core/src/main/java/com/arangodb/entity/MultiDocumentEntity.java +++ b/core/src/main/java/com/arangodb/entity/MultiDocumentEntity.java @@ -20,6 +20,7 @@ package com.arangodb.entity; +import java.util.ArrayList; import java.util.List; /** @@ -27,9 +28,9 @@ */ public final class MultiDocumentEntity { - private List documents; - private List errors; - private List documentsAndErrors; + private List documents = new ArrayList<>(); + private List errors = new ArrayList<>(); + private List documentsAndErrors = new ArrayList<>(); private boolean isPotentialDirtyRead = false; public MultiDocumentEntity() { diff --git a/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java b/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java index b2546565a..f794bcd31 100644 --- a/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java +++ b/core/src/main/java/com/arangodb/internal/InternalArangoCollection.java @@ -32,7 +32,6 @@ import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collection; -import java.util.List; import static com.arangodb.internal.serde.SerdeUtils.constructParametricType; @@ -109,31 +108,11 @@ private InternalRequest createInsertDocumentRequest(final DocumentCreateOptions return request; } - // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> insertDocumentsResponseDeserializer(Class userDataClass) { return (response) -> { - final MultiDocumentEntity> multiDocument = new MultiDocumentEntity<>(); - final List> docs = new ArrayList<>(); - final List errors = new ArrayList<>(); - final List documentsAndErrors = new ArrayList<>(); - final JsonNode body = getSerde().parse(response.getBody()); - for (final JsonNode next : body) { - JsonNode isError = next.get(ArangoResponseField.ERROR_FIELD_NAME); - if (isError != null && isError.booleanValue()) { - final ErrorEntity error = getSerde().deserialize(next, ErrorEntity.class); - errors.add(error); - documentsAndErrors.add(error); - } else { - Type type = constructParametricType(DocumentCreateEntity.class, userDataClass); - final DocumentCreateEntity doc = getSerde().deserialize(next, type); - docs.add(doc); - documentsAndErrors.add(doc); - } - } - multiDocument.setDocuments(docs); - multiDocument.setErrors(errors); - multiDocument.setDocumentsAndErrors(documentsAndErrors); - return multiDocument; + Type type = constructParametricType(MultiDocumentEntity.class, + constructParametricType(DocumentCreateEntity.class, userDataClass)); + return getSerde().deserialize(response.getBody(), type); }; } @@ -185,32 +164,12 @@ protected InternalRequest getDocumentsRequest(final Iterable keys, final return request; } - // TODO: avoid serialization-deserialization round-trip - protected ResponseDeserializer> getDocumentsResponseDeserializer( - final Class type) { + protected ResponseDeserializer> getDocumentsResponseDeserializer(final Class type) { return (response) -> { - final MultiDocumentEntity multiDocument = new MultiDocumentEntity<>(); + MultiDocumentEntity multiDocument = getSerde().deserialize(response.getBody(), + constructParametricType(MultiDocumentEntity.class, type)); boolean potentialDirtyRead = Boolean.parseBoolean(response.getMeta("X-Arango-Potential-Dirty-Read")); multiDocument.setPotentialDirtyRead(potentialDirtyRead); - final List docs = new ArrayList<>(); - final List errors = new ArrayList<>(); - final List documentsAndErrors = new ArrayList<>(); - final JsonNode body = getSerde().parse(response.getBody()); - for (final JsonNode next : body) { - JsonNode isError = next.get(ArangoResponseField.ERROR_FIELD_NAME); - if (isError != null && isError.booleanValue()) { - final ErrorEntity error = getSerde().deserialize(next, ErrorEntity.class); - errors.add(error); - documentsAndErrors.add(error); - } else { - final T doc = getSerde().deserializeUserData(getSerde().serialize(next), type); - docs.add(doc); - documentsAndErrors.add(doc); - } - } - multiDocument.setDocuments(docs); - multiDocument.setErrors(errors); - multiDocument.setDocumentsAndErrors(documentsAndErrors); return multiDocument; }; } @@ -249,32 +208,12 @@ private InternalRequest createReplaceDocumentRequest(final DocumentReplaceOption return request; } - // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> replaceDocumentsResponseDeserializer( final Class returnType) { return (response) -> { - final MultiDocumentEntity> multiDocument = new MultiDocumentEntity<>(); - final List> docs = new ArrayList<>(); - final List errors = new ArrayList<>(); - final List documentsAndErrors = new ArrayList<>(); - final JsonNode body = getSerde().parse(response.getBody()); - for (final JsonNode next : body) { - JsonNode isError = next.get(ArangoResponseField.ERROR_FIELD_NAME); - if (isError != null && isError.booleanValue()) { - final ErrorEntity error = getSerde().deserialize(next, ErrorEntity.class); - errors.add(error); - documentsAndErrors.add(error); - } else { - Type type = constructParametricType(DocumentUpdateEntity.class, returnType); - final DocumentUpdateEntity doc = getSerde().deserialize(next, type); - docs.add(doc); - documentsAndErrors.add(doc); - } - } - multiDocument.setDocuments(docs); - multiDocument.setErrors(errors); - multiDocument.setDocumentsAndErrors(documentsAndErrors); - return multiDocument; + Type type = constructParametricType(MultiDocumentEntity.class, + constructParametricType(DocumentUpdateEntity.class, returnType)); + return getSerde().deserialize(response.getBody(), type); }; } @@ -313,32 +252,12 @@ private InternalRequest createUpdateDocumentRequest(final DocumentUpdateOptions return request; } - // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> updateDocumentsResponseDeserializer( final Class returnType) { return (response) -> { - final MultiDocumentEntity> multiDocument = new MultiDocumentEntity<>(); - final List> docs = new ArrayList<>(); - final List errors = new ArrayList<>(); - final List documentsAndErrors = new ArrayList<>(); - final JsonNode body = getSerde().parse(response.getBody()); - for (final JsonNode next : body) { - JsonNode isError = next.get(ArangoResponseField.ERROR_FIELD_NAME); - if (isError != null && isError.booleanValue()) { - final ErrorEntity error = getSerde().deserialize(next, ErrorEntity.class); - errors.add(error); - documentsAndErrors.add(error); - } else { - Type type = constructParametricType(DocumentUpdateEntity.class, returnType); - final DocumentUpdateEntity doc = getSerde().deserialize(next, type); - docs.add(doc); - documentsAndErrors.add(doc); - } - } - multiDocument.setDocuments(docs); - multiDocument.setErrors(errors); - multiDocument.setDocumentsAndErrors(documentsAndErrors); - return multiDocument; + Type type = constructParametricType(MultiDocumentEntity.class, + constructParametricType(DocumentUpdateEntity.class, returnType)); + return getSerde().deserialize(response.getBody(), type); }; } @@ -371,32 +290,12 @@ private InternalRequest createDeleteDocumentRequest(final DocumentDeleteOptions return request; } - // TODO: avoid serialization-deserialization round-trip protected ResponseDeserializer>> deleteDocumentsResponseDeserializer( final Class userDataClass) { return (response) -> { - final MultiDocumentEntity> multiDocument = new MultiDocumentEntity<>(); - final List> docs = new ArrayList<>(); - final List errors = new ArrayList<>(); - final List documentsAndErrors = new ArrayList<>(); - final JsonNode body = getSerde().parse(response.getBody()); - for (final JsonNode next : body) { - JsonNode isError = next.get(ArangoResponseField.ERROR_FIELD_NAME); - if (isError != null && isError.booleanValue()) { - final ErrorEntity error = getSerde().deserialize(next, ErrorEntity.class); - errors.add(error); - documentsAndErrors.add(error); - } else { - Type type = constructParametricType(DocumentDeleteEntity.class, userDataClass); - final DocumentDeleteEntity doc = getSerde().deserialize(next, type); - docs.add(doc); - documentsAndErrors.add(doc); - } - } - multiDocument.setDocuments(docs); - multiDocument.setErrors(errors); - multiDocument.setDocumentsAndErrors(documentsAndErrors); - return multiDocument; + Type type = constructParametricType(MultiDocumentEntity.class, + constructParametricType(DocumentDeleteEntity.class, userDataClass)); + return getSerde().deserialize(response.getBody(), type); }; } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java index 92227f56c..8eaee6f56 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java @@ -34,7 +34,6 @@ public RawJson deserialize(JsonParser p, DeserializationContext ctxt) throws IOE if (JsonFactory.FORMAT_NAME_JSON.equals(p.getCodec().getFactory().getFormatName())) { return RawJson.of(new String(SerdeUtils.extractBytes(p), StandardCharsets.UTF_8)); } else { - // TODO: compare perfs using ByteArrayOutputStream and StringWriter StringWriter w = new StringWriter(); try (JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().createGenerator(w)) { gen.copyCurrentStructure(p); diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalModule.java b/core/src/main/java/com/arangodb/internal/serde/InternalModule.java index 26133ad2f..392a9c334 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalModule.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalModule.java @@ -3,6 +3,7 @@ import com.arangodb.entity.CollectionStatus; import com.arangodb.entity.CollectionType; import com.arangodb.entity.InvertedIndexPrimarySort; +import com.arangodb.entity.MultiDocumentEntity; import com.arangodb.entity.ReplicationFactor; import com.arangodb.util.RawBytes; import com.arangodb.util.RawJson; @@ -11,15 +12,12 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.module.SimpleModule; -import java.util.function.Supplier; +class InternalModule { -enum InternalModule implements Supplier { - INSTANCE; + static Module get(InternalSerde serde) { + SimpleModule module = new SimpleModule(); - private final SimpleModule module; - - InternalModule() { - module = new SimpleModule(); + module.addDeserializer(MultiDocumentEntity.class, new MultiDocumentEntityDeserializer(serde)); module.addSerializer(RawJson.class, InternalSerializers.RAW_JSON_SERIALIZER); module.addSerializer(InternalRequest.class, InternalSerializers.REQUEST); @@ -32,11 +30,7 @@ enum InternalModule implements Supplier { module.addDeserializer(ReplicationFactor.class, InternalDeserializers.REPLICATION_FACTOR); module.addDeserializer(InternalResponse.class, InternalDeserializers.RESPONSE); module.addDeserializer(InvertedIndexPrimarySort.Field.class, InternalDeserializers.INVERTED_INDEX_PRIMARY_SORT_FIELD); - } - @Override - public Module get() { return module; } - } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java index a4b70e2ca..9dc6b5bf6 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java @@ -3,6 +3,8 @@ import com.arangodb.arch.UsedInApi; import com.arangodb.serde.ArangoSerde; import com.arangodb.ContentType; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import java.lang.reflect.Type; @@ -59,14 +61,6 @@ default T deserialize(JsonNode node, Class clazz) { */ T deserialize(JsonNode node, Type type); - /** - * Parses the content. - * - * @param content VPack or byte encoded JSON string - * @return root of the parsed tree - */ - JsonNode parse(byte[] content); - /** * Parses the content at json pointer. * @@ -136,6 +130,16 @@ default T deserialize(byte[] content, String jsonPointer, Type type) { */ T deserializeUserData(byte[] content, Type type); + /** + * Deserializes the parsed json node and binds it to the target data type. + * The parser is not closed. + * + * @param parser json parser + * @param clazz class of target data type + * @return deserialized object + */ + T deserializeUserData(JsonParser parser, JavaType clazz); + /** * @return the user serde */ diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index f01c732d4..2c52721dd 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; @@ -39,7 +40,7 @@ final class InternalSerdeImpl implements InternalSerde { mapper.deactivateDefaultTyping(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.enable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION); - mapper.registerModule(InternalModule.INSTANCE.get()); + mapper.registerModule(InternalModule.get(this)); if (protocolModule != null) { mapper.registerModule(protocolModule); } @@ -113,16 +114,6 @@ public byte[] extract(final byte[] content, final String jsonPointer) { } } - // TODO: remove - @Override - public JsonNode parse(byte[] content) { - try { - return mapper.readTree(content); - } catch (IOException e) { - throw ArangoDBException.of(e); - } - } - @Override public JsonNode parse(byte[] content, String jsonPointer) { try { @@ -184,6 +175,19 @@ public T deserializeUserData(byte[] content, Type type) { } } + @Override + public T deserializeUserData(JsonParser parser, JavaType clazz) { + try { + if (SerdeUtils.isManagedClass(clazz.getRawClass())) { + return mapper.readerFor(clazz).readValue(parser); + } else { + return deserializeUserData(extractBytes(parser), clazz); + } + } catch (IOException e) { + throw ArangoDBException.of(e); + } + } + @Override public ArangoSerde getUserSerde() { return userSerde; diff --git a/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java new file mode 100644 index 000000000..5a3cf57ee --- /dev/null +++ b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java @@ -0,0 +1,85 @@ +package com.arangodb.internal.serde; + +import com.arangodb.entity.ErrorEntity; +import com.arangodb.entity.MultiDocumentEntity; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.BeanProperty; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.deser.ContextualDeserializer; + +import java.io.IOException; + +public class MultiDocumentEntityDeserializer extends JsonDeserializer> implements ContextualDeserializer { + private final JavaType containedType; + private final InternalSerde serde; + + MultiDocumentEntityDeserializer(InternalSerde serde) { + this(serde, null); + } + + MultiDocumentEntityDeserializer(InternalSerde serde, JavaType containedType) { + this.serde = serde; + this.containedType = containedType; + } + + @Override + public MultiDocumentEntity deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + MultiDocumentEntity multiDocument = new MultiDocumentEntity<>(); + + // silent=true returns an empty object + if (p.currentToken() == JsonToken.START_OBJECT) { + if (p.nextToken() == JsonToken.END_OBJECT) { + return multiDocument; + } else { + throw new JsonMappingException(p, "Unexpected token sequence: START_OBJECT, " + p.currentToken()); + } + } + + if (p.currentToken() != JsonToken.START_ARRAY) { + throw new JsonMappingException(p, "Expected START_ARRAY but got " + p.currentToken()); + } + p.nextToken(); + while (p.currentToken() != JsonToken.END_ARRAY) { + if (p.currentToken() != JsonToken.START_OBJECT) { + throw new JsonMappingException(p, "Expected START_OBJECT but got " + p.currentToken()); + } + p.nextToken(); + if (p.currentToken() != JsonToken.FIELD_NAME) { + throw new JsonMappingException(p, "Expected FIELD_NAME but got " + p.currentToken()); + } + String fieldName = p.getText(); + switch (fieldName) { + case "_id": + case "_key": + case "_rev": + case "_oldRev": + case "new": + case "old": + Object d = serde.deserializeUserData(p, containedType); + multiDocument.getDocuments().add(d); + multiDocument.getDocumentsAndErrors().add(d); + break; + case "error": + case "errorNum": + case "errorMessage": + ErrorEntity e = ctxt.readValue(p, ErrorEntity.class); + multiDocument.getErrors().add(e); + multiDocument.getDocumentsAndErrors().add(e); + break; + default: + throw new JsonMappingException(p, "Unrecognized field '" + fieldName + "'"); + } + p.nextToken(); // END_OBJECT + } + return multiDocument; + } + + @Override + public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) { + return new MultiDocumentEntityDeserializer(serde, ctxt.getContextualType().containedType(0)); + } +} diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 6da20d535..80cb7ec48 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -98,6 +98,7 @@ public String writeJson(final JsonNode data) { * @param parser JsonParser with current token pointing to the node to extract * @return byte array */ + // TODO: move to InternalSerdeImpl, non-static, keep reference to serde to check content-type public static byte[] extractBytes(JsonParser parser) throws IOException { JsonToken t = parser.currentToken(); if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { diff --git a/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java b/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java index b70da4d4d..d4d7017b1 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java @@ -2537,9 +2537,10 @@ void insertDocumentsReturnNew(ArangoCollection collection) { for (final DocumentCreateEntity doc : docs.getDocuments()) { assertThat(doc.getNew()).isNotNull(); final BaseDocument baseDocument = doc.getNew(); + assertThat(baseDocument.getId()).isNotNull(); assertThat(baseDocument.getKey()).isNotNull(); + assertThat(baseDocument.getRevision()).isNotNull(); } - } @ParameterizedTest diff --git a/test-functional/src/test/java/com/arangodb/ArangoDatabaseAsyncTest.java b/test-functional/src/test/java/com/arangodb/ArangoDatabaseAsyncTest.java index cc8aab2ad..bd6f45a2b 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoDatabaseAsyncTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoDatabaseAsyncTest.java @@ -675,7 +675,7 @@ void queryRawBytes(ArangoDatabaseAsync db) throws ExecutionException, Interrupte RawBytes doc = RawBytes.of(serde.serialize(Collections.singletonMap("value", 1))); RawBytes res = db.query("RETURN @doc", RawBytes.class, Collections.singletonMap("doc", doc)).get() .getResult().get(0); - JsonNode data = serde.parse(res.get()); + JsonNode data = serde.deserialize(res.get(), JsonNode.class); assertThat(data.isObject()).isTrue(); assertThat(data.get("value").isNumber()).isTrue(); assertThat(data.get("value").numberValue()).isEqualTo(1); diff --git a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java index d86f224b1..ea487e07c 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoDatabaseTest.java @@ -762,7 +762,7 @@ void queryRawBytes(ArangoDatabase db) { InternalSerde serde = db.getSerde(); RawBytes doc = RawBytes.of(serde.serialize(Collections.singletonMap("value", 1))); RawBytes res = db.query("RETURN @doc", RawBytes.class, Collections.singletonMap("doc", doc)).next(); - JsonNode data = serde.parse(res.get()); + JsonNode data = serde.deserialize(res.get(), JsonNode.class); assertThat(data.isObject()).isTrue(); assertThat(data.get("value").isNumber()).isTrue(); assertThat(data.get("value").numberValue()).isEqualTo(1); From fc8cffa35992e36ea73b36772aa9e38964bda707 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Thu, 5 Dec 2024 22:19:27 +0100 Subject: [PATCH 12/20] improved RawJson serialization --- .../arangodb/internal/serde/InternalSerializers.java | 12 +++++++++--- .../java/com/arangodb/internal/serde/SerdeUtils.java | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java index 59f226de3..9ac335666 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java @@ -6,12 +6,14 @@ import com.arangodb.internal.ArangoRequestParam; import com.arangodb.util.RawJson; import com.arangodb.internal.InternalRequest; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -22,9 +24,13 @@ public final class InternalSerializers { static final JsonSerializer RAW_JSON_SERIALIZER = new JsonSerializer() { @Override public void serialize(RawJson value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - try (JsonParser parser = SerdeUtils.INSTANCE.getJsonMapper().createParser(value.get())) { - parser.nextToken(); - gen.copyCurrentStructure(parser); + if (JsonFactory.FORMAT_NAME_JSON.equals(gen.getCodec().getFactory().getFormatName())) { + gen.writeRawValue(new RawUserDataValue(value.get().getBytes(StandardCharsets.UTF_8))); + } else { + try (JsonParser parser = SerdeUtils.INSTANCE.getJsonMapper().createParser(value.get())) { + parser.nextToken(); + gen.copyCurrentStructure(parser); + } } } }; diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 80cb7ec48..719522c08 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -5,6 +5,7 @@ import com.arangodb.entity.BaseEdgeDocument; import com.arangodb.util.RawBytes; import com.arangodb.util.RawJson; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; @@ -98,7 +99,6 @@ public String writeJson(final JsonNode data) { * @param parser JsonParser with current token pointing to the node to extract * @return byte array */ - // TODO: move to InternalSerdeImpl, non-static, keep reference to serde to check content-type public static byte[] extractBytes(JsonParser parser) throws IOException { JsonToken t = parser.currentToken(); if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { @@ -119,7 +119,7 @@ public static byte[] extractBytes(JsonParser parser) throws IOException { } } parser.finishToken(); - if ("JSON".equals(parser.getCodec().getFactory().getFormatName())) { + if (JsonFactory.FORMAT_NAME_JSON.equals(parser.getCodec().getFactory().getFormatName())) { end = (int) parser.currentLocation().getByteOffset(); } return Arrays.copyOfRange(data, start, end); From 70796f7f25a52e8520f7dc3381ffaff7187975d8 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Thu, 5 Dec 2024 22:34:40 +0100 Subject: [PATCH 13/20] wip --- .../internal/serde/MultiDocumentEntityDeserializer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java index 5a3cf57ee..a60c7b1f2 100644 --- a/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java @@ -52,6 +52,11 @@ public MultiDocumentEntity deserialize(JsonParser p, DeserializationContext c throw new JsonMappingException(p, "Expected FIELD_NAME but got " + p.currentToken()); } String fieldName = p.getText(); + // FIXME: this can potentially fail for: MultiDocumentEntity getDocuments() + // fix by scanning the 1st level field names and checking if any matches: + // - "_id" + // - "_key" + // - "_rev" switch (fieldName) { case "_id": case "_key": From 330762494058d5dab858d793b428dee48bdbbbcb Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Fri, 6 Dec 2024 16:18:43 +0100 Subject: [PATCH 14/20] v7.15.0-SNAPSHOT --- core/pom.xml | 2 +- driver/pom.xml | 2 +- http-protocol/pom.xml | 2 +- jackson-serde-json/pom.xml | 2 +- jackson-serde-vpack/pom.xml | 2 +- jsonb-serde/pom.xml | 2 +- pom.xml | 2 +- release-parent/pom.xml | 2 +- shaded/pom.xml | 2 +- test-functional/pom.xml | 2 +- test-functional/src/test/java/com/arangodb/UserAgentTest.java | 2 +- test-non-functional/pom.xml | 2 +- test-parent/pom.xml | 2 +- test-perf/pom.xml | 2 +- test-resilience/pom.xml | 2 +- tutorial/gradle/build.gradle | 2 +- tutorial/maven/pom.xml | 2 +- vst-protocol/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 7e768fc1f..587102e05 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT core diff --git a/driver/pom.xml b/driver/pom.xml index a2791cf21..bc2c2e293 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT arangodb-java-driver diff --git a/http-protocol/pom.xml b/http-protocol/pom.xml index 2a783fba1..a7e2e10e3 100644 --- a/http-protocol/pom.xml +++ b/http-protocol/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT http-protocol diff --git a/jackson-serde-json/pom.xml b/jackson-serde-json/pom.xml index fb6ed0929..417c3b619 100644 --- a/jackson-serde-json/pom.xml +++ b/jackson-serde-json/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT jackson-serde-json diff --git a/jackson-serde-vpack/pom.xml b/jackson-serde-vpack/pom.xml index 83e1a0edf..7d258b789 100644 --- a/jackson-serde-vpack/pom.xml +++ b/jackson-serde-vpack/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT jackson-serde-vpack diff --git a/jsonb-serde/pom.xml b/jsonb-serde/pom.xml index 6f883f06f..d8dffbe49 100644 --- a/jsonb-serde/pom.xml +++ b/jsonb-serde/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT jsonb-serde diff --git a/pom.xml b/pom.xml index ab863dd16..ebbb5d3f4 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.arangodb arangodb-java-driver-parent - 7.14.0 + 7.15.0-SNAPSHOT 2016 release-parent diff --git a/release-parent/pom.xml b/release-parent/pom.xml index ecb3a61f4..c674c6f35 100644 --- a/release-parent/pom.xml +++ b/release-parent/pom.xml @@ -6,7 +6,7 @@ com.arangodb arangodb-java-driver-parent - 7.14.0 + 7.15.0-SNAPSHOT pom diff --git a/shaded/pom.xml b/shaded/pom.xml index 887edb7bc..858269964 100644 --- a/shaded/pom.xml +++ b/shaded/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT arangodb-java-driver-shaded diff --git a/test-functional/pom.xml b/test-functional/pom.xml index b53baa7c6..3eaecde1d 100644 --- a/test-functional/pom.xml +++ b/test-functional/pom.xml @@ -8,7 +8,7 @@ ../test-parent com.arangodb test-parent - 7.14.0 + 7.15.0-SNAPSHOT test-functional diff --git a/test-functional/src/test/java/com/arangodb/UserAgentTest.java b/test-functional/src/test/java/com/arangodb/UserAgentTest.java index 1aac8adf1..46458f609 100644 --- a/test-functional/src/test/java/com/arangodb/UserAgentTest.java +++ b/test-functional/src/test/java/com/arangodb/UserAgentTest.java @@ -10,7 +10,7 @@ class UserAgentTest extends BaseJunit5 { - private static final String EXPECTED_VERSION = "7.14.0"; + private static final String EXPECTED_VERSION = "7.15.0-SNAPSHOT"; private static final boolean SHADED = Boolean.parseBoolean(System.getProperty("shaded")); diff --git a/test-non-functional/pom.xml b/test-non-functional/pom.xml index 8c1da5cf6..8e33bde71 100644 --- a/test-non-functional/pom.xml +++ b/test-non-functional/pom.xml @@ -8,7 +8,7 @@ ../test-parent com.arangodb test-parent - 7.14.0 + 7.15.0-SNAPSHOT test-non-functional diff --git a/test-parent/pom.xml b/test-parent/pom.xml index 684ed7b78..3199ae192 100644 --- a/test-parent/pom.xml +++ b/test-parent/pom.xml @@ -7,7 +7,7 @@ com.arangodb arangodb-java-driver-parent - 7.14.0 + 7.15.0-SNAPSHOT pom diff --git a/test-perf/pom.xml b/test-perf/pom.xml index bcf304dd3..1479f6da5 100644 --- a/test-perf/pom.xml +++ b/test-perf/pom.xml @@ -7,7 +7,7 @@ ../test-parent com.arangodb test-parent - 7.13.1 + 7.15.0-SNAPSHOT test-perf diff --git a/test-resilience/pom.xml b/test-resilience/pom.xml index fb67d0306..8af00cdd6 100644 --- a/test-resilience/pom.xml +++ b/test-resilience/pom.xml @@ -6,7 +6,7 @@ ../test-parent com.arangodb test-parent - 7.14.0 + 7.15.0-SNAPSHOT 4.0.0 diff --git a/tutorial/gradle/build.gradle b/tutorial/gradle/build.gradle index 1dc14f8f5..fa2c1c9cb 100644 --- a/tutorial/gradle/build.gradle +++ b/tutorial/gradle/build.gradle @@ -12,7 +12,7 @@ repositories { } dependencies { - implementation 'com.arangodb:arangodb-java-driver:7.14.0' + implementation 'com.arangodb:arangodb-java-driver:7.15.0-SNAPSHOT' } ext { diff --git a/tutorial/maven/pom.xml b/tutorial/maven/pom.xml index 236cbf5f4..9da9981dd 100644 --- a/tutorial/maven/pom.xml +++ b/tutorial/maven/pom.xml @@ -19,7 +19,7 @@ com.arangodb arangodb-java-driver - 7.14.0 + 7.15.0-SNAPSHOT diff --git a/vst-protocol/pom.xml b/vst-protocol/pom.xml index 60909d93f..0baa3e41a 100644 --- a/vst-protocol/pom.xml +++ b/vst-protocol/pom.xml @@ -8,7 +8,7 @@ ../release-parent com.arangodb release-parent - 7.14.0 + 7.15.0-SNAPSHOT vst-protocol From f3edee2e8a2d39fd8ae5b0cbda1eb7c4c70f94e1 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Mon, 9 Dec 2024 12:14:52 +0100 Subject: [PATCH 15/20] fixed deserialization of multiple documents --- .../internal/serde/InternalSerde.java | 10 +- .../internal/serde/InternalSerdeImpl.java | 48 ++++++- .../MultiDocumentEntityDeserializer.java | 39 ++---- .../internal/serde/UserDataDeserializer.java | 2 +- .../src/test/java/resilience/MockTest.java | 40 ++++++ .../MockTest.java => mock/SerdeTest.java} | 123 ++++++------------ .../mock/ServiceUnavailableTest.java | 65 +++++++++ 7 files changed, 204 insertions(+), 123 deletions(-) create mode 100644 test-resilience/src/test/java/resilience/MockTest.java rename test-resilience/src/test/java/resilience/{http/MockTest.java => mock/SerdeTest.java} (57%) create mode 100644 test-resilience/src/test/java/resilience/mock/ServiceUnavailableTest.java diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java index 9dc6b5bf6..d9ca65ef1 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerde.java @@ -125,10 +125,10 @@ default T deserialize(byte[] content, String jsonPointer, Type type) { * Deserializes the content and binds it to the target data type, using the user serde. * * @param content byte array to deserialize - * @param type target data type + * @param clazz class of target data type * @return deserialized object */ - T deserializeUserData(byte[] content, Type type); + T deserializeUserData(byte[] content, JavaType clazz); /** * Deserializes the parsed json node and binds it to the target data type. @@ -140,6 +140,12 @@ default T deserialize(byte[] content, String jsonPointer, Type type) { */ T deserializeUserData(JsonParser parser, JavaType clazz); + /** + * @param content byte array to deserialize + * @return whether the content represents a document (i.e. it has at least one field name equal to _id, _key, _rev) + */ + boolean isDocument(byte[] content); + /** * @return the user serde */ diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index 2c52721dd..479df52e3 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; @@ -166,12 +167,15 @@ public T deserializeUserData(byte[] content, Class clazz) { } @Override - @SuppressWarnings("unchecked") - public T deserializeUserData(byte[] content, Type type) { - if (type instanceof Class) { - return deserializeUserData(content, (Class) type); - } else { - throw new UnsupportedOperationException(); + public T deserializeUserData(byte[] content, JavaType clazz) { + try { + if (SerdeUtils.isManagedClass(clazz.getRawClass())) { + return mapper.readerFor(clazz).readValue(content); + } else { + return deserializeUserData(content, clazz); + } + } catch (IOException e) { + throw ArangoDBException.of(e); } } @@ -188,6 +192,38 @@ public T deserializeUserData(JsonParser parser, JavaType clazz) { } } + @Override + public boolean isDocument(byte[] content) { + try (JsonParser p = mapper.createParser(content)) { + if (p.nextToken() != JsonToken.START_OBJECT) { + return false; + } + + int level = 1; + while (level >= 1) { + JsonToken t = p.nextToken(); + if (level == 1 && t == JsonToken.FIELD_NAME) { + String fieldName = p.getText(); + if (fieldName.equals("_id") || fieldName.equals("_key") || fieldName.equals("_rev")) { + return true; + } + } + if (t.isStructStart()) { + level++; + } else if (t.isStructEnd()) { + level--; + } + } + + if (p.currentToken() != JsonToken.END_OBJECT) { + throw new JsonMappingException(p, "Expected END_OBJECT but got " + p.currentToken()); + } + } catch (IOException e) { + throw ArangoDBException.of(e); + } + return false; + } + @Override public ArangoSerde getUserSerde() { return userSerde; diff --git a/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java index a60c7b1f2..ca650569d 100644 --- a/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/MultiDocumentEntityDeserializer.java @@ -47,36 +47,15 @@ public MultiDocumentEntity deserialize(JsonParser p, DeserializationContext c if (p.currentToken() != JsonToken.START_OBJECT) { throw new JsonMappingException(p, "Expected START_OBJECT but got " + p.currentToken()); } - p.nextToken(); - if (p.currentToken() != JsonToken.FIELD_NAME) { - throw new JsonMappingException(p, "Expected FIELD_NAME but got " + p.currentToken()); - } - String fieldName = p.getText(); - // FIXME: this can potentially fail for: MultiDocumentEntity getDocuments() - // fix by scanning the 1st level field names and checking if any matches: - // - "_id" - // - "_key" - // - "_rev" - switch (fieldName) { - case "_id": - case "_key": - case "_rev": - case "_oldRev": - case "new": - case "old": - Object d = serde.deserializeUserData(p, containedType); - multiDocument.getDocuments().add(d); - multiDocument.getDocumentsAndErrors().add(d); - break; - case "error": - case "errorNum": - case "errorMessage": - ErrorEntity e = ctxt.readValue(p, ErrorEntity.class); - multiDocument.getErrors().add(e); - multiDocument.getDocumentsAndErrors().add(e); - break; - default: - throw new JsonMappingException(p, "Unrecognized field '" + fieldName + "'"); + byte[] element = SerdeUtils.extractBytes(p); + if (serde.isDocument(element)) { + Object d = serde.deserializeUserData(element, containedType); + multiDocument.getDocuments().add(d); + multiDocument.getDocumentsAndErrors().add(d); + } else { + ErrorEntity e = serde.deserialize(element, ErrorEntity.class); + multiDocument.getErrors().add(e); + multiDocument.getDocumentsAndErrors().add(e); } p.nextToken(); // END_OBJECT } diff --git a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java index 56b5e5703..ecb8c83f3 100644 --- a/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java +++ b/core/src/main/java/com/arangodb/internal/serde/UserDataDeserializer.java @@ -33,7 +33,7 @@ public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx if (SerdeUtils.isManagedClass(clazz)) { return p.readValueAs(clazz); } else { - return serde.deserializeUserData(SerdeUtils.extractBytes(p), targetType); + return serde.deserializeUserData(SerdeUtils.extractBytes(p), clazz); } } diff --git a/test-resilience/src/test/java/resilience/MockTest.java b/test-resilience/src/test/java/resilience/MockTest.java new file mode 100644 index 000000000..c75ecc27a --- /dev/null +++ b/test-resilience/src/test/java/resilience/MockTest.java @@ -0,0 +1,40 @@ +package resilience; + +import ch.qos.logback.classic.Level; +import com.arangodb.ArangoDB; +import com.arangodb.Protocol; +import com.arangodb.internal.net.Communication; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.mockserver.integration.ClientAndServer; + +import java.util.Collections; + +import static org.mockserver.integration.ClientAndServer.startClientAndServer; + +public class MockTest extends SingleServerTest { + + protected ClientAndServer mockServer; + protected ArangoDB arangoDB; + + public MockTest() { + super(Collections.singletonMap(Communication.class, Level.DEBUG)); + } + + @BeforeEach + void before() { + mockServer = startClientAndServer(getEndpoint().getHost(), getEndpoint().getPort()); + arangoDB = new ArangoDB.Builder() + .protocol(Protocol.HTTP_JSON) + .password(PASSWORD) + .host("127.0.0.1", mockServer.getPort()) + .build(); + } + + @AfterEach + void after() { + arangoDB.shutdown(); + mockServer.stop(); + } + +} diff --git a/test-resilience/src/test/java/resilience/http/MockTest.java b/test-resilience/src/test/java/resilience/mock/SerdeTest.java similarity index 57% rename from test-resilience/src/test/java/resilience/http/MockTest.java rename to test-resilience/src/test/java/resilience/mock/SerdeTest.java index 7e3e958d4..2d559c2aa 100644 --- a/test-resilience/src/test/java/resilience/http/MockTest.java +++ b/test-resilience/src/test/java/resilience/mock/SerdeTest.java @@ -1,99 +1,23 @@ -package resilience.http; +package resilience.mock; import ch.qos.logback.classic.Level; -import com.arangodb.ArangoDB; import com.arangodb.ArangoDBException; -import com.arangodb.Protocol; -import com.arangodb.internal.net.Communication; +import com.arangodb.entity.MultiDocumentEntity; import com.fasterxml.jackson.core.JsonParseException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.Test; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.matchers.Times; -import resilience.SingleServerTest; +import resilience.MockTest; -import java.util.Collections; -import java.util.concurrent.ExecutionException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; -import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; -class MockTest extends SingleServerTest { - - private ClientAndServer mockServer; - private ArangoDB arangoDB; - - public MockTest() { - super(Collections.singletonMap(Communication.class, Level.DEBUG)); - } - - @BeforeEach - void before() { - mockServer = startClientAndServer(getEndpoint().getHost(), getEndpoint().getPort()); - arangoDB = new ArangoDB.Builder() - .protocol(Protocol.HTTP_JSON) - .password(PASSWORD) - .host("127.0.0.1", mockServer.getPort()) - .build(); - } - - @AfterEach - void after() { - arangoDB.shutdown(); - mockServer.stop(); - } - - @Test - void retryOn503() { - arangoDB.getVersion(); - - mockServer - .when( - request() - .withMethod("GET") - .withPath("/.*/_api/version"), - Times.exactly(2) - ) - .respond( - response() - .withStatusCode(503) - .withBody("{\"error\":true,\"errorNum\":503,\"errorMessage\":\"boom\",\"code\":503}") - ); - - logs.reset(); - arangoDB.getVersion(); - assertThat(logs.getLogs()) - .filteredOn(e -> e.getLevel().equals(Level.WARN)) - .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); - } - - @Test - void retryOn503Async() throws ExecutionException, InterruptedException { - arangoDB.async().getVersion().get(); - - mockServer - .when( - request() - .withMethod("GET") - .withPath("/.*/_api/version"), - Times.exactly(2) - ) - .respond( - response() - .withStatusCode(503) - .withBody("{\"error\":true,\"errorNum\":503,\"errorMessage\":\"boom\",\"code\":503}") - ); - - logs.reset(); - arangoDB.async().getVersion().get(); - assertThat(logs.getLogs()) - .filteredOn(e -> e.getLevel().equals(Level.WARN)) - .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); - } +public class SerdeTest extends MockTest { @Test void unparsableData() { @@ -178,4 +102,35 @@ void textPlainDataWithCharset() { .hasMessageContaining("upstream timed out"); } + @Test + void getDocumentsWithErrorField() { + List keys = Arrays.asList("1", "2", "3"); + + String resp = "[" + + "{\"error\":true,\"_key\":\"1\",\"_id\":\"col/1\",\"_rev\":\"_i4otI-q---\"}," + + "{\"_key\":\"2\",\"_id\":\"col/2\",\"_rev\":\"_i4otI-q--_\"}," + + "{\"_key\":\"3\",\"_id\":\"col/3\",\"_rev\":\"_i4otI-q--A\"}" + + "]"; + + mockServer + .when( + request() + .withMethod("PUT") + .withPath("/.*/_api/document/col") + .withQueryStringParameter("onlyget", "true") + ) + .respond( + response() + .withStatusCode(200) + .withHeader("Content-Type", "application/json; charset=utf-8") + .withBody(resp.getBytes(StandardCharsets.UTF_8)) + ); + + MultiDocumentEntity res = arangoDB.db().collection("col").getDocuments(keys, JsonNode.class); + assertThat(res.getErrors()).isEmpty(); + assertThat(res.getDocuments()).hasSize(3) + .anySatisfy(d -> assertThat(d.get("_key").textValue()).isEqualTo("1")) + .anySatisfy(d -> assertThat(d.get("_key").textValue()).isEqualTo("2")) + .anySatisfy(d -> assertThat(d.get("_key").textValue()).isEqualTo("3")); + } } diff --git a/test-resilience/src/test/java/resilience/mock/ServiceUnavailableTest.java b/test-resilience/src/test/java/resilience/mock/ServiceUnavailableTest.java new file mode 100644 index 000000000..358311cf4 --- /dev/null +++ b/test-resilience/src/test/java/resilience/mock/ServiceUnavailableTest.java @@ -0,0 +1,65 @@ +package resilience.mock; + +import ch.qos.logback.classic.Level; +import org.junit.jupiter.api.Test; +import org.mockserver.matchers.Times; +import resilience.MockTest; + +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +class ServiceUnavailableTest extends MockTest { + + @Test + void retryOn503() { + arangoDB.getVersion(); + + mockServer + .when( + request() + .withMethod("GET") + .withPath("/.*/_api/version"), + Times.exactly(2) + ) + .respond( + response() + .withStatusCode(503) + .withBody("{\"error\":true,\"errorNum\":503,\"errorMessage\":\"boom\",\"code\":503}") + ); + + logs.reset(); + arangoDB.getVersion(); + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + } + + @Test + void retryOn503Async() throws ExecutionException, InterruptedException { + arangoDB.async().getVersion().get(); + + mockServer + .when( + request() + .withMethod("GET") + .withPath("/.*/_api/version"), + Times.exactly(2) + ) + .respond( + response() + .withStatusCode(503) + .withBody("{\"error\":true,\"errorNum\":503,\"errorMessage\":\"boom\",\"code\":503}") + ); + + logs.reset(); + arangoDB.async().getVersion().get(); + assertThat(logs.getLogs()) + .filteredOn(e -> e.getLevel().equals(Level.WARN)) + .anyMatch(e -> e.getFormattedMessage().contains("Could not connect to host")); + } + + +} From 88eb770a19ea88294313bff468f1e87282a381c6 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Mon, 9 Dec 2024 14:56:41 +0100 Subject: [PATCH 16/20] fixed deserialization of multiple documents to user data --- .circleci/config.yml | 4 +- .../internal/serde/InternalSerdeImpl.java | 3 +- .../com/arangodb/ArangoCollectionTest.java | 55 +++++++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ad66ad46..6af266804 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -551,7 +551,7 @@ workflows: only: - main - test: - name: test-native-ssl=<> + name: test-native-ssl=<>-(<>) matrix: parameters: native: @@ -571,7 +571,7 @@ workflows: only: - main - test-shaded: - name: test-native-shaded-ssl=<> + name: test-native-shaded-ssl=<>-(<>) matrix: parameters: native: diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index 479df52e3..ec449f6f5 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -167,12 +167,13 @@ public T deserializeUserData(byte[] content, Class clazz) { } @Override + @SuppressWarnings("unchecked") public T deserializeUserData(byte[] content, JavaType clazz) { try { if (SerdeUtils.isManagedClass(clazz.getRawClass())) { return mapper.readerFor(clazz).readValue(content); } else { - return deserializeUserData(content, clazz); + return deserializeUserData(content, (Class) clazz.getRawClass()); } } catch (IOException e) { throw ArangoDBException.of(e); diff --git a/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java b/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java index d4d7017b1..ea21964db 100644 --- a/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java +++ b/test-functional/src/test/java/com/arangodb/ArangoCollectionTest.java @@ -754,6 +754,34 @@ void getDocuments(ArangoCollection collection) { } } + @ParameterizedTest + @MethodSource("cols") + void getDocumentsUserData(ArangoCollection collection) { + Cat a = new Cat(); + a.setKey(UUID.randomUUID().toString()); + a.setName("a"); + + Cat b = new Cat(); + b.setKey(UUID.randomUUID().toString()); + b.setName("b"); + + final List values = Arrays.asList(a, b); + collection.insertDocuments(values); + final MultiDocumentEntity documents = collection.getDocuments(Arrays.asList(a.getKey(), b.getKey()), + Cat.class); + assertThat(documents).isNotNull(); + assertThat(documents.getDocuments()) + .hasSize(2) + .anySatisfy(d -> { + assertThat(d.getKey()).isEqualTo(a.getKey()); + assertThat(d.getName()).isEqualTo(a.getName()); + }) + .anySatisfy(d -> { + assertThat(d.getKey()).isEqualTo(b.getKey()); + assertThat(d.getName()).isEqualTo(b.getName()); + }); + } + @ParameterizedTest @MethodSource("cols") void getDocumentsWithCustomShardingKey(ArangoCollection c) { @@ -2413,6 +2441,33 @@ void insertDocuments(ArangoCollection collection) { assertThat(docs.getErrors()).isEmpty(); } + @ParameterizedTest + @MethodSource("cols") + void insertDocumentsReturnNewUserData(ArangoCollection collection) { + Cat a = new Cat(); + a.setKey(UUID.randomUUID().toString()); + a.setName("a"); + + Cat b = new Cat(); + b.setKey(UUID.randomUUID().toString()); + b.setName("b"); + + final List values = Arrays.asList(a, b); + MultiDocumentEntity> res = + collection.insertDocuments(values, new DocumentCreateOptions().returnNew(true), Cat.class); + assertThat(res).isNotNull(); + assertThat(res.getDocuments()) + .hasSize(2) + .anySatisfy(d -> { + assertThat(d.getKey()).isEqualTo(a.getKey()); + assertThat(d.getNew().getName()).isEqualTo(a.getName()); + }) + .anySatisfy(d -> { + assertThat(d.getKey()).isEqualTo(b.getKey()); + assertThat(d.getNew().getName()).isEqualTo(b.getName()); + }); + } + @ParameterizedTest @MethodSource("cols") void insertDocumentsOverwriteModeUpdate(ArangoCollection collection) { From 9266dd2c1da1ef230fb9ebe2d6157703843564da Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Mon, 9 Dec 2024 15:03:13 +0100 Subject: [PATCH 17/20] fixed compatibility with older Jackson versions --- .circleci/config.yml | 4 ++-- .../main/java/com/arangodb/internal/serde/SerdeUtils.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6af266804..79a7b4c38 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -551,7 +551,7 @@ workflows: only: - main - test: - name: test-native-ssl=<>-(<>) + name: test-native-ssl=<>-<> matrix: parameters: native: @@ -571,7 +571,7 @@ workflows: only: - main - test-shaded: - name: test-native-shaded-ssl=<>-(<>) + name: test-native-shaded-ssl=<>-<> matrix: parameters: native: diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 719522c08..23804d3e6 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -104,9 +104,9 @@ public static byte[] extractBytes(JsonParser parser) throws IOException { if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { throw new ArangoDBException("Unexpected token: " + t); } - byte[] data = (byte[]) parser.currentTokenLocation().contentReference().getRawContent(); - int start = (int) parser.currentTokenLocation().getByteOffset(); - int end = (int) parser.currentLocation().getByteOffset(); + byte[] data = (byte[]) parser.getTokenLocation().contentReference().getRawContent(); + int start = (int) parser.getTokenLocation().getByteOffset(); + int end = (int) parser.getCurrentLocation().getByteOffset(); if (t.isStructStart()) { int open = 1; while (open > 0) { @@ -120,7 +120,7 @@ public static byte[] extractBytes(JsonParser parser) throws IOException { } parser.finishToken(); if (JsonFactory.FORMAT_NAME_JSON.equals(parser.getCodec().getFactory().getFormatName())) { - end = (int) parser.currentLocation().getByteOffset(); + end = (int) parser.getCurrentLocation().getByteOffset(); } return Arrays.copyOfRange(data, start, end); } From cdce964cf06769b3fb3648b9d24ddd97dea5b623 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Mon, 9 Dec 2024 15:44:36 +0100 Subject: [PATCH 18/20] fixed compatibility with older Jackson versions --- .circleci/config.yml | 4 ++-- .../com/arangodb/internal/serde/InternalDeserializers.java | 2 +- .../java/com/arangodb/internal/serde/InternalSerdeImpl.java | 6 +++--- .../com/arangodb/internal/serde/InternalSerializers.java | 2 +- .../main/java/com/arangodb/internal/serde/SerdeUtils.java | 3 ++- pom.xml | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79a7b4c38..33bc358ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -535,8 +535,8 @@ workflows: matrix: parameters: args: - - '-Dadb.jackson.version=2.18.0' - - '-Dadb.jackson.version=2.17.2' + - '-Dadb.jackson.version=2.18.2' + - '-Dadb.jackson.version=2.17.3' - '-Dadb.jackson.version=2.16.2' - '-Dadb.jackson.version=2.15.4' - '-Dadb.jackson.version=2.14.3' diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java index 8eaee6f56..20b3ce3b7 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalDeserializers.java @@ -35,7 +35,7 @@ public RawJson deserialize(JsonParser p, DeserializationContext ctxt) throws IOE return RawJson.of(new String(SerdeUtils.extractBytes(p), StandardCharsets.UTF_8)); } else { StringWriter w = new StringWriter(); - try (JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().createGenerator(w)) { + try (JsonGenerator gen = SerdeUtils.INSTANCE.getJsonMapper().getFactory().createGenerator(w)) { gen.copyCurrentStructure(p); gen.flush(); } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java index ec449f6f5..1cb1412b2 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerdeImpl.java @@ -84,7 +84,7 @@ public byte[] extract(final byte[] content, final String jsonPointer) { throw new ArangoDBException("Unsupported JSON pointer: " + jsonPointer); } String[] parts = jsonPointer.substring(1).split("/"); - try (JsonParser parser = mapper.createParser(content)) { + try (JsonParser parser = mapper.getFactory().createParser(content)) { int match = 0; int level = 0; JsonToken token = parser.nextToken(); @@ -144,7 +144,7 @@ public byte[] serializeUserData(Object value) { @Override public byte[] serializeCollectionUserData(Iterable value) { ByteArrayOutputStream os = new ByteArrayOutputStream(); - try (JsonGenerator gen = mapper.createGenerator(os)) { + try (JsonGenerator gen = mapper.getFactory().createGenerator(os)) { gen.writeStartArray(); for (Object o : value) { gen.writeRawValue(new RawUserDataValue(serializeUserData(o))); @@ -195,7 +195,7 @@ public T deserializeUserData(JsonParser parser, JavaType clazz) { @Override public boolean isDocument(byte[] content) { - try (JsonParser p = mapper.createParser(content)) { + try (JsonParser p = mapper.getFactory().createParser(content)) { if (p.nextToken() != JsonToken.START_OBJECT) { return false; } diff --git a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java index 9ac335666..db156dff8 100644 --- a/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java +++ b/core/src/main/java/com/arangodb/internal/serde/InternalSerializers.java @@ -27,7 +27,7 @@ public void serialize(RawJson value, JsonGenerator gen, SerializerProvider seria if (JsonFactory.FORMAT_NAME_JSON.equals(gen.getCodec().getFactory().getFormatName())) { gen.writeRawValue(new RawUserDataValue(value.get().getBytes(StandardCharsets.UTF_8))); } else { - try (JsonParser parser = SerdeUtils.INSTANCE.getJsonMapper().createParser(value.get())) { + try (JsonParser parser = SerdeUtils.INSTANCE.getJsonMapper().getFactory().createParser(value.get())) { parser.nextToken(); gen.copyCurrentStructure(parser); } diff --git a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java index 23804d3e6..9783386c6 100644 --- a/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java +++ b/core/src/main/java/com/arangodb/internal/serde/SerdeUtils.java @@ -99,12 +99,13 @@ public String writeJson(final JsonNode data) { * @param parser JsonParser with current token pointing to the node to extract * @return byte array */ + @SuppressWarnings("deprecation") public static byte[] extractBytes(JsonParser parser) throws IOException { JsonToken t = parser.currentToken(); if (t.isStructEnd() || t == JsonToken.FIELD_NAME) { throw new ArangoDBException("Unexpected token: " + t); } - byte[] data = (byte[]) parser.getTokenLocation().contentReference().getRawContent(); + byte[] data = (byte[]) parser.getTokenLocation().getSourceRef(); int start = (int) parser.getTokenLocation().getByteOffset(); int end = (int) parser.getCurrentLocation().getByteOffset(); if (t.isStructStart()) { diff --git a/pom.xml b/pom.xml index ebbb5d3f4..86712ef6d 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ com.fasterxml.jackson jackson-bom - 2.18.0 + 2.18.2 import pom From 4dc9ae4e992e5c8b5ec528a987be5ab6ed17038e Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Tue, 10 Dec 2024 12:30:35 +0100 Subject: [PATCH 19/20] benchmark deserialize multiple documents --- .../test/java/com/arangodb/SerdeBench.java | 69 +++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/test-perf/src/test/java/com/arangodb/SerdeBench.java b/test-perf/src/test/java/com/arangodb/SerdeBench.java index def0c298a..ae3d90bc3 100644 --- a/test-perf/src/test/java/com/arangodb/SerdeBench.java +++ b/test-perf/src/test/java/com/arangodb/SerdeBench.java @@ -1,5 +1,10 @@ package com.arangodb; +import com.arangodb.entity.MultiDocumentEntity; +import com.arangodb.internal.ArangoCollectionImpl; +import com.arangodb.internal.ArangoDatabaseImpl; +import com.arangodb.internal.ArangoExecutor; +import com.arangodb.internal.InternalResponse; import com.arangodb.internal.serde.InternalSerde; import com.arangodb.internal.serde.InternalSerdeProvider; import com.arangodb.jackson.dataformat.velocypack.VPackMapper; @@ -25,6 +30,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -39,6 +45,35 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @Fork(1) public class SerdeBench { + public static class MyCol extends ArangoCollectionImpl { + static ArangoDB jsonAdb = new ArangoDB.Builder() + .host("127.0.0.1", 8529) + .protocol(Protocol.HTTP_JSON) + .build(); + + static ArangoDB vpackAdb = new ArangoDB.Builder() + .host("127.0.0.1", 8529) + .protocol(Protocol.HTTP_VPACK) + .build(); + + private MyCol(ArangoDB adb) { + super((ArangoDatabaseImpl) adb.db(), "foo"); + } + + public static MyCol ofJson() { + return new MyCol(jsonAdb); + } + + public static MyCol ofVpack() { + return new MyCol(vpackAdb); + } + + @Override + public ArangoExecutor.ResponseDeserializer> getDocumentsResponseDeserializer(Class type) { + return super.getDocumentsResponseDeserializer(type); + } + } + @State(Scope.Benchmark) public static class Data { public final byte[] vpack; @@ -46,26 +81,38 @@ public static class Data { public final RawBytes rawJsonBytes; public final RawBytes rawVPackBytes; public final RawJson rawJson; + public final MyCol jsonCol = MyCol.ofJson(); + public final MyCol vpackCol = MyCol.ofVpack(); + public final InternalResponse jsonResp = new InternalResponse(); + public final InternalResponse vpackResp = new InternalResponse(); public Data() { ObjectMapper jsonMapper = new ObjectMapper(); VPackMapper vpackMapper = new VPackMapper(); try { - String str = new String(Files.readAllBytes( - Paths.get(SerdeBench.class.getResource("/api-docs.json").toURI()))); - JsonNode jn = jsonMapper.readTree(str); - + JsonNode jn = readFile("/api-docs.json", jsonMapper); json = jsonMapper.writeValueAsBytes(jn); vpack = vpackMapper.writeValueAsBytes(jn); rawJsonBytes = RawBytes.of(json); rawVPackBytes = RawBytes.of(vpack); rawJson = RawJson.of(jsonMapper.writeValueAsString(jsonMapper.readTree(json))); + JsonNode docs = readFile("/multi-docs.json", jsonMapper); + jsonResp.setResponseCode(200); + jsonResp.setBody(jsonMapper.writeValueAsBytes(docs)); + vpackResp.setResponseCode(200); + vpackResp.setBody(vpackMapper.writeValueAsBytes(docs)); } catch (Exception e) { throw new RuntimeException(e); } } + + private JsonNode readFile(String filename, ObjectMapper mapper) throws IOException, URISyntaxException { + String str = new String(Files.readAllBytes( + Paths.get(SerdeBench.class.getResource(filename).toURI()))); + return mapper.readTree(str); + } } public static void main(String[] args) throws RunnerException, IOException { @@ -122,4 +169,18 @@ public void extractBytesJson(Data data, Blackhole bh) { ); } + @Benchmark + public void deserializeDocsJson(Data data, Blackhole bh) { + bh.consume( + data.jsonCol.getDocumentsResponseDeserializer(RawBytes.class).deserialize(data.jsonResp) + ); + } + + @Benchmark + public void deserializeDocsVPack(Data data, Blackhole bh) { + bh.consume( + data.vpackCol.getDocumentsResponseDeserializer(RawBytes.class).deserialize(data.vpackResp) + ); + } + } From 094d08d0aa4312d87aaf3a40c636b269f84c4188 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Tue, 10 Dec 2024 13:56:21 +0100 Subject: [PATCH 20/20] updated jackson-dataformat-velocypack 4.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86712ef6d..727b4f4bb 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ com.arangodb jackson-dataformat-velocypack - 4.5.0-SNAPSHOT + 4.5.0 com.arangodb