From 065171d32722219f931b81c356d61efc55b8fd9d Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Fri, 3 Nov 2023 17:29:55 +0100 Subject: [PATCH 01/21] rebased --- client-spark/build.gradle.kts | 38 +++++ .../java/client/ITDeltaSharingClient.java | 60 ++++++++ .../java/models/AddTableToSchemaInput.java | 38 +++++ .../test/java/models/CreateShareInput.java | 35 +++++ .../src/test/java/models/CreateStorage.java | 39 ++++++ .../test/java/models/CreateTableInput.java | 56 ++++++++ .../java/models/MrFoxDeltaTableSchema.java | 13 ++ .../src/test/java/models/ProviderInput.java | 26 ++++ .../src/test/java/models/S3Properties.java | 37 +++++ .../src/test/java/models/S3TestConfig.java | 26 ++++ .../src/test/java/utils/EnvReader.java | 38 +++++ .../java/utils/StorageManagerInitializer.java | 130 ++++++++++++++++++ .../src/test/resources/MrFoxProfile.json | 6 + protocol/delta-sharing-protocol-api.yml | 35 +++-- server/app/build.gradle.kts | 14 +- .../api/configuration/JsonConfiguration.java | 2 + .../api/deltasharing/DeltaMappers.java | 5 +- .../server/DeltaSharesApiImpl.java | 26 ++-- .../whitefox/api/server/WhitefoxMappers.java | 2 +- .../api/deltasharing/SampleTables.java | 7 +- .../server/DeltaSharesApiImplTest.java | 2 +- .../io/whitefox/core/ReadTableRequest.java | 19 ++- settings.gradle.kts | 1 + 23 files changed, 614 insertions(+), 41 deletions(-) create mode 100644 client-spark/build.gradle.kts create mode 100644 client-spark/src/test/java/client/ITDeltaSharingClient.java create mode 100644 client-spark/src/test/java/models/AddTableToSchemaInput.java create mode 100644 client-spark/src/test/java/models/CreateShareInput.java create mode 100644 client-spark/src/test/java/models/CreateStorage.java create mode 100644 client-spark/src/test/java/models/CreateTableInput.java create mode 100644 client-spark/src/test/java/models/MrFoxDeltaTableSchema.java create mode 100644 client-spark/src/test/java/models/ProviderInput.java create mode 100644 client-spark/src/test/java/models/S3Properties.java create mode 100644 client-spark/src/test/java/models/S3TestConfig.java create mode 100644 client-spark/src/test/java/utils/EnvReader.java create mode 100644 client-spark/src/test/java/utils/StorageManagerInitializer.java create mode 100644 client-spark/src/test/resources/MrFoxProfile.json diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts new file mode 100644 index 000000000..2688acda0 --- /dev/null +++ b/client-spark/build.gradle.kts @@ -0,0 +1,38 @@ +plugins { + java + id("com.diffplug.spotless") + id("whitefox.java-conventions") +} + +group = "io.whitefox" +version = "spark-connector" + +repositories { + mavenCentral() +} + +val hadoopVersion = "3.3.6" +dependencies { + // DELTA + testImplementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) + testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2") + + //SPARK + testImplementation("org.apache.spark:spark-core_2.12:3.3.2") + testImplementation("org.apache.spark:spark-sql_2.12:3.3.2") + testImplementation("com.github.mrpowers:spark-fast-tests_2.12:1.3.0") + + //JUNIT + testImplementation("org.junit.jupiter:junit-jupiter:5.8.1") + +} + +// region code formatting +spotless { + java {} +} +// endregion + +tasks.getByName("test") { + useJUnitPlatform() +} \ No newline at end of file diff --git a/client-spark/src/test/java/client/ITDeltaSharingClient.java b/client-spark/src/test/java/client/ITDeltaSharingClient.java new file mode 100644 index 000000000..2f6e3203b --- /dev/null +++ b/client-spark/src/test/java/client/ITDeltaSharingClient.java @@ -0,0 +1,60 @@ +package client; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.github.mrpowers.spark.fast.tests.DatasetComparer; +import java.net.URISyntaxException; +import java.util.List; +import models.*; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import scala.collection.GenMap; +import utils.StorageManagerInitializer; + +public class ITDeltaSharingClient implements DatasetComparer { + + private final String tablePath = String.format( + "%s#%s.%s.%s", + getClass().getClassLoader().getResource("MrFoxProfile.json"), + "s3share", + "s3schema", + "s3Table1"); + + private final SparkSession spark = SparkSession.builder() + .appName("delta sharing client test") + .master("local[1, 4]") + .getOrCreate(); + + @BeforeAll + static void initStorageManager() throws URISyntaxException, JsonProcessingException { + new StorageManagerInitializer().initStorageManager(); + } + + @Test + void showS3Table1withQueryTableApi() { + var ds = spark.read().format("deltaSharing").load(tablePath); + var expectedSchema = new StructType(new StructField[] { + new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty())) + }); + var expectedData = spark + .createDataFrame( + List.of( + new MrFoxDeltaTableSchema(0), + new MrFoxDeltaTableSchema(3), + new MrFoxDeltaTableSchema(2), + new MrFoxDeltaTableSchema(1), + new MrFoxDeltaTableSchema(4)), + MrFoxDeltaTableSchema.class) + .toDF(); + + assertEquals(expectedSchema.json(), ds.schema().json()); + assertEquals(5, ds.count()); + assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); + } +} \ No newline at end of file diff --git a/client-spark/src/test/java/models/AddTableToSchemaInput.java b/client-spark/src/test/java/models/AddTableToSchemaInput.java new file mode 100644 index 000000000..a81dacb6f --- /dev/null +++ b/client-spark/src/test/java/models/AddTableToSchemaInput.java @@ -0,0 +1,38 @@ +package models; + +public class AddTableToSchemaInput { + private final String name; + private final TableReference reference; + + public AddTableToSchemaInput(String name, TableReference reference) { + this.name = name; + this.reference = reference; + } + + public String getName() { + return name; + } + + public TableReference getReference() { + return reference; + } + + public static class TableReference { + + private final String providerName; + private final String name; + + public TableReference(String providerName, String name) { + this.providerName = providerName; + this.name = name; + } + + public String getProviderName() { + return providerName; + } + + public String getName() { + return name; + } + } +} diff --git a/client-spark/src/test/java/models/CreateShareInput.java b/client-spark/src/test/java/models/CreateShareInput.java new file mode 100644 index 000000000..4d22334a9 --- /dev/null +++ b/client-spark/src/test/java/models/CreateShareInput.java @@ -0,0 +1,35 @@ +package models; + +import java.util.List; + +public class CreateShareInput { + + private final String name; + private final String comment; + private final List recipients; + private final List schemas; + + public CreateShareInput( + String name, String comment, List recipients, List schemas) { + this.name = name; + this.comment = comment; + this.recipients = recipients; + this.schemas = schemas; + } + + public String getName() { + return name; + } + + public String getComment() { + return comment; + } + + public List getRecipients() { + return recipients; + } + + public List getSchemas() { + return schemas; + } +} diff --git a/client-spark/src/test/java/models/CreateStorage.java b/client-spark/src/test/java/models/CreateStorage.java new file mode 100644 index 000000000..500f5a246 --- /dev/null +++ b/client-spark/src/test/java/models/CreateStorage.java @@ -0,0 +1,39 @@ +package models; + +public class CreateStorage { + + private final String name; + private final String comment; + private final String type; + private final S3Properties properties; + private final boolean skipValidation; + + public CreateStorage( + String name, String comment, String type, S3Properties properties, boolean skipValidation) { + this.name = name; + this.comment = comment; + this.type = type; + this.properties = properties; + this.skipValidation = skipValidation; + } + + public String getName() { + return name; + } + + public String getComment() { + return comment; + } + + public String getType() { + return type; + } + + public S3Properties getProperties() { + return properties; + } + + public boolean isSkipValidation() { + return skipValidation; + } +} diff --git a/client-spark/src/test/java/models/CreateTableInput.java b/client-spark/src/test/java/models/CreateTableInput.java new file mode 100644 index 000000000..d9720c76c --- /dev/null +++ b/client-spark/src/test/java/models/CreateTableInput.java @@ -0,0 +1,56 @@ +package models; + +public class CreateTableInput { + private final String name; + private final String comment; + private final boolean skipValidation; + private final DeltaTableProperties properties; + + public CreateTableInput( + String name, String comment, boolean skipValidation, DeltaTableProperties properties) { + this.name = name; + this.comment = comment; + this.skipValidation = skipValidation; + this.properties = properties; + } + + public String getName() { + return name; + } + + public String getComment() { + return comment; + } + + public boolean isSkipValidation() { + return skipValidation; + } + + public DeltaTableProperties getProperties() { + return properties; + } + + public static class DeltaTableProperties { + private final String type; + private final String location; + private final String additionalProperties; + + public DeltaTableProperties(String type, String location, String additionalProperties) { + this.type = type; + this.location = location; + this.additionalProperties = additionalProperties; + } + + public String getType() { + return type; + } + + public String getLocation() { + return location; + } + + public String getAdditionalProperties() { + return additionalProperties; + } + } +} diff --git a/client-spark/src/test/java/models/MrFoxDeltaTableSchema.java b/client-spark/src/test/java/models/MrFoxDeltaTableSchema.java new file mode 100644 index 000000000..0dd49bcc2 --- /dev/null +++ b/client-spark/src/test/java/models/MrFoxDeltaTableSchema.java @@ -0,0 +1,13 @@ +package models; + +public class MrFoxDeltaTableSchema { + private final long id; + + public MrFoxDeltaTableSchema(long id) { + this.id = id; + } + + public long getId() { + return id; + } +} diff --git a/client-spark/src/test/java/models/ProviderInput.java b/client-spark/src/test/java/models/ProviderInput.java new file mode 100644 index 000000000..8f78031e5 --- /dev/null +++ b/client-spark/src/test/java/models/ProviderInput.java @@ -0,0 +1,26 @@ +package models; + +public class ProviderInput { + + private final String name; + private final String storageName; + private final String metastoreName; + + public ProviderInput(String name, String storageName, String metastoreName) { + this.name = name; + this.storageName = storageName; + this.metastoreName = metastoreName; + } + + public String getName() { + return name; + } + + public String getStorageName() { + return storageName; + } + + public String getMetastoreName() { + return metastoreName; + } +} diff --git a/client-spark/src/test/java/models/S3Properties.java b/client-spark/src/test/java/models/S3Properties.java new file mode 100644 index 000000000..f00427837 --- /dev/null +++ b/client-spark/src/test/java/models/S3Properties.java @@ -0,0 +1,37 @@ +package models; + +public class S3Properties { + private final AwsCredentials credentials; + + public S3Properties(AwsCredentials credentials) { + this.credentials = credentials; + } + + public AwsCredentials getCredentials() { + return credentials; + } + + public static class AwsCredentials { + private final String awsAccessKeyId; + private final String awsSecretAccessKey; + private final String region; + + public AwsCredentials(String awsAccessKeyId, String awsSecretAccessKey, String region) { + this.awsAccessKeyId = awsAccessKeyId; + this.awsSecretAccessKey = awsSecretAccessKey; + this.region = region; + } + + public String getAwsAccessKeyId() { + return awsAccessKeyId; + } + + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; + } + + public String getRegion() { + return region; + } + } +} diff --git a/client-spark/src/test/java/models/S3TestConfig.java b/client-spark/src/test/java/models/S3TestConfig.java new file mode 100644 index 000000000..efa5862b3 --- /dev/null +++ b/client-spark/src/test/java/models/S3TestConfig.java @@ -0,0 +1,26 @@ +package models; + +public class S3TestConfig { + + private final String region; + private final String accessKey; + private final String secretKey; + + public String getRegion() { + return region; + } + + public String getAccessKey() { + return accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public S3TestConfig(String region, String accessKey, String secretKey) { + this.region = region; + this.accessKey = accessKey; + this.secretKey = secretKey; + } +} diff --git a/client-spark/src/test/java/utils/EnvReader.java b/client-spark/src/test/java/utils/EnvReader.java new file mode 100644 index 000000000..248e9a5ef --- /dev/null +++ b/client-spark/src/test/java/utils/EnvReader.java @@ -0,0 +1,38 @@ +package utils; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Properties; +import models.S3TestConfig; + +public class EnvReader { + + public S3TestConfig readS3TestConfig() { + Properties properties = readProperties(); + String region = properties.getProperty("WHITEFOX_TEST_AWS_REGION"); + String accessKey = properties.getProperty("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"); + String secretKey = properties.getProperty("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"); + return new S3TestConfig(region, accessKey, secretKey); + } + + private Properties readProperties() { + Properties properties = new Properties(); + FileInputStream input = null; + try { + input = new FileInputStream(String.format( + "%s/.env", + Paths.get(".").toAbsolutePath().getParent().getParent().toUri().getPath())); + properties.load(input); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + input.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return properties; + } +} diff --git a/client-spark/src/test/java/utils/StorageManagerInitializer.java b/client-spark/src/test/java/utils/StorageManagerInitializer.java new file mode 100644 index 000000000..bab61e5e3 --- /dev/null +++ b/client-spark/src/test/java/utils/StorageManagerInitializer.java @@ -0,0 +1,130 @@ +package utils; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.List; +import models.*; + +public class StorageManagerInitializer { + + private final ObjectWriter objectWriter; + private final HttpClient httpClient; + private final String server; + private final EnvReader envReader; + + public StorageManagerInitializer() { + this.objectWriter = new ObjectMapper().writer(); + this.httpClient = HttpClient.newBuilder().build(); + this.server = "http://localhost:8080"; + this.envReader = new EnvReader(); + } + + public void initStorageManager() throws JsonProcessingException, URISyntaxException { + List.of( + createStorageRequest(objectWriter), + createProviderRequest(objectWriter), + createTableRequest(objectWriter), + createShareRequest(objectWriter), + createSchemaRequest(objectWriter), + addTableToSchemaRequest(objectWriter)) + .stream() + .forEach(request -> { + try { + callWhiteFoxServer(httpClient, request); + } catch (Throwable e) { + throw new RuntimeException(e); + } + }); + } + + private HttpRequest addTableToSchemaRequest(ObjectWriter writer) + throws JsonProcessingException, URISyntaxException { + AddTableToSchemaInput addTableToSchemaInput = new AddTableToSchemaInput( + "s3Table1", new AddTableToSchemaInput.TableReference("MrFoxProvider", "s3Table1")); + return HttpRequest.newBuilder() + .header("content", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(addTableToSchemaInput))) + .uri(URI.create(String.format( + "%s/%s/%s/%s/tables", server, "/whitefox-api/v1/shares", "s3share", "s3schema"))) + .build(); + } + + private HttpRequest createSchemaRequest(ObjectWriter writer) + throws JsonProcessingException, URISyntaxException { + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.noBody()) + .header("content", "application/json") + .uri(URI.create( + String.format("%s/%s/%s/%s", server, "/whitefox-api/v1/shares", "s3share", "s3schema"))) + .build(); + } + + private HttpRequest createShareRequest(ObjectWriter writer) + throws JsonProcessingException, URISyntaxException { + CreateShareInput createShareInput = + new CreateShareInput("s3share", "", List.of("Mr.Fox"), List.of()); + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createShareInput))) + .header("content", "application/json") + .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/shares"))) + .build(); + } + + private HttpRequest createTableRequest(ObjectWriter writer) + throws JsonProcessingException, URISyntaxException { + CreateTableInput createTableInput = new CreateTableInput( + "s3Table1", + "", + true, + new CreateTableInput.DeltaTableProperties( + "delta", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table", null)); + + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createTableInput))) + .header("content", "application/json") + .uri(URI.create( + String.format("%s/%s/%s/tables", server, "whitefox-api/v1/providers", "MrFoxProvider"))) + .build(); + } + + private HttpRequest createProviderRequest(ObjectWriter writer) throws JsonProcessingException { + ProviderInput providerInput = new ProviderInput("MrFoxProvider", "MrFoxStorage", null); + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(providerInput))) + .header("content", "application/json") + .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/providers"))) + .build(); + } + + private HttpRequest createStorageRequest(ObjectWriter writer) throws JsonProcessingException { + S3TestConfig s3TestConfig = envReader.readS3TestConfig(); + CreateStorage createStorage = new CreateStorage( + "MrFoxStorage", + "", + "s3", + new S3Properties(new S3Properties.AwsCredentials( + s3TestConfig.getAccessKey(), s3TestConfig.getSecretKey(), s3TestConfig.getRegion())), + true); + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createStorage))) + .header("content", "application/json") + .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/storage"))) + .build(); + } + + private void callWhiteFoxServer(HttpClient httpClient, HttpRequest httpRequest) + throws IOException, InterruptedException { + HttpResponse response = + httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + assertTrue(List.of(200, 201).contains(response.statusCode())); + } +} diff --git a/client-spark/src/test/resources/MrFoxProfile.json b/client-spark/src/test/resources/MrFoxProfile.json new file mode 100644 index 000000000..05c48f70a --- /dev/null +++ b/client-spark/src/test/resources/MrFoxProfile.json @@ -0,0 +1,6 @@ +{ + "shareCredentialsVersion": 1, + "endpoint": "http://localhost:8080/delta-api/v1/", + "bearerToken": "fakeToken", + "expirationTime": null +} \ No newline at end of file diff --git a/protocol/delta-sharing-protocol-api.yml b/protocol/delta-sharing-protocol-api.yml index 32c822709..fbc435827 100644 --- a/protocol/delta-sharing-protocol-api.yml +++ b/protocol/delta-sharing-protocol-api.yml @@ -419,6 +419,12 @@ paths: description: 'Starting Timestamp ISO8601 format, in the UTC timezone' schema: type: string + - in: header + name: delta-sharing-capabilities + required: false + description: 'Delta Sharing Capabilities' + schema: + type: string requestBody: required: true content: @@ -670,7 +676,7 @@ components: items: type: string jsonPredicateHints: - type: object + type: string description: | query predicates on partition columns specified using a structured JSON format. When it’s present, the server will try to use the predicates to filter table's @@ -680,19 +686,20 @@ components: If the server encounters any errors during predicate processing (for example, invalid syntax or non existing columns), it will skip filtering and return all the files. When it’s absent, the server will return all the files in the table. - properties: - op: - $ref: '#/components/schemas/Ops' - children: - type: string - name: - type: string - value: - type: string - valueType: - type: string +# properties: +# op: +# $ref: '#/components/schemas/Ops' +# children: +# type: string +# name: +# type: string +# value: +# type: string +# valueType: +# type: string limitHint: type: integer + format: int64 example: 1000 description: | It’s a hint from the client to tell the server how many rows the @@ -717,6 +724,7 @@ components: timestamp. This is only supported on tables with history sharing enabled. startingVersion: type: integer + format: int64 example: 1000 description: | an optional version number. If set, will return all data change files @@ -724,6 +732,7 @@ components: in the delta log. endingVersion: type: integer + format: int64 example: 1000 description: | an optional version number, only used if startingVersion is set. If set, @@ -836,7 +845,7 @@ components: MetadataObject: type: object properties: - metadata: + metaData: type: object properties: id: diff --git a/server/app/build.gradle.kts b/server/app/build.gradle.kts index c00d3bcf3..4d1f7ba57 100644 --- a/server/app/build.gradle.kts +++ b/server/app/build.gradle.kts @@ -11,12 +11,12 @@ val quarkusPlatformArtifactId: String by project val quarkusPlatformVersion: String by project // region dependencies - +val hadoopVersion = "3.3.6" dependencies { // INTERNAL implementation(project(":server:core")) implementation(project(":server:persistence:memory")) - + // QUARKUS implementation(enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}")) implementation("io.quarkus:quarkus-arc") @@ -29,6 +29,10 @@ dependencies { implementation("org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1.1") implementation("jakarta.validation:jakarta.validation-api:3.0.2") + // DELTA + implementation("io.delta:delta-standalone_2.13:0.6.0") + implementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) + // TEST testImplementation(testFixtures(project(":server:core"))) testImplementation("io.quarkus:quarkus-junit5") @@ -36,6 +40,12 @@ dependencies { testImplementation("io.rest-assured:json-path") testImplementation("org.openapi4j:openapi-operation-validator:1.0.7") testImplementation("org.openapi4j:openapi-operation-restassured:1.0.7") + + //AWS + compileOnly("com.amazonaws:aws-java-sdk-bom:1.12.367") + compileOnly("com.amazonaws:aws-java-sdk-s3:1.12.367") + implementation(String.format("org.apache.hadoop:hadoop-aws:%s", hadoopVersion)) + } // endregion diff --git a/server/app/src/main/java/io/whitefox/api/configuration/JsonConfiguration.java b/server/app/src/main/java/io/whitefox/api/configuration/JsonConfiguration.java index 4ae92e567..079903d63 100644 --- a/server/app/src/main/java/io/whitefox/api/configuration/JsonConfiguration.java +++ b/server/app/src/main/java/io/whitefox/api/configuration/JsonConfiguration.java @@ -1,6 +1,7 @@ package io.whitefox.api.configuration; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import io.quarkus.arc.All; import io.quarkus.jackson.ObjectMapperCustomizer; @@ -16,6 +17,7 @@ public class JsonConfiguration { ObjectMapper objectMapper(@All List customizers) { ObjectMapper mapper = new ObjectMapper(); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); // Apply all ObjectMapperCustomizer beans (incl. Quarkus) for (ObjectMapperCustomizer customizer : customizers) { customizer.customize(mapper); diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 1ba496428..f5847b100 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -26,8 +26,7 @@ public static io.whitefox.api.deltasharing.model.v1.generated.Schema schema2api( } public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { - if (request.getEndingVersion() != null || request.getStartingVersion() != null) - throw new NotImplementedYet(); + if (request.getEndingVersion() != null) throw new NotImplementedYet(); if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( request.getPredicateHints(), @@ -57,7 +56,7 @@ public static TableQueryResponseObject readTableResult2api(ReadTableResult readT private static MetadataObject metadata2Api(Metadata metadata) { return new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id(metadata.id()) .name(metadata.name().orElse(null)) .description(metadata.description().orElse(null)) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java b/server/app/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java index b1450be75..8bb0b18ca 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java @@ -188,15 +188,25 @@ public Response queryTable( String schema, String table, QueryRequest queryRequest, - String startingTimestamp) { - + String startingTimestamp, + String deltaSharingCapabilities) { return wrapExceptions( - () -> Response.ok( - tableQueryResponseSerializer.serialize( - DeltaMappers.readTableResult2api(deltaSharesService.queryTable( - share, schema, table, DeltaMappers.api2ReadTableRequest(queryRequest)))), - ndjsonMediaType) - .build(), + () -> optionalToNotFound( + deltaSharesService.getTableVersion(share, schema, table, startingTimestamp), + version -> Response.ok( + tableQueryResponseSerializer.serialize( + DeltaMappers.readTableResult2api(deltaSharesService.queryTable( + share, + schema, + table, + DeltaMappers.api2ReadTableRequest(queryRequest)))), + ndjsonMediaType) + .header(DELTA_TABLE_VERSION_HEADER, version) + .header( + DELTA_SHARE_CAPABILITIES_HEADER, + getResponseFormatHeader( + DeltaMappers.toHeaderCapabilitiesMap(deltaSharingCapabilities))) + .build()), exceptionToResponse); } diff --git a/server/app/src/main/java/io/whitefox/api/server/WhitefoxMappers.java b/server/app/src/main/java/io/whitefox/api/server/WhitefoxMappers.java index c08516e83..ac5dd4d36 100644 --- a/server/app/src/main/java/io/whitefox/api/server/WhitefoxMappers.java +++ b/server/app/src/main/java/io/whitefox/api/server/WhitefoxMappers.java @@ -175,7 +175,7 @@ public static MetastoreType api2MetastoreType( private static MetadataObject metadata2Api(Metadata metadata) { return new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id(metadata.id()) .name(metadata.name().orElse(null)) .description(metadata.description().orElse(null)) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index d490d9a53..e2dd1564f 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -8,6 +8,7 @@ import io.whitefox.api.deltasharing.model.v1.generated.*; import io.whitefox.core.InternalTable; import io.whitefox.core.Principal; +import io.whitefox.core.Share; import io.whitefox.core.SharedTable; import io.whitefox.persistence.StorageManager; import io.whitefox.persistence.memory.InMemoryStorageManager; @@ -53,7 +54,7 @@ public static StorageManager createStorageManager() { } public static final MetadataObject deltaTable1Metadata = new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") .name("table1") .format(new FormatObject().provider("parquet")) @@ -64,7 +65,7 @@ public static StorageManager createStorageManager() { ._configuration(Map.of())); public static final MetadataObject s3DeltaTable1Metadata = new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id("ed2297c4-8bb8-4c74-963d-8fed6bebfd8b") .name("s3Table1") .format(new FormatObject().provider("parquet")) @@ -74,7 +75,7 @@ public static StorageManager createStorageManager() { .version(0L) ._configuration(Map.of())); public static final MetadataObject deltaTableWithHistory1Metadata = new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") .name("table-with-history") .format(new FormatObject().provider("parquet")) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index ae520482d..6a89f0f32 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -192,7 +192,7 @@ public void tableMetadata() throws IOException { objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); assertEquals( new MetadataObject() - .metadata(new MetadataObjectMetadata() + .metaData(new MetadataObjectMetaData() .id("56d48189-cdbc-44f2-9b0e-2bded4c79ed7") .name("table1") .format(new FormatObject().provider("parquet")) diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index b2232f48c..aa6df33ab 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -9,12 +9,11 @@ public interface ReadTableRequest { public static class ReadTableVersion implements ReadTableRequest { private final List predicateHints; - private final Optional limitHint; + private final Optional limitHint; private final Long version; - public ReadTableVersion( - List predicateHints, Optional limitHint, Long version) { + public ReadTableVersion(List predicateHints, Optional limitHint, Long version) { this.predicateHints = predicateHints; this.limitHint = limitHint; this.version = version; @@ -24,7 +23,7 @@ public List predicateHints() { return predicateHints; } - public Optional limitHint() { + public Optional limitHint() { return limitHint; } @@ -61,11 +60,11 @@ public String toString() { public static class ReadTableAsOfTimestamp implements ReadTableRequest { private final List predicateHints; - private final Optional limitHint; + private final Optional limitHint; private final Long timestamp; public ReadTableAsOfTimestamp( - List predicateHints, Optional limitHint, Long timestamp) { + List predicateHints, Optional limitHint, Long timestamp) { this.predicateHints = predicateHints; this.limitHint = limitHint; this.timestamp = timestamp; @@ -101,7 +100,7 @@ public List predicateHints() { return predicateHints; } - public Optional limitHint() { + public Optional limitHint() { return limitHint; } @@ -112,9 +111,9 @@ public Long timestamp() { public static class ReadTableCurrentVersion implements ReadTableRequest { private final List predicateHints; - private final Optional limitHint; + private final Optional limitHint; - public ReadTableCurrentVersion(List predicateHints, Optional limitHint) { + public ReadTableCurrentVersion(List predicateHints, Optional limitHint) { this.predicateHints = predicateHints; this.limitHint = limitHint; } @@ -123,7 +122,7 @@ public List predicateHints() { return predicateHints; } - public Optional limitHint() { + public Optional limitHint() { return limitHint; } diff --git a/settings.gradle.kts b/settings.gradle.kts index 8aa5e835f..7c9fd8e30 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,6 +4,7 @@ include("server:core") include("server:persistence:memory") include("server:app") include("docsite") +include("client-spark") pluginManagement { val quarkusPluginVersion: String by settings From 1d180fcf00ecad7fb5813c28bf85c49afd97e21e Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 23 Nov 2023 14:37:50 +0100 Subject: [PATCH 02/21] remove delta and aws dependencies from server:app --- server/app/build.gradle.kts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/app/build.gradle.kts b/server/app/build.gradle.kts index 4d1f7ba57..c85b7577c 100644 --- a/server/app/build.gradle.kts +++ b/server/app/build.gradle.kts @@ -29,10 +29,6 @@ dependencies { implementation("org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1.1") implementation("jakarta.validation:jakarta.validation-api:3.0.2") - // DELTA - implementation("io.delta:delta-standalone_2.13:0.6.0") - implementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) - // TEST testImplementation(testFixtures(project(":server:core"))) testImplementation("io.quarkus:quarkus-junit5") @@ -40,12 +36,6 @@ dependencies { testImplementation("io.rest-assured:json-path") testImplementation("org.openapi4j:openapi-operation-validator:1.0.7") testImplementation("org.openapi4j:openapi-operation-restassured:1.0.7") - - //AWS - compileOnly("com.amazonaws:aws-java-sdk-bom:1.12.367") - compileOnly("com.amazonaws:aws-java-sdk-s3:1.12.367") - implementation(String.format("org.apache.hadoop:hadoop-aws:%s", hadoopVersion)) - } // endregion From a02298dbd94c888571a023c41bcb0df3a3573760 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 23 Nov 2023 15:47:10 +0100 Subject: [PATCH 03/21] trying to autogenerate the http client --- client-spark/build.gradle.kts | 26 +++++++++++++++++++ .../api}/client/ITDeltaSharingClient.java | 15 ++++++----- .../api}/models/AddTableToSchemaInput.java | 2 +- .../api}/models/CreateShareInput.java | 2 +- .../whitefox/api}/models/CreateStorage.java | 2 +- .../api}/models/CreateTableInput.java | 2 +- .../api}/models/MrFoxDeltaTableSchema.java | 2 +- .../whitefox/api}/models/ProviderInput.java | 2 +- .../whitefox/api}/models/S3Properties.java | 2 +- .../whitefox/api}/models/S3TestConfig.java | 2 +- .../whitefox/api}/utils/EnvReader.java | 5 ++-- .../api}/utils/StorageManagerInitializer.java | 13 +++++----- 12 files changed, 52 insertions(+), 23 deletions(-) rename client-spark/src/test/java/{ => io/whitefox/api}/client/ITDeltaSharingClient.java (93%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/AddTableToSchemaInput.java (95%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/CreateShareInput.java (95%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/CreateStorage.java (96%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/CreateTableInput.java (97%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/MrFoxDeltaTableSchema.java (84%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/ProviderInput.java (93%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/S3Properties.java (96%) rename client-spark/src/test/java/{ => io/whitefox/api}/models/S3TestConfig.java (93%) rename client-spark/src/test/java/{ => io/whitefox/api}/utils/EnvReader.java (93%) rename client-spark/src/test/java/{ => io/whitefox/api}/utils/StorageManagerInitializer.java (97%) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 2688acda0..f2ab7e302 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -13,6 +13,10 @@ repositories { val hadoopVersion = "3.3.6" dependencies { + // OPENAPI + implementation("org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1.1") + implementation("org.openapitools:jackson-databind-nullable:0.2.6") + // DELTA testImplementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2") @@ -35,4 +39,26 @@ spotless { tasks.getByName("test") { useJUnitPlatform() +} + +val openApiCodeGenDir = "generated/openapi" +val generatedCodeDirectory = generatedCodeDirectory(layout, openApiCodeGenDir) + +tasks.register("openapiGenerateClientApi") { + generatorName.set("java") + inputSpec.set("$rootDir/protocol/whitefox-protocol-api.yml") + library.set("native") + outputDir.set(generatedCodeDirectory) + additionalProperties.set(mapOf( + "apiPackage" to "io.whitefox.api.client", + "invokerPackage" to "io.whitefox.api.utils", + "modelPackage" to "io.whitefox.api.client.model", + "dateLibrary" to "java8", + "sourceFolder" to "src/gen/java", + "openApiNullable" to "true", + "annotationLibrary" to "none", + "serializationLibrary" to "jackson", + "useJakartaEe" to "true", + "useRuntimeException" to "true" + )) } \ No newline at end of file diff --git a/client-spark/src/test/java/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java similarity index 93% rename from client-spark/src/test/java/client/ITDeltaSharingClient.java rename to client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index 2f6e3203b..cd7ba0e9c 100644 --- a/client-spark/src/test/java/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -1,12 +1,9 @@ -package client; - -import static org.junit.jupiter.api.Assertions.assertEquals; +package io.whitefox.api.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.mrpowers.spark.fast.tests.DatasetComparer; -import java.net.URISyntaxException; -import java.util.List; -import models.*; +import io.whitefox.api.models.MrFoxDeltaTableSchema; +import io.whitefox.api.utils.StorageManagerInitializer; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.Metadata; @@ -15,7 +12,11 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import scala.collection.GenMap; -import utils.StorageManagerInitializer; + +import java.net.URISyntaxException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; public class ITDeltaSharingClient implements DatasetComparer { diff --git a/client-spark/src/test/java/models/AddTableToSchemaInput.java b/client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java similarity index 95% rename from client-spark/src/test/java/models/AddTableToSchemaInput.java rename to client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java index a81dacb6f..7eeec40dc 100644 --- a/client-spark/src/test/java/models/AddTableToSchemaInput.java +++ b/client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class AddTableToSchemaInput { private final String name; diff --git a/client-spark/src/test/java/models/CreateShareInput.java b/client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java similarity index 95% rename from client-spark/src/test/java/models/CreateShareInput.java rename to client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java index 4d22334a9..c286824d4 100644 --- a/client-spark/src/test/java/models/CreateShareInput.java +++ b/client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; import java.util.List; diff --git a/client-spark/src/test/java/models/CreateStorage.java b/client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java similarity index 96% rename from client-spark/src/test/java/models/CreateStorage.java rename to client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java index 500f5a246..98afe50ec 100644 --- a/client-spark/src/test/java/models/CreateStorage.java +++ b/client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class CreateStorage { diff --git a/client-spark/src/test/java/models/CreateTableInput.java b/client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java similarity index 97% rename from client-spark/src/test/java/models/CreateTableInput.java rename to client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java index d9720c76c..72e492d17 100644 --- a/client-spark/src/test/java/models/CreateTableInput.java +++ b/client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class CreateTableInput { private final String name; diff --git a/client-spark/src/test/java/models/MrFoxDeltaTableSchema.java b/client-spark/src/test/java/io/whitefox/api/models/MrFoxDeltaTableSchema.java similarity index 84% rename from client-spark/src/test/java/models/MrFoxDeltaTableSchema.java rename to client-spark/src/test/java/io/whitefox/api/models/MrFoxDeltaTableSchema.java index 0dd49bcc2..0f01846a3 100644 --- a/client-spark/src/test/java/models/MrFoxDeltaTableSchema.java +++ b/client-spark/src/test/java/io/whitefox/api/models/MrFoxDeltaTableSchema.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class MrFoxDeltaTableSchema { private final long id; diff --git a/client-spark/src/test/java/models/ProviderInput.java b/client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java similarity index 93% rename from client-spark/src/test/java/models/ProviderInput.java rename to client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java index 8f78031e5..011d5cb85 100644 --- a/client-spark/src/test/java/models/ProviderInput.java +++ b/client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class ProviderInput { diff --git a/client-spark/src/test/java/models/S3Properties.java b/client-spark/src/test/java/io/whitefox/api/models/S3Properties.java similarity index 96% rename from client-spark/src/test/java/models/S3Properties.java rename to client-spark/src/test/java/io/whitefox/api/models/S3Properties.java index f00427837..8a0c422b9 100644 --- a/client-spark/src/test/java/models/S3Properties.java +++ b/client-spark/src/test/java/io/whitefox/api/models/S3Properties.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class S3Properties { private final AwsCredentials credentials; diff --git a/client-spark/src/test/java/models/S3TestConfig.java b/client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java similarity index 93% rename from client-spark/src/test/java/models/S3TestConfig.java rename to client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java index efa5862b3..75e163fb8 100644 --- a/client-spark/src/test/java/models/S3TestConfig.java +++ b/client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java @@ -1,4 +1,4 @@ -package models; +package io.whitefox.api.models; public class S3TestConfig { diff --git a/client-spark/src/test/java/utils/EnvReader.java b/client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java similarity index 93% rename from client-spark/src/test/java/utils/EnvReader.java rename to client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java index 248e9a5ef..3ab01e3fe 100644 --- a/client-spark/src/test/java/utils/EnvReader.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java @@ -1,10 +1,11 @@ -package utils; +package io.whitefox.api.utils; + +import io.whitefox.api.models.S3TestConfig; import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Paths; import java.util.Properties; -import models.S3TestConfig; public class EnvReader { diff --git a/client-spark/src/test/java/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java similarity index 97% rename from client-spark/src/test/java/utils/StorageManagerInitializer.java rename to client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java index bab61e5e3..47a634063 100644 --- a/client-spark/src/test/java/utils/StorageManagerInitializer.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java @@ -1,10 +1,10 @@ -package utils; - -import static org.junit.jupiter.api.Assertions.assertTrue; +package io.whitefox.api.utils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import io.whitefox.api.models.*; + import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -12,7 +12,9 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.List; -import models.*; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertTrue; public class StorageManagerInitializer { @@ -29,14 +31,13 @@ public StorageManagerInitializer() { } public void initStorageManager() throws JsonProcessingException, URISyntaxException { - List.of( + Stream.of( createStorageRequest(objectWriter), createProviderRequest(objectWriter), createTableRequest(objectWriter), createShareRequest(objectWriter), createSchemaRequest(objectWriter), addTableToSchemaRequest(objectWriter)) - .stream() .forEach(request -> { try { callWhiteFoxServer(httpClient, request); From 434b5e60dc67de1e170c3cd3aff97d9d5bcba861 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Thu, 23 Nov 2023 17:17:35 +0200 Subject: [PATCH 04/21] Fix build --- client-spark/build.gradle.kts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index f2ab7e302..fa6de8b8c 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -16,6 +16,8 @@ dependencies { // OPENAPI implementation("org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1.1") implementation("org.openapitools:jackson-databind-nullable:0.2.6") + testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") + testImplementation("jakarta.annotation:jakarta.annotation-api:2.1.1") // DELTA testImplementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) @@ -44,7 +46,7 @@ tasks.getByName("test") { val openApiCodeGenDir = "generated/openapi" val generatedCodeDirectory = generatedCodeDirectory(layout, openApiCodeGenDir) -tasks.register("openapiGenerateClientApi") { +val whitefoxGenerate = tasks.register("openapiGenerateClientApi") { generatorName.set("java") inputSpec.set("$rootDir/protocol/whitefox-protocol-api.yml") library.set("native") @@ -61,4 +63,18 @@ tasks.register("ope "useJakartaEe" to "true", "useRuntimeException" to "true" )) +} + +sourceSets { + getByName("test") { + java { + srcDir("${generatedCodeDirectory(layout, openApiCodeGenDir)}/src/gen/java") + } + } +} + +tasks.withType { + options.encoding = "UTF-8" + options.compilerArgs.add("-parameters") + dependsOn(whitefoxGenerate) } \ No newline at end of file From 009bfe10334aecab822a8b956c1649e9c6761a95 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 23 Nov 2023 17:07:43 +0100 Subject: [PATCH 05/21] use the openapi autogen client --- client-spark/build.gradle.kts | 8 +- .../api/client/ITDeltaSharingClient.java | 6 +- .../api/models/AddTableToSchemaInput.java | 38 ---- .../whitefox/api/models/CreateShareInput.java | 35 ---- .../io/whitefox/api/models/CreateStorage.java | 39 ----- .../whitefox/api/models/CreateTableInput.java | 56 ------ .../io/whitefox/api/models/ProviderInput.java | 26 --- .../io/whitefox/api/models/S3Properties.java | 37 ---- .../java/io/whitefox/api/utils/EnvReader.java | 39 ----- .../api/{models => utils}/S3TestConfig.java | 10 +- .../api/utils/StorageManagerInitializer.java | 162 ++++++------------ server/core/build.gradle.kts | 3 - 12 files changed, 70 insertions(+), 389 deletions(-) delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/models/S3Properties.java delete mode 100644 client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java rename client-spark/src/test/java/io/whitefox/api/{models => utils}/S3TestConfig.java (61%) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index fa6de8b8c..2ffbcc577 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -43,10 +43,14 @@ tasks.getByName("test") { useJUnitPlatform() } +tasks.withType { + environment = env.allVariables +} + val openApiCodeGenDir = "generated/openapi" val generatedCodeDirectory = generatedCodeDirectory(layout, openApiCodeGenDir) -val whitefoxGenerate = tasks.register("openapiGenerateClientApi") { +val whiteFoxGenerate = tasks.register("openapiGenerateClientApi") { generatorName.set("java") inputSpec.set("$rootDir/protocol/whitefox-protocol-api.yml") library.set("native") @@ -76,5 +80,5 @@ sourceSets { tasks.withType { options.encoding = "UTF-8" options.compilerArgs.add("-parameters") - dependsOn(whitefoxGenerate) + dependsOn(whiteFoxGenerate) } \ No newline at end of file diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index cd7ba0e9c..aecdb1e1a 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -1,6 +1,5 @@ package io.whitefox.api.client; -import com.fasterxml.jackson.core.JsonProcessingException; import com.github.mrpowers.spark.fast.tests.DatasetComparer; import io.whitefox.api.models.MrFoxDeltaTableSchema; import io.whitefox.api.utils.StorageManagerInitializer; @@ -13,7 +12,6 @@ import org.junit.jupiter.api.Test; import scala.collection.GenMap; -import java.net.URISyntaxException; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,7 +31,7 @@ public class ITDeltaSharingClient implements DatasetComparer { .getOrCreate(); @BeforeAll - static void initStorageManager() throws URISyntaxException, JsonProcessingException { + static void initStorageManager() { new StorageManagerInitializer().initStorageManager(); } @@ -58,4 +56,4 @@ void showS3Table1withQueryTableApi() { assertEquals(5, ds.count()); assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); } -} \ No newline at end of file +} diff --git a/client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java b/client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java deleted file mode 100644 index 7eeec40dc..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/AddTableToSchemaInput.java +++ /dev/null @@ -1,38 +0,0 @@ -package io.whitefox.api.models; - -public class AddTableToSchemaInput { - private final String name; - private final TableReference reference; - - public AddTableToSchemaInput(String name, TableReference reference) { - this.name = name; - this.reference = reference; - } - - public String getName() { - return name; - } - - public TableReference getReference() { - return reference; - } - - public static class TableReference { - - private final String providerName; - private final String name; - - public TableReference(String providerName, String name) { - this.providerName = providerName; - this.name = name; - } - - public String getProviderName() { - return providerName; - } - - public String getName() { - return name; - } - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java b/client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java deleted file mode 100644 index c286824d4..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/CreateShareInput.java +++ /dev/null @@ -1,35 +0,0 @@ -package io.whitefox.api.models; - -import java.util.List; - -public class CreateShareInput { - - private final String name; - private final String comment; - private final List recipients; - private final List schemas; - - public CreateShareInput( - String name, String comment, List recipients, List schemas) { - this.name = name; - this.comment = comment; - this.recipients = recipients; - this.schemas = schemas; - } - - public String getName() { - return name; - } - - public String getComment() { - return comment; - } - - public List getRecipients() { - return recipients; - } - - public List getSchemas() { - return schemas; - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java b/client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java deleted file mode 100644 index 98afe50ec..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/CreateStorage.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.whitefox.api.models; - -public class CreateStorage { - - private final String name; - private final String comment; - private final String type; - private final S3Properties properties; - private final boolean skipValidation; - - public CreateStorage( - String name, String comment, String type, S3Properties properties, boolean skipValidation) { - this.name = name; - this.comment = comment; - this.type = type; - this.properties = properties; - this.skipValidation = skipValidation; - } - - public String getName() { - return name; - } - - public String getComment() { - return comment; - } - - public String getType() { - return type; - } - - public S3Properties getProperties() { - return properties; - } - - public boolean isSkipValidation() { - return skipValidation; - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java b/client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java deleted file mode 100644 index 72e492d17..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/CreateTableInput.java +++ /dev/null @@ -1,56 +0,0 @@ -package io.whitefox.api.models; - -public class CreateTableInput { - private final String name; - private final String comment; - private final boolean skipValidation; - private final DeltaTableProperties properties; - - public CreateTableInput( - String name, String comment, boolean skipValidation, DeltaTableProperties properties) { - this.name = name; - this.comment = comment; - this.skipValidation = skipValidation; - this.properties = properties; - } - - public String getName() { - return name; - } - - public String getComment() { - return comment; - } - - public boolean isSkipValidation() { - return skipValidation; - } - - public DeltaTableProperties getProperties() { - return properties; - } - - public static class DeltaTableProperties { - private final String type; - private final String location; - private final String additionalProperties; - - public DeltaTableProperties(String type, String location, String additionalProperties) { - this.type = type; - this.location = location; - this.additionalProperties = additionalProperties; - } - - public String getType() { - return type; - } - - public String getLocation() { - return location; - } - - public String getAdditionalProperties() { - return additionalProperties; - } - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java b/client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java deleted file mode 100644 index 011d5cb85..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/ProviderInput.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.whitefox.api.models; - -public class ProviderInput { - - private final String name; - private final String storageName; - private final String metastoreName; - - public ProviderInput(String name, String storageName, String metastoreName) { - this.name = name; - this.storageName = storageName; - this.metastoreName = metastoreName; - } - - public String getName() { - return name; - } - - public String getStorageName() { - return storageName; - } - - public String getMetastoreName() { - return metastoreName; - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/S3Properties.java b/client-spark/src/test/java/io/whitefox/api/models/S3Properties.java deleted file mode 100644 index 8a0c422b9..000000000 --- a/client-spark/src/test/java/io/whitefox/api/models/S3Properties.java +++ /dev/null @@ -1,37 +0,0 @@ -package io.whitefox.api.models; - -public class S3Properties { - private final AwsCredentials credentials; - - public S3Properties(AwsCredentials credentials) { - this.credentials = credentials; - } - - public AwsCredentials getCredentials() { - return credentials; - } - - public static class AwsCredentials { - private final String awsAccessKeyId; - private final String awsSecretAccessKey; - private final String region; - - public AwsCredentials(String awsAccessKeyId, String awsSecretAccessKey, String region) { - this.awsAccessKeyId = awsAccessKeyId; - this.awsSecretAccessKey = awsSecretAccessKey; - this.region = region; - } - - public String getAwsAccessKeyId() { - return awsAccessKeyId; - } - - public String getAwsSecretAccessKey() { - return awsSecretAccessKey; - } - - public String getRegion() { - return region; - } - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java b/client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java deleted file mode 100644 index 3ab01e3fe..000000000 --- a/client-spark/src/test/java/io/whitefox/api/utils/EnvReader.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.whitefox.api.utils; - -import io.whitefox.api.models.S3TestConfig; - -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Properties; - -public class EnvReader { - - public S3TestConfig readS3TestConfig() { - Properties properties = readProperties(); - String region = properties.getProperty("WHITEFOX_TEST_AWS_REGION"); - String accessKey = properties.getProperty("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"); - String secretKey = properties.getProperty("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"); - return new S3TestConfig(region, accessKey, secretKey); - } - - private Properties readProperties() { - Properties properties = new Properties(); - FileInputStream input = null; - try { - input = new FileInputStream(String.format( - "%s/.env", - Paths.get(".").toAbsolutePath().getParent().getParent().toUri().getPath())); - properties.load(input); - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - input.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return properties; - } -} diff --git a/client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java similarity index 61% rename from client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java rename to client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java index 75e163fb8..3f1a15f85 100644 --- a/client-spark/src/test/java/io/whitefox/api/models/S3TestConfig.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java @@ -1,7 +1,6 @@ -package io.whitefox.api.models; +package io.whitefox.api.utils; public class S3TestConfig { - private final String region; private final String accessKey; private final String secretKey; @@ -23,4 +22,11 @@ public S3TestConfig(String region, String accessKey, String secretKey) { this.accessKey = accessKey; this.secretKey = secretKey; } + + public static S3TestConfig loadFromEnv() { + return new S3TestConfig( + System.getenv().get("WHITEFOX_TEST_AWS_REGION"), + System.getenv().get("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"), + System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY")); + } } diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java index 47a634063..fd6639943 100644 --- a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java @@ -1,131 +1,77 @@ package io.whitefox.api.utils; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import io.whitefox.api.models.*; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; +import io.whitefox.api.client.*; +import io.whitefox.api.client.model.*; import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Map; public class StorageManagerInitializer { - - private final ObjectWriter objectWriter; - private final HttpClient httpClient; - private final String server; - private final EnvReader envReader; + private final S3TestConfig s3TestConfig; + private final StorageV1Api storageV1Api; + private final ProviderV1Api providerV1Api; + private final TableV1Api tableV1Api; + private final ShareV1Api shareV1Api; + private final SchemaV1Api schemaV1Api; public StorageManagerInitializer() { - this.objectWriter = new ObjectMapper().writer(); - this.httpClient = HttpClient.newBuilder().build(); - this.server = "http://localhost:8080"; - this.envReader = new EnvReader(); + var apiClient = new ApiClient(); + this.s3TestConfig = S3TestConfig.loadFromEnv(); + this.storageV1Api = new StorageV1Api(apiClient); + this.providerV1Api = new ProviderV1Api(apiClient); + this.tableV1Api = new TableV1Api(apiClient); + this.shareV1Api = new ShareV1Api(apiClient); + this.schemaV1Api = new SchemaV1Api(apiClient); } - public void initStorageManager() throws JsonProcessingException, URISyntaxException { - Stream.of( - createStorageRequest(objectWriter), - createProviderRequest(objectWriter), - createTableRequest(objectWriter), - createShareRequest(objectWriter), - createSchemaRequest(objectWriter), - addTableToSchemaRequest(objectWriter)) - .forEach(request -> { - try { - callWhiteFoxServer(httpClient, request); - } catch (Throwable e) { - throw new RuntimeException(e); - } - }); + public void initStorageManager() { + storageV1Api.createStorage(createStorageRequest(s3TestConfig)); + providerV1Api.addProvider(addProviderRequest()); + tableV1Api.createTableInProvider(addProviderRequest().getName(), createTableRequest()); + shareV1Api.createShare(createShareRequest()); + schemaV1Api.createSchema(createShareRequest().getName(), createSchemaRequest()); + schemaV1Api.addTableToSchema( + createShareRequest().getName(), createSchemaRequest(), addTableToSchemaRequest()); } - private HttpRequest addTableToSchemaRequest(ObjectWriter writer) - throws JsonProcessingException, URISyntaxException { - AddTableToSchemaInput addTableToSchemaInput = new AddTableToSchemaInput( - "s3Table1", new AddTableToSchemaInput.TableReference("MrFoxProvider", "s3Table1")); - return HttpRequest.newBuilder() - .header("content", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(addTableToSchemaInput))) - .uri(URI.create(String.format( - "%s/%s/%s/%s/tables", server, "/whitefox-api/v1/shares", "s3share", "s3schema"))) - .build(); + private String createSchemaRequest() { + return "s3schema"; } - private HttpRequest createSchemaRequest(ObjectWriter writer) - throws JsonProcessingException, URISyntaxException { - return HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.noBody()) - .header("content", "application/json") - .uri(URI.create( - String.format("%s/%s/%s/%s", server, "/whitefox-api/v1/shares", "s3share", "s3schema"))) - .build(); + private AddTableToSchemaRequest addTableToSchemaRequest() { + return new AddTableToSchemaRequest() + .name("s3Table1") + .reference(new TableReference().providerName("MrFoxProvider").name("s3Table1")); } - private HttpRequest createShareRequest(ObjectWriter writer) - throws JsonProcessingException, URISyntaxException { - CreateShareInput createShareInput = - new CreateShareInput("s3share", "", List.of("Mr.Fox"), List.of()); - return HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createShareInput))) - .header("content", "application/json") - .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/shares"))) - .build(); - } - - private HttpRequest createTableRequest(ObjectWriter writer) - throws JsonProcessingException, URISyntaxException { - CreateTableInput createTableInput = new CreateTableInput( - "s3Table1", - "", - true, - new CreateTableInput.DeltaTableProperties( - "delta", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table", null)); - - return HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createTableInput))) - .header("content", "application/json") - .uri(URI.create( - String.format("%s/%s/%s/tables", server, "whitefox-api/v1/providers", "MrFoxProvider"))) - .build(); + private CreateShareInput createShareRequest() { + return new CreateShareInput().name("s3share").recipients(List.of("Mr.Fox")).schemas(List.of()); } - private HttpRequest createProviderRequest(ObjectWriter writer) throws JsonProcessingException { - ProviderInput providerInput = new ProviderInput("MrFoxProvider", "MrFoxStorage", null); - return HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(providerInput))) - .header("content", "application/json") - .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/providers"))) - .build(); + private CreateTableInput createTableRequest() { + return new CreateTableInput() + .name("s3Table1") + .skipValidation(true) + .properties(Map.of( + "type", "delta", + "location", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table")); } - private HttpRequest createStorageRequest(ObjectWriter writer) throws JsonProcessingException { - S3TestConfig s3TestConfig = envReader.readS3TestConfig(); - CreateStorage createStorage = new CreateStorage( - "MrFoxStorage", - "", - "s3", - new S3Properties(new S3Properties.AwsCredentials( - s3TestConfig.getAccessKey(), s3TestConfig.getSecretKey(), s3TestConfig.getRegion())), - true); - return HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(writer.writeValueAsString(createStorage))) - .header("content", "application/json") - .uri(URI.create(String.format("%s/%s", server, "whitefox-api/v1/storage"))) - .build(); + private ProviderInput addProviderRequest() { + return new ProviderInput() + .name("MrFoxProvider") + .storageName("MrFoxStorage") + .metastoreName(null); } - private void callWhiteFoxServer(HttpClient httpClient, HttpRequest httpRequest) - throws IOException, InterruptedException { - HttpResponse response = - httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - assertTrue(List.of(200, 201).contains(response.statusCode())); + private CreateStorage createStorageRequest(S3TestConfig s3TestConfig) { + return new CreateStorage() + .name("MrFoxStorage") + .type(CreateStorage.TypeEnum.S3) + .properties(new StorageProperties(new S3Properties() + .credentials(new SimpleAwsCredentials() + .region(s3TestConfig.getRegion()) + .awsAccessKeyId(s3TestConfig.getAccessKey()) + .awsSecretAccessKey(s3TestConfig.getSecretKey())))) + .skipValidation(true); } } diff --git a/server/core/build.gradle.kts b/server/core/build.gradle.kts index b529ee65a..0fd2f8dc2 100644 --- a/server/core/build.gradle.kts +++ b/server/core/build.gradle.kts @@ -1,6 +1,3 @@ -import org.gradle.api.tasks.testing.logging.TestExceptionFormat -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - plugins { `java-library` `java-test-fixtures` From 54a02568ac21ec522a297085da85d2ed8d3763b4 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 23 Nov 2023 17:08:23 +0100 Subject: [PATCH 06/21] apply spotless --- .../java/io/whitefox/api/client/ITDeltaSharingClient.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index aecdb1e1a..422fb6995 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -1,8 +1,11 @@ package io.whitefox.api.client; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.github.mrpowers.spark.fast.tests.DatasetComparer; import io.whitefox.api.models.MrFoxDeltaTableSchema; import io.whitefox.api.utils.StorageManagerInitializer; +import java.util.List; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.Metadata; @@ -12,10 +15,6 @@ import org.junit.jupiter.api.Test; import scala.collection.GenMap; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - public class ITDeltaSharingClient implements DatasetComparer { private final String tablePath = String.format( From 701b96d410e2929b86072be47ca95fcc48d33d2d Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Wed, 29 Nov 2023 17:36:55 +0100 Subject: [PATCH 07/21] run IT tests in CI/CD --- .github/workflows/compile.yaml | 10 ++++++++++ client-spark/build.gradle.kts | 4 ++-- .../io/whitefox/api/client/ITDeltaSharingClient.java | 1 + .../io/whitefox/api/deltasharing/SampleTables.java | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index d31b328c9..7a4e9512b 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -35,6 +35,16 @@ jobs: ./gradlew build testNative --no-daemon ./gradlew server:app:printVersion --no-daemon -q shell: bash + - name: Run integration test + shell: bash + env: + WHITEFOX_TEST_AWS_REGION: ${{ vars.WHITEFOX_AWS_REGION }} + WHITEFOX_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.WHITEFOX_AWS_ACCESS_KEY_ID }} + WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} + run: | + java -jar server/build/quarkus-app/quarkus-run.jar & + ./gradlew :client-spark:test + kill -9 %1 - name: Build container image if: runner.os == 'Linux' run: | diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 2ffbcc577..1a58646e8 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -11,7 +11,6 @@ repositories { mavenCentral() } -val hadoopVersion = "3.3.6" dependencies { // OPENAPI implementation("org.eclipse.microprofile.openapi:microprofile-openapi-api:3.1.1") @@ -20,7 +19,7 @@ dependencies { testImplementation("jakarta.annotation:jakarta.annotation-api:2.1.1") // DELTA - testImplementation(String.format("org.apache.hadoop:hadoop-common:%s", hadoopVersion)) + testImplementation("org.apache.hadoop:hadoop-common:3.3.6") testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2") //SPARK @@ -45,6 +44,7 @@ tasks.getByName("test") { tasks.withType { environment = env.allVariables + systemProperty ("java.util.logging.manager", "java.util.logging.LogManager") //TODO modularize the whitefox-conventions plugin } val openApiCodeGenDir = "generated/openapi" diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index 422fb6995..a4b35622b 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -26,6 +26,7 @@ public class ITDeltaSharingClient implements DatasetComparer { private final SparkSession spark = SparkSession.builder() .appName("delta sharing client test") + .config("spark.driver.host", "localhost") .master("local[1, 4]") .getOrCreate(); diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index e2dd1564f..777cc5523 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -8,7 +8,6 @@ import io.whitefox.api.deltasharing.model.v1.generated.*; import io.whitefox.core.InternalTable; import io.whitefox.core.Principal; -import io.whitefox.core.Share; import io.whitefox.core.SharedTable; import io.whitefox.persistence.StorageManager; import io.whitefox.persistence.memory.InMemoryStorageManager; From ab7f78f34ce58eebfa4fad24a034097922bc1ba0 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Wed, 29 Nov 2023 20:31:22 +0100 Subject: [PATCH 08/21] fix gradle build: Task ':client-spark:spotlessJava' uses this output of task ':client-spark:openapiGenerateClientApi' without declaring an explicit or implicit dependency --- client-spark/build.gradle.kts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 1a58646e8..6b47b93b0 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -1,3 +1,6 @@ +import com.diffplug.gradle.spotless.SpotlessApply +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + plugins { java id("com.diffplug.spotless") @@ -50,7 +53,8 @@ tasks.withType { val openApiCodeGenDir = "generated/openapi" val generatedCodeDirectory = generatedCodeDirectory(layout, openApiCodeGenDir) -val whiteFoxGenerate = tasks.register("openapiGenerateClientApi") { +val whiteFoxGenerate = tasks.register("openapiGenerateClientApi") { + dependsOn(tasks.spotlessApply) generatorName.set("java") inputSpec.set("$rootDir/protocol/whitefox-protocol-api.yml") library.set("native") From a6de41f0e59649e865a7dd9d452641c59e47cbde Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Wed, 29 Nov 2023 22:17:39 +0100 Subject: [PATCH 09/21] Don't run spark-client:test early --- .github/workflows/compile.yaml | 6 +++--- client-spark/build.gradle.kts | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index 7a4e9512b..4ca4a24fa 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -32,7 +32,7 @@ jobs: if [ "$RUNNER_OS" == "Windows" ]; then export HADOOP_HOME="$(pwd)/.github/workflows/hadoop3-win-binaries" fi - ./gradlew build testNative --no-daemon + ./gradlew build testNative -x ':client-spark:test' --no-daemon ./gradlew server:app:printVersion --no-daemon -q shell: bash - name: Run integration test @@ -42,8 +42,8 @@ jobs: WHITEFOX_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.WHITEFOX_AWS_ACCESS_KEY_ID }} WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} run: | - java -jar server/build/quarkus-app/quarkus-run.jar & - ./gradlew :client-spark:test + java -jar server/build/quarkus-app/quarkus-run.jar & + ./gradlew :client-spark:test kill -9 %1 - name: Build container image if: runner.os == 'Linux' diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 6b47b93b0..009f4ac7f 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -32,14 +32,8 @@ dependencies { //JUNIT testImplementation("org.junit.jupiter:junit-jupiter:5.8.1") - } -// region code formatting -spotless { - java {} -} -// endregion tasks.getByName("test") { useJUnitPlatform() From 204f01023bcb3e574f3152e8755e3741be3f0588 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Wed, 29 Nov 2023 22:24:49 +0100 Subject: [PATCH 10/21] fix jar path --- .github/workflows/compile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index 4ca4a24fa..39f7895b5 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -42,7 +42,7 @@ jobs: WHITEFOX_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.WHITEFOX_AWS_ACCESS_KEY_ID }} WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} run: | - java -jar server/build/quarkus-app/quarkus-run.jar & + java -jar server/app/build/quarkus-app/quarkus-run.jar & ./gradlew :client-spark:test kill -9 %1 - name: Build container image From 8d6949e0f275cc84de2cc6b4eb80f09c20d9cc7c Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Wed, 29 Nov 2023 22:30:54 +0100 Subject: [PATCH 11/21] Run --no-daemon --- .github/workflows/compile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index 39f7895b5..db181e7fd 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -43,7 +43,7 @@ jobs: WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} run: | java -jar server/app/build/quarkus-app/quarkus-run.jar & - ./gradlew :client-spark:test + ./gradlew :client-spark:test --no-daemon kill -9 %1 - name: Build container image if: runner.os == 'Linux' From 702e4f4bfad0a3a1519a459014166af85015fb43 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 30 Nov 2023 09:56:37 +0100 Subject: [PATCH 12/21] tag ITDeltaSharingClient as an integrationTest --- .github/workflows/compile.yaml | 4 ++-- client-spark/build.gradle.kts | 1 - .../java/io/whitefox/api/client/ITDeltaSharingClient.java | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index db181e7fd..8a3d2eca2 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -32,7 +32,7 @@ jobs: if [ "$RUNNER_OS" == "Windows" ]; then export HADOOP_HOME="$(pwd)/.github/workflows/hadoop3-win-binaries" fi - ./gradlew build testNative -x ':client-spark:test' --no-daemon + ./gradlew build testNative --no-daemon ./gradlew server:app:printVersion --no-daemon -q shell: bash - name: Run integration test @@ -43,7 +43,7 @@ jobs: WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} run: | java -jar server/app/build/quarkus-app/quarkus-run.jar & - ./gradlew :client-spark:test --no-daemon + ./gradlew :client-spark:integrationTest --no-daemon kill -9 %1 - name: Build container image if: runner.os == 'Linux' diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 009f4ac7f..bd07c5188 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -1,4 +1,3 @@ -import com.diffplug.gradle.spotless.SpotlessApply import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index a4b35622b..f22cb5376 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -12,9 +12,11 @@ import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import scala.collection.GenMap; +@Tag("integration") public class ITDeltaSharingClient implements DatasetComparer { private final String tablePath = String.format( From 976d0e578d85452024ec048186746a9bbfa23220 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 30 Nov 2023 10:18:16 +0100 Subject: [PATCH 13/21] define ClientSparkTest tag --- .github/workflows/compile.yaml | 2 +- client-spark/build.gradle.kts | 8 ++++++++ .../java/io/whitefox/api/client/ITDeltaSharingClient.java | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yaml b/.github/workflows/compile.yaml index 8a3d2eca2..e2509748e 100644 --- a/.github/workflows/compile.yaml +++ b/.github/workflows/compile.yaml @@ -43,7 +43,7 @@ jobs: WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }} run: | java -jar server/app/build/quarkus-app/quarkus-run.jar & - ./gradlew :client-spark:integrationTest --no-daemon + ./gradlew :client-spark:clientSparkTest --no-daemon kill -9 %1 - name: Build container image if: runner.os == 'Linux' diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index bd07c5188..a76c6952e 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -43,6 +43,14 @@ tasks.withType { systemProperty ("java.util.logging.manager", "java.util.logging.LogManager") //TODO modularize the whitefox-conventions plugin } +tasks.register("clientSparkTest") { + useJUnitPlatform { + excludeTags.add("integration") + excludeTags.add("test") + includeTags.add("clientSparkTest") + } +} + val openApiCodeGenDir = "generated/openapi" val generatedCodeDirectory = generatedCodeDirectory(layout, openApiCodeGenDir) diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index f22cb5376..4124c87c0 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -16,7 +16,7 @@ import org.junit.jupiter.api.Test; import scala.collection.GenMap; -@Tag("integration") +@Tag("clientSparkTest") public class ITDeltaSharingClient implements DatasetComparer { private final String tablePath = String.format( From fa2b7218c4bff3f9597c1ea04d4c591f8edabc1e Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 30 Nov 2023 10:25:48 +0100 Subject: [PATCH 14/21] exclude ClientSparkTest tag from Test --- client-spark/build.gradle.kts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index a76c6952e..3b511987d 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -35,7 +35,9 @@ dependencies { tasks.getByName("test") { - useJUnitPlatform() + useJUnitPlatform { + excludeTags.add("clientSparkTest") + } } tasks.withType { @@ -45,7 +47,6 @@ tasks.withType { tasks.register("clientSparkTest") { useJUnitPlatform { - excludeTags.add("integration") excludeTags.add("test") includeTags.add("clientSparkTest") } From 534bb2ce5efaf9fdae9fccea5f249875cc5bf291 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 30 Nov 2023 10:26:40 +0100 Subject: [PATCH 15/21] remove exclusion --- client-spark/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index 3b511987d..c418c1df0 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -47,7 +47,6 @@ tasks.withType { tasks.register("clientSparkTest") { useJUnitPlatform { - excludeTags.add("test") includeTags.add("clientSparkTest") } } From 28798762b2ac79b0de2e1147f15f16a84e72ba19 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Thu, 30 Nov 2023 11:02:28 +0100 Subject: [PATCH 16/21] remove group and version --- client-spark/build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/client-spark/build.gradle.kts b/client-spark/build.gradle.kts index c418c1df0..7c5391c1a 100644 --- a/client-spark/build.gradle.kts +++ b/client-spark/build.gradle.kts @@ -6,9 +6,6 @@ plugins { id("whitefox.java-conventions") } -group = "io.whitefox" -version = "spark-connector" - repositories { mavenCentral() } From 2ea08734cff9451c42f2c7d690abb1c339ee04e8 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Fri, 1 Dec 2023 10:33:18 +0100 Subject: [PATCH 17/21] add starting version --- .../api/deltasharing/DeltaMappers.java | 15 +++--- .../io/whitefox/core/ReadTableRequest.java | 51 +++++++++++++++++++ .../core/services/DeltaSharedTable.java | 3 ++ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index f5847b100..4df983b82 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -5,11 +5,8 @@ import io.whitefox.core.*; import io.whitefox.core.Schema; import io.whitefox.core.Share; -import java.util.Arrays; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; -import org.jboss.resteasy.reactive.common.NotImplementedYet; public class DeltaMappers { @@ -26,8 +23,14 @@ public static io.whitefox.api.deltasharing.model.v1.generated.Schema schema2api( } public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { - if (request.getEndingVersion() != null) throw new NotImplementedYet(); - if (request.getVersion() != null && request.getTimestamp() == null) { + if (request.getVersion() != null && request.getVersion() < 0) { + throw new IllegalArgumentException("version cannot be negative."); + } else if (request.getStartingVersion() != null) { + return new ReadTableRequest.ReadTableStartingVersion( + request.getPredicateHints(), + Optional.ofNullable(request.getLimitHint()), + request.getStartingVersion()); + } else if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( request.getPredicateHints(), Optional.ofNullable(request.getLimitHint()), diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index aa6df33ab..070f7c938 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -58,6 +58,57 @@ public String toString() { } } + public static class ReadTableStartingVersion implements ReadTableRequest { + private final List predicateHints; + private final Optional limitHint; + + private final Long startingVersion; + + public ReadTableStartingVersion(List predicateHints, Optional limitHint, Long startingVersion) { + this.predicateHints = predicateHints; + this.limitHint = limitHint; + this.startingVersion = startingVersion; + } + + public List predicateHints() { + return predicateHints; + } + + public Optional limitHint() { + return limitHint; + } + + public Long startingVersion() { + return startingVersion; + } + + @Override + @SkipCoverageGenerated + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReadTableStartingVersion that = (ReadTableStartingVersion) o; + return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(limitHint, that.limitHint) + && Objects.equals(startingVersion, that.startingVersion); + } + + @Override + @SkipCoverageGenerated + public int hashCode() { + return Objects.hash(predicateHints, limitHint, startingVersion); + } + + @Override + @SkipCoverageGenerated + public String toString() { + return "ReadTableStartingVersion{" + "predicateHints=" + + predicateHints + ", limitHint=" + + limitHint + ", startingVersion=" + + startingVersion + '}'; + } + } + public static class ReadTableAsOfTimestamp implements ReadTableRequest { private final List predicateHints; private final Optional limitHint; diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 69fff6530..496b47471 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -88,6 +88,9 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) { snapshot = deltaLog.getSnapshotForVersionAsOf( ((ReadTableRequest.ReadTableVersion) readTableRequest).version()); + } else if (readTableRequest instanceof ReadTableRequest.ReadTableStartingVersion) { + snapshot = deltaLog.getSnapshotForVersionAsOf(((ReadTableRequest.ReadTableStartingVersion) readTableRequest).startingVersion()); + //TODO I think it's not correct } else { throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest); } From cb511f4a63a0e89e4fa03873c341576097dbeae8 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Fri, 1 Dec 2023 10:51:37 +0100 Subject: [PATCH 18/21] throw an exception when getting starting/ending version --- .../api/deltasharing/DeltaMappers.java | 9 ++-- .../io/whitefox/core/ReadTableRequest.java | 51 ------------------- .../core/services/DeltaSharedTable.java | 3 -- 3 files changed, 4 insertions(+), 59 deletions(-) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 4df983b82..7c5182b78 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -25,11 +25,6 @@ public static io.whitefox.api.deltasharing.model.v1.generated.Schema schema2api( public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { if (request.getVersion() != null && request.getVersion() < 0) { throw new IllegalArgumentException("version cannot be negative."); - } else if (request.getStartingVersion() != null) { - return new ReadTableRequest.ReadTableStartingVersion( - request.getPredicateHints(), - Optional.ofNullable(request.getLimitHint()), - request.getStartingVersion()); } else if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( request.getPredicateHints(), @@ -43,6 +38,10 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { } else if (request.getVersion() == null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableCurrentVersion( request.getPredicateHints(), Optional.ofNullable(request.getLimitHint())); + } else if (request.getStartingVersion() != null && request.getEndingVersion() != null) { + throw new IllegalArgumentException("The startingVersion and endingVersion are not supported"); + } else if (request.getStartingVersion() != null) { + throw new IllegalArgumentException("The startingVersion is not supported"); } else { throw new IllegalArgumentException("Cannot specify both version and timestamp"); } diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index 070f7c938..aa6df33ab 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -58,57 +58,6 @@ public String toString() { } } - public static class ReadTableStartingVersion implements ReadTableRequest { - private final List predicateHints; - private final Optional limitHint; - - private final Long startingVersion; - - public ReadTableStartingVersion(List predicateHints, Optional limitHint, Long startingVersion) { - this.predicateHints = predicateHints; - this.limitHint = limitHint; - this.startingVersion = startingVersion; - } - - public List predicateHints() { - return predicateHints; - } - - public Optional limitHint() { - return limitHint; - } - - public Long startingVersion() { - return startingVersion; - } - - @Override - @SkipCoverageGenerated - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ReadTableStartingVersion that = (ReadTableStartingVersion) o; - return Objects.equals(predicateHints, that.predicateHints) - && Objects.equals(limitHint, that.limitHint) - && Objects.equals(startingVersion, that.startingVersion); - } - - @Override - @SkipCoverageGenerated - public int hashCode() { - return Objects.hash(predicateHints, limitHint, startingVersion); - } - - @Override - @SkipCoverageGenerated - public String toString() { - return "ReadTableStartingVersion{" + "predicateHints=" - + predicateHints + ", limitHint=" - + limitHint + ", startingVersion=" - + startingVersion + '}'; - } - } - public static class ReadTableAsOfTimestamp implements ReadTableRequest { private final List predicateHints; private final Optional limitHint; diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 496b47471..69fff6530 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -88,9 +88,6 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) { snapshot = deltaLog.getSnapshotForVersionAsOf( ((ReadTableRequest.ReadTableVersion) readTableRequest).version()); - } else if (readTableRequest instanceof ReadTableRequest.ReadTableStartingVersion) { - snapshot = deltaLog.getSnapshotForVersionAsOf(((ReadTableRequest.ReadTableStartingVersion) readTableRequest).startingVersion()); - //TODO I think it's not correct } else { throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest); } From 3157de3e27d0b12a937fbc82b4991a9c95498118 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Fri, 1 Dec 2023 11:17:23 +0100 Subject: [PATCH 19/21] move up version checks --- .../io/whitefox/api/deltasharing/DeltaMappers.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 7c5182b78..0aaff813b 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -23,7 +23,11 @@ public static io.whitefox.api.deltasharing.model.v1.generated.Schema schema2api( } public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { - if (request.getVersion() != null && request.getVersion() < 0) { + if (request.getStartingVersion() != null && request.getEndingVersion() != null) { + throw new IllegalArgumentException("The startingVersion and endingVersion are not supported"); + } else if (request.getStartingVersion() != null) { + throw new IllegalArgumentException("The startingVersion is not supported"); + } else if (request.getVersion() != null && request.getVersion() < 0) { throw new IllegalArgumentException("version cannot be negative."); } else if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( @@ -38,10 +42,6 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { } else if (request.getVersion() == null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableCurrentVersion( request.getPredicateHints(), Optional.ofNullable(request.getLimitHint())); - } else if (request.getStartingVersion() != null && request.getEndingVersion() != null) { - throw new IllegalArgumentException("The startingVersion and endingVersion are not supported"); - } else if (request.getStartingVersion() != null) { - throw new IllegalArgumentException("The startingVersion is not supported"); } else { throw new IllegalArgumentException("Cannot specify both version and timestamp"); } From 79d23886d1333ab1ddd31e4a9364013451fa25c3 Mon Sep 17 00:00:00 2001 From: Marco Scalzo Date: Fri, 1 Dec 2023 11:29:51 +0100 Subject: [PATCH 20/21] check also endingVersion --- .../main/java/io/whitefox/api/deltasharing/DeltaMappers.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 0aaff813b..5efe4dd59 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -27,6 +27,8 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { throw new IllegalArgumentException("The startingVersion and endingVersion are not supported"); } else if (request.getStartingVersion() != null) { throw new IllegalArgumentException("The startingVersion is not supported"); + } else if (request.getEndingVersion() != null) { + throw new IllegalArgumentException("The startingVersion is not supported"); } else if (request.getVersion() != null && request.getVersion() < 0) { throw new IllegalArgumentException("version cannot be negative."); } else if (request.getVersion() != null && request.getTimestamp() == null) { From 8db51e38813f1b4184a51e90badb7c8a3f583490 Mon Sep 17 00:00:00 2001 From: Antonio Murgia Date: Fri, 1 Dec 2023 11:31:03 +0100 Subject: [PATCH 21/21] Update server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java --- .../main/java/io/whitefox/api/deltasharing/DeltaMappers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 5efe4dd59..6948065c7 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -28,7 +28,7 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { } else if (request.getStartingVersion() != null) { throw new IllegalArgumentException("The startingVersion is not supported"); } else if (request.getEndingVersion() != null) { - throw new IllegalArgumentException("The startingVersion is not supported"); + throw new IllegalArgumentException("The endingVersion is not supported"); } else if (request.getVersion() != null && request.getVersion() < 0) { throw new IllegalArgumentException("version cannot be negative."); } else if (request.getVersion() != null && request.getTimestamp() == null) {