Update openapi spec and written form

protocol/delta-sharing-protocol-api.yml

@@ -521,6 +521,12 @@ paths:
           description: 'If set to true, return the historical metadata if seen in the delta log. This is for the streaming client to check if the table schema is still read compatible.'
           schema:
             type: boolean
+        - in: header
+          name: delta-sharing-capabilities
+          required: false
+          description: 'Delta Sharing Capabilities'
+          schema:
+            type: string
       responses:
         '400':
           $ref: "#/components/responses/400"
@@ -575,11 +581,14 @@ components:
         not 	            Unary 	Represents a logical not check. This op should have once child.
         The supported value types:
           ValueType 	Description
-          "bool" 	Represents an Boolean type.
-          "int" 	Represents an Integer type.
-          "long" 	Represents a Long type.
-          "string" 	Represents a String type.
-          "date" 	Represents a Date type in "yyyy-mm-dd" format.
+          "bool" 	  Represents an Boolean type.
+          "int" 	  Represents an Integer type.
+          "long" 	  Represents a Long type.
+          "string" 	  Represents a String type.
+          "date" 	  Represents a Date type in "yyyy-mm-dd" format.
+          "float"     Represents a Float type.
+          "double"    Represents a Double type.
+          "timestamp" Represents a timestamp in ISO8601 format, in the UTC timezone.
 
     ListShareResponse:
       type: object
@@ -758,124 +767,6 @@ components:
           type: string
         message:
           type: string
-
-    # This is not used for the spec but comes handy for autogeneration
-    TableMetadataResponseObject:
-      type: object
-      properties:
-        protocol:
-          # it refers to ./delta-sharing-protocol.md#protocol
-          $ref: '#/components/schemas/ProtocolObject'
-        metadata:
-          # it refers to ./delta-sharing-protocol.md#metadata
-          $ref: '#/components/schemas/MetadataObject'
-
-    # This is not used for the spec but comes handy for autogeneration
-    TableQueryResponseObject:
-      type: object
-      properties:
-        protocol:
-          # it refers to ./delta-sharing-protocol.md#protocol
-          $ref: '#/components/schemas/ProtocolObject'
-        metadata:
-          # it refers to ./delta-sharing-protocol.md#metadata
-          $ref: '#/components/schemas/MetadataObject'
-        files:
-          type: array
-          items:
-            # it refers to ./delta-sharing-protocol.md#file
-            $ref: '#/components/schemas/FileObject'
-    FileObject:
-      type: object
-      properties:
-        file:
-          type: object
-          properties:
-            url:
-              type: string
-            id:
-              type: string
-            partitionValues:
-              type: object
-              additionalProperties:
-                type:
-                  string
-            size:
-              type: integer
-              format: int64
-            stats:
-              type: string
-            version:
-              type: integer
-              format: int64
-            timestamp:
-              type: integer
-              format: int64
-            expirationTimestamp:
-              type: integer
-              format: int64
-          required:
-            - url
-            - id
-            - partitionValues
-            - size
-    ProtocolObject:
-      type: object
-      properties:
-        protocol:
-          type: object
-          properties:
-            minReaderVersion:
-              type: integer
-              format: int32
-    FormatObject:
-      type: object
-      properties:
-        provider:
-          type: string
-      required:
-        - provider
-
-    MetadataObject:
-      type: object
-      properties:
-        metaData:
-          type: object
-          properties:
-            id:
-              type: string
-            name:
-              type: string
-            description:
-              type: string
-            format:
-              $ref: '#/components/schemas/FormatObject'
-            schemaString:
-              type: string
-            partitionColumns:
-              type: array
-              items:
-                type: string
-            configuration:
-              type: object
-              additionalProperties:
-                type:
-                  string
-            version:
-              type: integer
-              format: int64
-            size:
-              type: integer
-              format: int64
-            numFiles:
-              type: integer
-              format: int64
-          required:
-            - id
-            - format
-            - schemaString
-            - partitionColumns
-
   responses:
     "400":
       description: The request is malformed

server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

@@ -1,7 +1,13 @@
 package io.whitefox.api.deltasharing;
 
+import io.whitefox.api.deltasharing.model.v1.TableMetadataResponse;
+import io.whitefox.api.deltasharing.model.v1.TableQueryResponse;
 import io.whitefox.api.deltasharing.model.v1.generated.*;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
 import io.whitefox.api.server.CommonMappers;
+import io.whitefox.api.server.WhitefoxMappers;
 import io.whitefox.core.*;
 import io.whitefox.core.Schema;
 import io.whitefox.core.Share;
@@ -53,46 +59,46 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) {
     }
   }
 
-  public static TableQueryResponseObject readTableResult2api(ReadTableResult readTableResult) {
-    return new TableQueryResponseObject()
-        .metadata(metadata2Api(readTableResult.metadata()))
-        .protocol(protocol2Api(readTableResult.protocol()))
-        .files(readTableResult.files().stream()
-            .map(DeltaMappers::file2Api)
-            .collect(Collectors.toList()));
+  public static TableQueryResponse readTableResult2api(ReadTableResult readTableResult) {
+    return new TableQueryResponse(
+        protocol2Api(readTableResult.protocol()),
+        metadata2Api(readTableResult.metadata()),
+        readTableResult.files().stream().map(DeltaMappers::file2Api).collect(Collectors.toList()));
   }
 
-  private static MetadataObject metadata2Api(Metadata metadata) {
-    return new MetadataObject()
-        .metaData(new MetadataObjectMetaData()
+  private static ParquetMetadata metadata2Api(Metadata metadata) {
+    return ParquetMetadata.builder()
+        .metadata(ParquetMetadata.Metadata.builder()
             .id(metadata.id())
-            .name(metadata.name().orElse(null))
-            .description(metadata.description().orElse(null))
-            .format(new FormatObject().provider(metadata.format().provider()))
+            .name(metadata.name())
+            .description(metadata.description())
+            .format(WhitefoxMappers.format2api(metadata.format()))
             .schemaString(metadata.tableSchema().structType().toJson())
             .partitionColumns(metadata.partitionColumns())
-            ._configuration(metadata.configuration())
-            .version(metadata.version().orElse(null))
-            .numFiles(metadata.numFiles().orElse(null)));
+            .configuration(Optional.of(metadata.configuration()))
+            .version(metadata.version())
+            .numFiles(metadata.numFiles())
+            .build())
+        .build();
   }
 
-  private static ProtocolObject protocol2Api(Protocol protocol) {
-    return new ProtocolObject()
-        .protocol(new ProtocolObjectProtocol()
-            .minReaderVersion(protocol.minReaderVersion().orElse(1)));
+  private static ParquetProtocol protocol2Api(Protocol protocol) {
+    return ParquetProtocol.ofMinReaderVersion(protocol.minReaderVersion().orElse(1));
   }
 
-  private static FileObject file2Api(TableFile f) {
-    return new FileObject()
-        ._file(new FileObjectFile()
+  private static ParquetFile file2Api(TableFile f) {
+    return ParquetFile.builder()
+        .file(ParquetFile.File.builder()
             .id(f.id())
             .url(f.url())
             .partitionValues(f.partitionValues())
             .size(f.size())
-            .stats(f.stats().orElse(null))
-            .version(f.version().orElse(null))
-            .timestamp(f.timestamp().orElse(null))
-            .expirationTimestamp(f.expirationTimestamp()));
+            .stats(f.stats())
+            .version(f.version())
+            .timestamp(f.timestamp())
+            .expirationTimestamp(Optional.of(f.expirationTimestamp()))
+            .build())
+        .build();
   }
 
   public static TableReferenceAndReadRequest api2TableReferenceAndReadRequest(
@@ -127,9 +133,9 @@ public static Map<String, String> toHeaderCapabilitiesMap(String headerCapabilit
         .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 
-  public static TableMetadataResponseObject toTableResponseMetadata(Metadata m) {
-    return new TableMetadataResponseObject()
-        .protocol(new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)))
-        .metadata(metadata2Api(m));
+  public static TableMetadataResponse toTableResponseMetadata(Metadata m) {
+    return new TableMetadataResponse(
+        ParquetProtocol.ofMinReaderVersion(1), // smell
+        metadata2Api(m));
   }
 }

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/Format.java

@@ -0,0 +1,26 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode
+public class Format {
+  private static final String PARQUET = "parquet";
+
+  @JsonProperty
+  public String provider() {
+    return PARQUET;
+  }
+
+  public Format() {
+    this(PARQUET);
+  }
+
+  @JsonCreator
+  private Format(@JsonProperty("provider") String provider) {
+    if (!"parquet".equalsIgnoreCase(provider)) {
+      throw new IllegalArgumentException("Provider must be " + PARQUET);
+    }
+  }
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableMetadataResponse.java

@@ -0,0 +1,13 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
+import lombok.NonNull;
+import lombok.Value;
+
+@Value
+public class TableMetadataResponse {
+  @NonNull ParquetProtocol protocol;
+
+  @NonNull ParquetMetadata metadata;
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableQueryResponse.java

@@ -0,0 +1,17 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
+import java.util.List;
+import lombok.NonNull;
+import lombok.Value;
+
+@Value
+public class TableQueryResponse {
+  @NonNull ParquetProtocol protocol;
+
+  @NonNull ParquetMetadata metadata;
+
+  @NonNull List<ParquetFile> files;
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/delta/DeltaFile.java

@@ -0,0 +1,73 @@
+package io.whitefox.api.deltasharing.model.v1.delta;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Optional;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.experimental.SuperBuilder;
+import lombok.extern.jackson.Jacksonized;
+
+@Value
+@SuperBuilder
+@Jacksonized
+public class DeltaFile {
+
+  @JsonProperty
+  @NonNull File file;
+
+  @Value
+  @SuperBuilder
+  @Jacksonized
+  @JsonInclude(JsonInclude.Include.NON_ABSENT)
+  public static class File {
+
+    /**
+     * A unique string for the file in a table.
+     * The same file is guaranteed to have the same id across multiple requests.
+     * A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+     */
+    @JsonProperty
+    @NonNull String id;
+
+    /**
+     * A unique string for the deletion vector file in a table.
+     * The same deletion vector file is guaranteed to have the same id across multiple requests.
+     * A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+     */
+    @JsonProperty
+    @Builder.Default
+    Optional<String> deletionVectorFileId = Optional.empty();
+
+    /**
+     * The table version of the file, returned when querying a table data with a version or timestamp parameter.
+     */
+    @JsonProperty
+    @Builder.Default
+    Optional<Long> version = Optional.empty();
+
+    /**
+     * The unix timestamp corresponding to the table version of the file, in milliseconds,
+     * returned when querying a table data with a version or timestamp parameter.
+     */
+    @JsonProperty
+    @Builder.Default
+    Optional<Long> timestamp = Optional.empty();
+
+    /**
+     * The unix timestamp corresponding to the expiration of the url, in milliseconds,
+     * returned when the server supports the feature.
+     */
+    @JsonProperty
+    @Builder.Default
+    Optional<Long> expirationTimestamp = Optional.empty();
+
+    /**
+     * Need to be parsed by a delta library as a delta single action, the path field is replaced by pr-signed url.
+     */
+    @JsonProperty
+    @NonNull JsonNode deltaSingleAction;
+  }
+}