Skip to content

Commit 8ca863e

Browse files
agilelab-tmnd1991agilelab-tmnd1991
committed
Update the protocol and manually generate TableQuery/Metadata response for parquet
still to do: do the same for Delta ones
1 parent d7df57d commit 8ca863e

27 files changed

+1092
-276
lines changed

‎protocol/delta-sharing-protocol-api.yml

Lines changed: 0 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -768,125 +768,6 @@ components:
768768
message:
769769
type: string
770770

771-
# This is not used for the spec but comes handy for autogeneration
772-
TableMetadataResponseObject:
773-
type: object
774-
properties:
775-
protocol:
776-
# it refers to ./delta-sharing-protocol.md#protocol
777-
$ref: '#/components/schemas/ParquetProtocolObject'
778-
metadata:
779-
# it refers to ./delta-sharing-protocol.md#metadata
780-
$ref: '#/components/schemas/ParquetMetadataObject'
781-
782-
# This is not used for the spec but comes handy for autogeneration
783-
TableQueryResponseObject:
784-
oneOf:
785-
- $ref: '#/components/schemas/ParquetTableQueryResponseObject'
786-
- $ref: '#/components/schemas/DeltaTableQueryResponseObject'
787-
ParquetTableQueryResponseObject:
788-
type: object
789-
properties:
790-
protocol:
791-
# it refers to ./delta-sharing-protocol.md#protocol
792-
$ref: '#/components/schemas/ParquetProtocolObject'
793-
metadata:
794-
# it refers to ./delta-sharing-protocol.md#metadata
795-
$ref: '#/components/schemas/ParquetMetadataObject'
796-
files:
797-
type: array
798-
items:
799-
# it refers to ./delta-sharing-protocol.md#file
800-
$ref: '#/components/schemas/ParquetFileObject'
801-
ParquetFileObject:
802-
type: object
803-
properties:
804-
file:
805-
type: object
806-
properties:
807-
url:
808-
type: string
809-
id:
810-
type: string
811-
partitionValues:
812-
type: object
813-
additionalProperties:
814-
type:
815-
string
816-
size:
817-
type: integer
818-
format: int64
819-
stats:
820-
type: string
821-
version:
822-
type: integer
823-
format: int64
824-
timestamp:
825-
type: integer
826-
format: int64
827-
expirationTimestamp:
828-
type: integer
829-
format: int64
830-
required:
831-
- url
832-
- id
833-
- partitionValues
834-
- size
835-
ParquetProtocolObject:
836-
type: object
837-
properties:
838-
protocol:
839-
type: object
840-
properties:
841-
minReaderVersion:
842-
type: integer
843-
format: int32
844-
ParquetFormatObject:
845-
type: object
846-
properties:
847-
provider:
848-
type: string
849-
required:
850-
- provider
851-
ParquetMetadataObject:
852-
type: object
853-
properties:
854-
metaData:
855-
type: object
856-
properties:
857-
id:
858-
type: string
859-
name:
860-
type: string
861-
description:
862-
type: string
863-
format:
864-
$ref: '#/components/schemas/ParquetFormatObject'
865-
schemaString:
866-
type: string
867-
partitionColumns:
868-
type: array
869-
items:
870-
type: string
871-
configuration:
872-
type: object
873-
additionalProperties:
874-
type:
875-
string
876-
version:
877-
type: integer
878-
format: int64
879-
size:
880-
type: integer
881-
format: int64
882-
numFiles:
883-
type: integer
884-
format: int64
885-
required:
886-
- id
887-
- format
888-
- schemaString
889-
- partitionColumns
890771
DeltaTableQueryResponseObject:
891772
type: object
892773
properties:

‎server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
package io.whitefox.api.deltasharing;
22

3+
import io.whitefox.api.deltasharing.model.v1.TableMetadataResponse;
4+
import io.whitefox.api.deltasharing.model.v1.TableQueryResponse;
35
import io.whitefox.api.deltasharing.model.v1.generated.*;
6+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
7+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
8+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
49
import io.whitefox.api.server.CommonMappers;
10+
import io.whitefox.api.server.WhitefoxMappers;
511
import io.whitefox.core.*;
612
import io.whitefox.core.Schema;
713
import io.whitefox.core.Share;
@@ -49,46 +55,46 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) {
4955
}
5056
}
5157

52-
public static TableQueryResponseObject readTableResult2api(ReadTableResult readTableResult) {
53-
return new TableQueryResponseObject()
54-
.metadata(metadata2Api(readTableResult.metadata()))
55-
.protocol(protocol2Api(readTableResult.protocol()))
56-
.files(readTableResult.files().stream()
57-
.map(DeltaMappers::file2Api)
58-
.collect(Collectors.toList()));
58+
public static TableQueryResponse readTableResult2api(ReadTableResult readTableResult) {
59+
return new TableQueryResponse(
60+
protocol2Api(readTableResult.protocol()),
61+
metadata2Api(readTableResult.metadata()),
62+
readTableResult.files().stream().map(DeltaMappers::file2Api).collect(Collectors.toList()));
5963
}
6064

61-
private static MetadataObject metadata2Api(Metadata metadata) {
62-
return new MetadataObject()
63-
.metaData(new MetadataObjectMetaData()
65+
private static ParquetMetadata metadata2Api(Metadata metadata) {
66+
return ParquetMetadata.builder()
67+
.metadata(ParquetMetadata.Metadata.builder()
6468
.id(metadata.id())
65-
.name(metadata.name().orElse(null))
66-
.description(metadata.description().orElse(null))
67-
.format(new FormatObject().provider(metadata.format().provider()))
69+
.name(metadata.name())
70+
.description(metadata.description())
71+
.format(WhitefoxMappers.format2api(metadata.format()))
6872
.schemaString(metadata.tableSchema().structType().toJson())
6973
.partitionColumns(metadata.partitionColumns())
70-
._configuration(metadata.configuration())
71-
.version(metadata.version().orElse(null))
72-
.numFiles(metadata.numFiles().orElse(null)));
74+
.configuration(Optional.of(metadata.configuration()))
75+
.version(metadata.version())
76+
.numFiles(metadata.numFiles())
77+
.build())
78+
.build();
7379
}
7480

75-
private static ProtocolObject protocol2Api(Protocol protocol) {
76-
return new ProtocolObject()
77-
.protocol(new ProtocolObjectProtocol()
78-
.minReaderVersion(protocol.minReaderVersion().orElse(1)));
81+
private static ParquetProtocol protocol2Api(Protocol protocol) {
82+
return ParquetProtocol.ofMinReaderVersion(protocol.minReaderVersion().orElse(1));
7983
}
8084

81-
private static FileObject file2Api(TableFile f) {
82-
return new FileObject()
83-
._file(new FileObjectFile()
85+
private static ParquetFile file2Api(TableFile f) {
86+
return ParquetFile.builder()
87+
.file(ParquetFile.File.builder()
8488
.id(f.id())
8589
.url(f.url())
8690
.partitionValues(f.partitionValues())
8791
.size(f.size())
88-
.stats(f.stats().orElse(null))
89-
.version(f.version().orElse(null))
90-
.timestamp(f.timestamp().orElse(null))
91-
.expirationTimestamp(f.expirationTimestamp()));
92+
.stats(f.stats())
93+
.version(f.version())
94+
.timestamp(f.timestamp())
95+
.expirationTimestamp(Optional.of(f.expirationTimestamp()))
96+
.build())
97+
.build();
9298
}
9399

94100
public static TableReferenceAndReadRequest api2TableReferenceAndReadRequest(
@@ -123,9 +129,9 @@ public static Map<String, String> toHeaderCapabilitiesMap(String headerCapabilit
123129
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
124130
}
125131

126-
public static TableMetadataResponseObject toTableResponseMetadata(Metadata m) {
127-
return new TableMetadataResponseObject()
128-
.protocol(new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)))
129-
.metadata(metadata2Api(m));
132+
public static TableMetadataResponse toTableResponseMetadata(Metadata m) {
133+
return new TableMetadataResponse(
134+
ParquetProtocol.ofMinReaderVersion(1), // smell
135+
metadata2Api(m));
130136
}
131137
}

‎server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/Format.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package io.whitefox.api.deltasharing.model.v1;
2+
3+
import com.fasterxml.jackson.annotation.JsonProperty;
4+
import lombok.Value;
5+
6+
@Value
7+
public class Format {
8+
@JsonProperty
9+
String provider = "parquet";
10+
}

‎server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableMetadataResponse.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package io.whitefox.api.deltasharing.model.v1;
2+
3+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
4+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
5+
import lombok.NonNull;
6+
import lombok.Value;
7+
8+
@Value
9+
public class TableMetadataResponse {
10+
@NonNull ParquetProtocol protocol;
11+
12+
@NonNull ParquetMetadata metadata;
13+
}

‎server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableQueryResponse.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package io.whitefox.api.deltasharing.model.v1;
2+
3+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
4+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
5+
import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
6+
import java.util.List;
7+
import lombok.NonNull;
8+
import lombok.Value;
9+
10+
@Value
11+
public class TableQueryResponse {
12+
@NonNull ParquetProtocol protocol;
13+
14+
@NonNull ParquetMetadata metadata;
15+
16+
@NonNull List<ParquetFile> files;
17+
}

‎server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/delta/DeltaProtocol.java

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package io.whitefox.api.deltasharing.model.v1.delta;
2+
3+
import com.fasterxml.jackson.annotation.JsonProperty;
4+
import lombok.Value;
5+
import lombok.experimental.SuperBuilder;
6+
import lombok.extern.jackson.Jacksonized;
7+
8+
@SuperBuilder
9+
@Jacksonized
10+
@Value
11+
public class DeltaProtocol {
12+
13+
@JsonProperty
14+
Protocol protocol;
15+
16+
@SuperBuilder
17+
@Jacksonized
18+
@Value
19+
private static class Protocol {
20+
@JsonProperty
21+
InternalDeltaProtocol deltaProtocol;
22+
}
23+
24+
@SuperBuilder
25+
@Jacksonized
26+
@Value
27+
private static class InternalDeltaProtocol {
28+
@JsonProperty
29+
int minReaderVersion;
30+
31+
@JsonProperty
32+
int minWriterVersion;
33+
}
34+
35+
public static DeltaProtocol of(int minReaderVersion, int maxReaderVersion) {
36+
return DeltaProtocol.builder()
37+
.protocol(Protocol.builder()
38+
.deltaProtocol(InternalDeltaProtocol.builder()
39+
.minReaderVersion(minReaderVersion)
40+
.minWriterVersion(maxReaderVersion)
41+
.build())
42+
.build())
43+
.build();
44+
}
45+
}

0 commit comments

Comments
 (0)