Skip to content

Commit 7bb1f7e

Browse files
agilelab-tmnd1991agilelab-tmnd1991
committed
wip
1 parent 7b77e5e commit 7bb1f7e

File tree

4 files changed

+133
-114
lines changed

4 files changed

+133
-114
lines changed

‎server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -58,37 +58,46 @@ public static TableQueryResponseObject readTableResult2api(ReadTableResult readT
5858
.collect(Collectors.toList()));
5959
}
6060

61-
private static MetadataObject metadata2Api(Metadata metadata) {
62-
return new MetadataObject()
63-
.metaData(new MetadataObjectMetaData()
64-
.id(metadata.id())
65-
.name(metadata.name().orElse(null))
66-
.description(metadata.description().orElse(null))
67-
.format(new FormatObject().provider(metadata.format().provider()))
68-
.schemaString(metadata.tableSchema().structType().toJson())
69-
.partitionColumns(metadata.partitionColumns())
70-
._configuration(metadata.configuration())
71-
.version(metadata.version().orElse(null))
72-
.numFiles(metadata.numFiles().orElse(null)));
61+
private static ParquetMetadataObject metadata2Api(Metadata metadata) {
62+
return new ParquetMetadataObject()
63+
.metaData(new ParquetMetadataObjectMetaData()
64+
.numFiles(metadata.numFiles().orElse(null))
65+
.version(metadata.version().orElse(null))
66+
.size(metadata.size().orElse(null))
67+
.id(metadata.id())
68+
.name(metadata.name().orElse(null))
69+
.description(metadata.description().orElse(null))
70+
.format(new ParquetFormatObject().provider(metadata.format().provider()))
71+
.schemaString(metadata.tableSchema().structType().toJson())
72+
.partitionColumns(metadata.partitionColumns())
73+
._configuration(metadata.configuration()));
7374
}
7475

75-
private static ProtocolObject protocol2Api(Protocol protocol) {
76-
return new ProtocolObject()
77-
.protocol(new ProtocolObjectProtocol()
78-
.minReaderVersion(protocol.minReaderVersion().orElse(1)));
76+
private static DeltaProtocolObject protocol2Api(Protocol protocol) {
77+
return new DeltaProtocolObject()
78+
.protocol(new DeltaProtocolObjectProtocol()
79+
.deltaProtocol(new DeltaProtocolObjectProtocolDeltaProtocol()
80+
.minReaderVersion(protocol.minReaderVersion().orElse(1))
81+
.minWriterVersion(protocol.minWriterVersion().orElse(1))));
7982
}
8083

81-
private static FileObject file2Api(TableFile f) {
82-
return new FileObject()
83-
._file(new FileObjectFile()
84+
private static DeltaFileObject file2Api(TableFile f) {
85+
return new DeltaFileObject()
8486
.id(f.id())
85-
.url(f.url())
86-
.partitionValues(f.partitionValues())
87-
.size(f.size())
88-
.stats(f.stats().orElse(null))
8987
.version(f.version().orElse(null))
88+
.deletionVectorFileId(null) // TODO
9089
.timestamp(f.timestamp().orElse(null))
91-
.expirationTimestamp(f.expirationTimestamp()));
90+
.expirationTimestamp(f.expirationTimestamp())
91+
.deltaSingleAction(new DeltaSingleAction()
92+
._file(new DeltaAddFileAction()
93+
.id(f.id())
94+
.url(f.url())
95+
.partitionValues(f.partitionValues())
96+
.size(f.size())
97+
.stats(f.stats().orElse(null))
98+
.version(f.version().orElse(null))
99+
.timestamp(f.timestamp().orElse(null))
100+
.expirationTimestamp(f.expirationTimestamp())));
92101
}
93102

94103
public static TableReferenceAndReadRequest api2TableReferenceAndReadRequest(
@@ -125,7 +134,7 @@ public static Map<String, String> toHeaderCapabilitiesMap(String headerCapabilit
125134

126135
public static TableMetadataResponseObject toTableResponseMetadata(Metadata m) {
127136
return new TableMetadataResponseObject()
128-
.protocol(new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)))
137+
.protocol(new ParquetProtocolObject().protocol(new ParquetProtocolObjectProtocol().minReaderVersion(1)))
129138
.metadata(metadata2Api(m));
130139
}
131140
}

‎server/core/src/main/java/io/whitefox/core/Protocol.java

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,19 @@
66

77
public class Protocol {
88
private final Optional<Integer> minReaderVersion;
9+
private final Optional<Integer> minWriterVersion;
910

10-
public Protocol(Optional<Integer> minReaderVersion) {
11+
public Protocol(Optional<Integer> minReaderVersion, Optional<Integer> minWriterVersion) {
1112
this.minReaderVersion = minReaderVersion;
13+
this.minWriterVersion = minWriterVersion;
1214
}
1315

1416
public Optional<Integer> minReaderVersion() {
1517
return minReaderVersion;
1618
}
1719

18-
@Override
19-
@SkipCoverageGenerated
20-
public String toString() {
21-
return "Protocol{" + "minReaderVersion=" + minReaderVersion + '}';
20+
public Optional<Integer> minWriterVersion() {
21+
return minWriterVersion;
2222
}
2323

2424
@Override
@@ -27,12 +27,21 @@ public boolean equals(Object o) {
2727
if (this == o) return true;
2828
if (o == null || getClass() != o.getClass()) return false;
2929
Protocol protocol = (Protocol) o;
30-
return Objects.equals(minReaderVersion, protocol.minReaderVersion);
30+
return Objects.equals(minReaderVersion, protocol.minReaderVersion)
31+
&& Objects.equals(minWriterVersion, protocol.minWriterVersion);
3132
}
3233

3334
@Override
3435
@SkipCoverageGenerated
3536
public int hashCode() {
36-
return Objects.hash(minReaderVersion);
37+
return Objects.hash(minReaderVersion, minWriterVersion);
38+
}
39+
40+
@Override
41+
@SkipCoverageGenerated
42+
public String toString() {
43+
return "Protocol{" + "minReaderVersion="
44+
+ minReaderVersion + ", minWriterVersion="
45+
+ minWriterVersion + '}';
3746
}
3847
}

‎server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) {
9292
throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest);
9393
}
9494
return new ReadTableResultToBeSigned(
95-
new Protocol(Optional.of(1)),
95+
new Protocol(Optional.of(1), Optional.of(1)),
9696
metadataFromSnapshot(snapshot),
9797
snapshot.getAllFiles().stream()
9898
.map(f -> new TableFileToBeSigned(

‎server/core/src/main/java/io/whitefox/core/services/DeltaSharingCapabilities.java

Lines changed: 82 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -6,99 +6,100 @@
66
import java.util.stream.Collectors;
77

88
public class DeltaSharingCapabilities {
9-
private final Map<String, String> values;
9+
private final Map<String, String> values;
1010

11-
public DeltaSharingCapabilities(Map<String, String> values) {
12-
this.values = Map.copyOf(values);
13-
}
14-
15-
public Map<String, String> values() {
16-
return values;
17-
}
11+
public DeltaSharingCapabilities(Map<String, String> values) {
12+
this.values = Map.copyOf(values);
13+
}
1814

19-
public DeltaSharingResponseFormat getResponseFormat() {
20-
var value = values.get(DELTA_SHARING_RESPONSE_FORMAT);
21-
if (value == null) {
22-
return DeltaSharingResponseFormat.PARQUET;
23-
} else {
24-
return DeltaSharingResponseFormat.valueOf(value.toUpperCase());
25-
}
26-
}
15+
public Map<String, String> values() {
16+
return values;
17+
}
2718

28-
public Set<DeltaSharingReaderFeatures> getReaderFeatures() {
29-
var value = values.get(DELTA_SHARING_READER_FEATURES);
30-
if (value == null) {
31-
return Set.of();
32-
} else {
33-
return Arrays.stream(value.split(","))
34-
.map(String::trim)
35-
.map(DeltaSharingReaderFeatures::fromString)
36-
.collect(Collectors.toSet());
37-
}
19+
public DeltaSharingResponseFormat getResponseFormat() {
20+
var value = values.get(DELTA_SHARING_RESPONSE_FORMAT);
21+
if (value == null) {
22+
return DeltaSharingResponseFormat.PARQUET;
23+
} else {
24+
return DeltaSharingResponseFormat.valueOf(value.toUpperCase());
3825
}
26+
}
3927

40-
public final static String DELTA_SHARING_RESPONSE_FORMAT = "responseformat";
41-
public final static String DELTA_SHARING_READER_FEATURES = "readerfeatures";
42-
43-
public enum DeltaSharingResponseFormat {
44-
DELTA, PARQUET
28+
public Set<DeltaSharingReaderFeatures> getReaderFeatures() {
29+
var value = values.get(DELTA_SHARING_READER_FEATURES);
30+
if (value == null) {
31+
return Set.of();
32+
} else {
33+
return Arrays.stream(value.split(","))
34+
.map(String::trim)
35+
.map(DeltaSharingReaderFeatures::fromString)
36+
.collect(Collectors.toSet());
4537
}
38+
}
4639

47-
public enum DeltaSharingReaderFeatures {
48-
DELETION_VECTORS("deletionvectors"),
49-
COLUMN_MAPPING("columnmapping"),
50-
TIMESTAMP_NTZ("timestampntz"),
51-
DOMAIN_METADATA("domainmetadata"),
52-
V2CHECKPOINT("v2checkpoint"),
53-
CHECK_CONSTRAINTS("checkconstraints"),
54-
GENERATED_COLUMNS("generatedcolumns"),
55-
ALLOW_COLUMN_DEFAULTS("allowcolumndefaults"),
56-
IDENTITY_COLUMNS("identitycolumns");
40+
public static final String DELTA_SHARING_RESPONSE_FORMAT = "responseformat";
41+
public static final String DELTA_SHARING_READER_FEATURES = "readerfeatures";
5742

58-
DeltaSharingReaderFeatures(String stringRepresentation) {
59-
this.stringRepresentation = stringRepresentation;
60-
}
43+
public enum DeltaSharingResponseFormat {
44+
DELTA,
45+
PARQUET
46+
}
6147

62-
private final String stringRepresentation;
48+
public enum DeltaSharingReaderFeatures {
49+
DELETION_VECTORS("deletionvectors"),
50+
COLUMN_MAPPING("columnmapping"),
51+
TIMESTAMP_NTZ("timestampntz"),
52+
DOMAIN_METADATA("domainmetadata"),
53+
V2CHECKPOINT("v2checkpoint"),
54+
CHECK_CONSTRAINTS("checkconstraints"),
55+
GENERATED_COLUMNS("generatedcolumns"),
56+
ALLOW_COLUMN_DEFAULTS("allowcolumndefaults"),
57+
IDENTITY_COLUMNS("identitycolumns");
6358

64-
public String stringRepresentation() {
65-
return stringRepresentation;
66-
}
59+
DeltaSharingReaderFeatures(String stringRepresentation) {
60+
this.stringRepresentation = stringRepresentation;
61+
}
6762

68-
public static DeltaSharingReaderFeatures fromString(String s) {
69-
switch (s) {
70-
case DELTA_SHARING_READER_FEATURE_DELETION_VECTOR:
71-
return DeltaSharingReaderFeatures.DELETION_VECTORS;
72-
case DELTA_SHARING_READER_FEATURE_COLUMN_MAPPING:
73-
return DeltaSharingReaderFeatures.COLUMN_MAPPING;
74-
case DELTA_SHARING_READER_FEATURE_TIMESTAMP_NTZ:
75-
return DeltaSharingReaderFeatures.TIMESTAMP_NTZ;
76-
case DELTA_SHARING_READER_FEATURE_DOMAIN_METADATA:
77-
return DeltaSharingReaderFeatures.DOMAIN_METADATA;
78-
case DELTA_SHARING_READER_FEATURE_V2CHECKPOINT:
79-
return DeltaSharingReaderFeatures.V2CHECKPOINT;
80-
case DELTA_SHARING_READER_FEATURE_CHECK_CONSTRAINTS:
81-
return DeltaSharingReaderFeatures.CHECK_CONSTRAINTS;
82-
case DELTA_SHARING_READER_FEATURE_GENERATED_COLUMNS:
83-
return DeltaSharingReaderFeatures.GENERATED_COLUMNS;
84-
case DELTA_SHARING_READER_FEATURE_ALLOW_COLUMN_DEFAULTS:
85-
return DeltaSharingReaderFeatures.ALLOW_COLUMN_DEFAULTS;
86-
case DELTA_SHARING_READER_FEATURE_IDENTITY_COLUMNS:
87-
return DeltaSharingReaderFeatures.IDENTITY_COLUMNS;
88-
default:
89-
throw new IllegalArgumentException("Unknown reader feature: " + s);
90-
}
91-
}
63+
private final String stringRepresentation;
9264

93-
private final static String DELTA_SHARING_READER_FEATURE_DELETION_VECTOR = "deletionvectors";
94-
private final static String DELTA_SHARING_READER_FEATURE_COLUMN_MAPPING = "columnmapping";
95-
private final static String DELTA_SHARING_READER_FEATURE_TIMESTAMP_NTZ = "timestampntz";
96-
private final static String DELTA_SHARING_READER_FEATURE_DOMAIN_METADATA = "domainmetadata";
97-
private final static String DELTA_SHARING_READER_FEATURE_V2CHECKPOINT = "v2checkpoint";
98-
private final static String DELTA_SHARING_READER_FEATURE_CHECK_CONSTRAINTS = "checkconstraints";
99-
private final static String DELTA_SHARING_READER_FEATURE_GENERATED_COLUMNS = "generatedcolumns";
100-
private final static String DELTA_SHARING_READER_FEATURE_ALLOW_COLUMN_DEFAULTS = "allowcolumndefaults";
101-
private final static String DELTA_SHARING_READER_FEATURE_IDENTITY_COLUMNS = "identitycolumns";
65+
public String stringRepresentation() {
66+
return stringRepresentation;
67+
}
10268

69+
public static DeltaSharingReaderFeatures fromString(String s) {
70+
switch (s) {
71+
case DELTA_SHARING_READER_FEATURE_DELETION_VECTOR:
72+
return DeltaSharingReaderFeatures.DELETION_VECTORS;
73+
case DELTA_SHARING_READER_FEATURE_COLUMN_MAPPING:
74+
return DeltaSharingReaderFeatures.COLUMN_MAPPING;
75+
case DELTA_SHARING_READER_FEATURE_TIMESTAMP_NTZ:
76+
return DeltaSharingReaderFeatures.TIMESTAMP_NTZ;
77+
case DELTA_SHARING_READER_FEATURE_DOMAIN_METADATA:
78+
return DeltaSharingReaderFeatures.DOMAIN_METADATA;
79+
case DELTA_SHARING_READER_FEATURE_V2CHECKPOINT:
80+
return DeltaSharingReaderFeatures.V2CHECKPOINT;
81+
case DELTA_SHARING_READER_FEATURE_CHECK_CONSTRAINTS:
82+
return DeltaSharingReaderFeatures.CHECK_CONSTRAINTS;
83+
case DELTA_SHARING_READER_FEATURE_GENERATED_COLUMNS:
84+
return DeltaSharingReaderFeatures.GENERATED_COLUMNS;
85+
case DELTA_SHARING_READER_FEATURE_ALLOW_COLUMN_DEFAULTS:
86+
return DeltaSharingReaderFeatures.ALLOW_COLUMN_DEFAULTS;
87+
case DELTA_SHARING_READER_FEATURE_IDENTITY_COLUMNS:
88+
return DeltaSharingReaderFeatures.IDENTITY_COLUMNS;
89+
default:
90+
throw new IllegalArgumentException("Unknown reader feature: " + s);
91+
}
10392
}
93+
94+
private static final String DELTA_SHARING_READER_FEATURE_DELETION_VECTOR = "deletionvectors";
95+
private static final String DELTA_SHARING_READER_FEATURE_COLUMN_MAPPING = "columnmapping";
96+
private static final String DELTA_SHARING_READER_FEATURE_TIMESTAMP_NTZ = "timestampntz";
97+
private static final String DELTA_SHARING_READER_FEATURE_DOMAIN_METADATA = "domainmetadata";
98+
private static final String DELTA_SHARING_READER_FEATURE_V2CHECKPOINT = "v2checkpoint";
99+
private static final String DELTA_SHARING_READER_FEATURE_CHECK_CONSTRAINTS = "checkconstraints";
100+
private static final String DELTA_SHARING_READER_FEATURE_GENERATED_COLUMNS = "generatedcolumns";
101+
private static final String DELTA_SHARING_READER_FEATURE_ALLOW_COLUMN_DEFAULTS =
102+
"allowcolumndefaults";
103+
private static final String DELTA_SHARING_READER_FEATURE_IDENTITY_COLUMNS = "identitycolumns";
104+
}
104105
}

0 commit comments

Comments
 (0)