Skip to content

Commit fa40ab3

Browse files
duhizjameduhizjame
committed
Handle non-existing columns in predicate; formatting
1 parent d1c2bfa commit fa40ab3

23 files changed

+278
-315
lines changed

‎server/core/src/main/java/io/whitefox/core/ColumnRange.java

Lines changed: 23 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package io.whitefox.core;
22

33
import io.whitefox.core.types.*;
4-
54
import java.sql.Date;
65
import java.sql.Timestamp;
76

@@ -24,43 +23,36 @@ public ColumnRange(String onlyVal, DataType valueType) {
2423
this.valueType = valueType;
2524
}
2625

27-
private Boolean typedContains(String point){
26+
private Boolean typedContains(String point) {
2827
if (valueType instanceof IntegerType) {
2928
var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point));
3029
var c2 = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point));
3130
return (c1 <= 0 && c2 >= 0);
32-
}
33-
else if (valueType instanceof LongType){
31+
} else if (valueType instanceof LongType) {
3432
var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point));
3533
var c2 = Long.compare(Long.parseLong(maxVal), Long.parseLong(point));
3634
return (c1 <= 0 && c2 >= 0);
37-
}
38-
else if (valueType instanceof TimestampType){
35+
} else if (valueType instanceof TimestampType) {
3936
var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point));
4037
var c2 = Timestamp.valueOf(maxVal).before(Timestamp.valueOf(point));
4138
return c1 && c2;
42-
}
43-
else if (valueType instanceof FloatType){
39+
} else if (valueType instanceof FloatType) {
4440
var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point));
4541
var c2 = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point));
4642
return (c1 <= 0 && c2 >= 0);
47-
}
48-
else if (valueType instanceof DoubleType){
43+
} else if (valueType instanceof DoubleType) {
4944
var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point));
5045
var c2 = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point));
5146
return (c1 <= 0 && c2 >= 0);
52-
}
53-
else if (valueType instanceof DateType){
47+
} else if (valueType instanceof DateType) {
5448
var c1 = Date.valueOf(minVal).before(Date.valueOf(point));
5549
var c2 = Date.valueOf(maxVal).before(Date.valueOf(point));
5650
return c1 && c2;
57-
}
58-
else if (valueType instanceof BooleanType){
51+
} else if (valueType instanceof BooleanType) {
5952
var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point);
6053
var c2 = Boolean.parseBoolean(maxVal) == Boolean.parseBoolean(point);
6154
return c1 || c2;
62-
}
63-
else {
55+
} else {
6456
var c1 = minVal.compareTo(point);
6557
var c2 = maxVal.compareTo(point);
6658
return (c1 <= 0 && c2 >= 0);
@@ -72,65 +64,53 @@ public static void main(String[] args) {
7264
var point = "5";
7365
var maxVal = "8";
7466

75-
var cr = new ColumnRange(minVal,maxVal, IntegerType.INTEGER);
67+
var cr = new ColumnRange(minVal, maxVal, IntegerType.INTEGER);
7668
cr.typedLessThan(point);
7769
}
7870

79-
private Boolean typedLessThan(String point){
71+
private Boolean typedLessThan(String point) {
8072
if (valueType instanceof IntegerType) {
8173
var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point));
8274
return (c1 < 0);
83-
}
84-
else if (valueType instanceof LongType){
75+
} else if (valueType instanceof LongType) {
8576
var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point));
8677
return (c1 < 0);
87-
}
88-
else if (valueType instanceof TimestampType){
78+
} else if (valueType instanceof TimestampType) {
8979
return Timestamp.valueOf(minVal).before(Timestamp.valueOf(point));
90-
}
91-
else if (valueType instanceof FloatType){
80+
} else if (valueType instanceof FloatType) {
9281
var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point));
9382
return (c1 < 0);
94-
}
95-
else if (valueType instanceof DoubleType){
83+
} else if (valueType instanceof DoubleType) {
9684
var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point));
9785
return (c1 < 0);
98-
}
99-
else if (valueType instanceof DateType){
86+
} else if (valueType instanceof DateType) {
10087
return Date.valueOf(minVal).before(Date.valueOf(point));
10188

102-
}
103-
else {
89+
} else {
10490
var c = minVal.compareTo(point);
10591
return (c < 0);
10692
}
10793
}
10894

109-
private Boolean typedGreaterThan(String point){
95+
private Boolean typedGreaterThan(String point) {
11096
if (valueType instanceof IntegerType) {
11197
var c = Integer.compare(Integer.parseInt(point), Integer.parseInt(maxVal));
11298
return (c < 0);
113-
}
114-
else if (valueType instanceof LongType){
99+
} else if (valueType instanceof LongType) {
115100
var c = Long.compare(Long.parseLong(point), Long.parseLong(maxVal));
116101
return (c < 0);
117-
}
118-
else if (valueType instanceof TimestampType){
102+
} else if (valueType instanceof TimestampType) {
119103
return Timestamp.valueOf(point).before(Timestamp.valueOf(maxVal));
120-
}
121-
else if (valueType instanceof FloatType){
104+
} else if (valueType instanceof FloatType) {
122105
var c = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point));
123106
return (c < 0);
124-
}
125-
else if (valueType instanceof DoubleType){
107+
} else if (valueType instanceof DoubleType) {
126108
var c = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point));
127109
return (c < 0);
128-
}
129-
else if (valueType instanceof DateType){
110+
} else if (valueType instanceof DateType) {
130111
return Date.valueOf(point).before(Date.valueOf(maxVal));
131112

132-
}
133-
else {
113+
} else {
134114
var c = point.compareTo(maxVal);
135115
return (c < 0);
136116
}

‎server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,20 @@
66
import io.whitefox.core.types.DataType;
77
import io.whitefox.core.types.predicates.DataTypeDeserializer;
88

9-
109
public class DeltaObjectMapper {
1110

12-
private static final ObjectMapper objectMapper = newInstance();
11+
private static final ObjectMapper objectMapper = newInstance();
1312

14-
private static ObjectMapper newInstance() {
15-
ObjectMapper mapper = new ObjectMapper();
16-
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
17-
var customSerializersModule = new SimpleModule();
18-
customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer());
19-
mapper.registerModule(customSerializersModule);
20-
return mapper;
21-
}
13+
private static ObjectMapper newInstance() {
14+
ObjectMapper mapper = new ObjectMapper();
15+
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
16+
var customSerializersModule = new SimpleModule();
17+
customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer());
18+
mapper.registerModule(customSerializersModule);
19+
return mapper;
20+
}
2221

23-
public static ObjectMapper getInstance() {
24-
return objectMapper;
25-
}
22+
public static ObjectMapper getInstance() {
23+
return objectMapper;
24+
}
2625
}

‎server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,30 @@
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import io.delta.standalone.actions.AddFile;
66
import io.whitefox.core.types.DataType;
7-
import io.whitefox.core.types.DateType;
8-
import io.whitefox.core.types.IntegerType;
97
import io.whitefox.core.types.predicates.BaseOp;
108
import io.whitefox.core.types.predicates.EvalContext;
11-
import java.util.Map;
12-
9+
import io.whitefox.core.types.predicates.NonExistingColumnException;
1310
import io.whitefox.core.types.predicates.PredicateParsingException;
11+
import java.util.Optional;
1412
import org.apache.commons.lang3.tuple.Pair;
1513

16-
import static io.whitefox.core.types.DateType.DATE;
17-
1814
public class JsonPredicatesUtils {
1915

2016
private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance();
2117

2218
public static BaseOp parsePredicate(String predicate) throws PredicateParsingException {
2319
try {
2420
return objectMapper.readValue(predicate, BaseOp.class);
25-
}
26-
catch (JsonProcessingException e){
21+
} catch (JsonProcessingException e) {
2722
throw new PredicateParsingException(e);
2823
}
2924
}
3025

31-
public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) {
26+
public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType)
27+
throws NonExistingColumnException {
3228
var fileStats = ctx.getStatsValues();
33-
var values = fileStats.get(name);
29+
var values = Optional.ofNullable(fileStats.get(name))
30+
.orElseThrow(() -> new NonExistingColumnException(name));
3431
return new ColumnRange(values.getLeft(), values.getRight(), valueType);
3532
}
3633

‎server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,13 @@
88
import io.whitefox.core.TableSchema;
99
import io.whitefox.core.types.predicates.EvalContext;
1010
import io.whitefox.core.types.predicates.PredicateException;
11-
import org.apache.log4j.Logger;
12-
1311
import java.sql.Timestamp;
1412
import java.time.OffsetDateTime;
1513
import java.time.format.DateTimeFormatter;
1614
import java.util.List;
1715
import java.util.Optional;
1816
import java.util.stream.Collectors;
17+
import org.apache.log4j.Logger;
1918

2019
public class DeltaSharedTable {
2120

@@ -87,17 +86,18 @@ public Optional<Long> getTableVersion(Optional<String> startingTimestamp) {
8786
return getSnapshot(startingTimestamp).map(Snapshot::getVersion);
8887
}
8988

90-
91-
private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) {
92-
try {
93-
var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate);
94-
return parsedPredicate.evalExpectBoolean(ctx);
95-
} catch (PredicateException e) {
96-
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
97-
logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate: " + predicate);
98-
return true;
99-
}
89+
private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) {
90+
try {
91+
var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate);
92+
return parsedPredicate.evalExpectBoolean(ctx);
93+
} catch (PredicateException e) {
94+
logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
95+
logger.info("File: " + f.getPath()
96+
+ " will be used in processing due to failure in parsing or processing the predicate: "
97+
+ predicate);
98+
return true;
10099
}
100+
}
101101

102102
public boolean filterFilesBasedOnPredicates(List<String> predicates, AddFile f) {
103103
// if there are no predicates return all possible files
@@ -108,11 +108,13 @@ public boolean filterFilesBasedOnPredicates(List<String> predicates, AddFile f)
108108
var ctx = JsonPredicatesUtils.createEvalContext(f);
109109
return predicates.stream().allMatch(p -> evaluatePredicate(p, ctx, f));
110110
} catch (PredicateException e) {
111-
logger.debug("Caught exception: " + e.getMessage());
112-
logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate");
113-
return true;
114-
}
115-
};
111+
logger.debug("Caught exception: " + e.getMessage());
112+
logger.info("File: " + f.getPath()
113+
+ " will be used in processing due to failure in parsing or processing the predicate");
114+
return true;
115+
}
116+
}
117+
;
116118

117119
public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) {
118120
List<String> predicates;

‎server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import com.fasterxml.jackson.annotation.JsonSubTypes;
44
import com.fasterxml.jackson.annotation.JsonTypeInfo;
55
import io.whitefox.core.types.*;
6-
76
import java.util.List;
87
import java.util.Objects;
98

@@ -28,18 +27,19 @@ public interface BaseOp {
2827
default Boolean isSupportedType(DataType valueType, EvaluatorVersion version) {
2928
if (version == EvaluatorVersion.V2) {
3029
return (valueType instanceof BooleanType
31-
|| valueType instanceof IntegerType
32-
|| valueType instanceof StringType
33-
|| valueType instanceof DateType
34-
|| valueType instanceof LongType
35-
|| valueType instanceof TimestampType
36-
|| valueType instanceof FloatType
37-
|| valueType instanceof DoubleType);
38-
} else return (valueType instanceof BooleanType
39-
|| valueType instanceof IntegerType
40-
|| valueType instanceof StringType
41-
|| valueType instanceof DateType
42-
|| valueType instanceof LongType);
30+
|| valueType instanceof IntegerType
31+
|| valueType instanceof StringType
32+
|| valueType instanceof DateType
33+
|| valueType instanceof LongType
34+
|| valueType instanceof TimestampType
35+
|| valueType instanceof FloatType
36+
|| valueType instanceof DoubleType);
37+
} else
38+
return (valueType instanceof BooleanType
39+
|| valueType instanceof IntegerType
40+
|| valueType instanceof StringType
41+
|| valueType instanceof DateType
42+
|| valueType instanceof LongType);
4343
}
4444

4545
Object eval(EvalContext ctx) throws PredicateException;
@@ -65,14 +65,14 @@ default Boolean treeDepthExceeds(Integer depth) {
6565
}
6666

6767
// marker interface for operator arity used for easier exception handling
68-
interface Arity {};
68+
interface Arity {}
69+
;
6970

7071
// Represents a unary operation.
7172
interface UnaryOp extends Arity {
7273
// Validates number of children to be 1.
7374
default void validateChildren(List<BaseOp> children) throws PredicateException {
74-
if (children.size() != 1)
75-
throw new PredicateValidationException(children.size(), this, 1);
75+
if (children.size() != 1) throw new PredicateValidationException(children.size(), this, 1);
7676
try {
7777
children.get(0).validate();
7878
} catch (PredicateException e) {
@@ -84,8 +84,7 @@ default void validateChildren(List<BaseOp> children) throws PredicateException {
8484
interface BinaryOp extends Arity {
8585
// Validates number of children to be 2.
8686
default void validateChildren(List<BaseOp> children) throws PredicateException {
87-
if (children.size() != 2)
88-
throw new PredicateValidationException(children.size(), this, 2);
87+
if (children.size() != 2) throw new PredicateValidationException(children.size(), this, 2);
8988

9089
// otherwise cannot throw exception in method call of lambda
9190
for (BaseOp c : children) {

‎server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
11
package io.whitefox.core.types.predicates;
22

33
public enum BasePrimitiveTypeNames {
4-
DATE("date"),
5-
INT("int"),
6-
FLOAT("float"),
7-
DOUBLE("double"),
8-
TIMESTAMP("timestamp"),
9-
LONG("long"),
10-
STRING("string")
11-
;
4+
DATE("date"),
5+
INT("int"),
6+
FLOAT("float"),
7+
DOUBLE("double"),
8+
TIMESTAMP("timestamp"),
9+
LONG("long"),
10+
STRING("string");
1211

13-
final public String value;
12+
public final String value;
1413

15-
BasePrimitiveTypeNames(String value) {
16-
this.value = value;
17-
}
14+
BasePrimitiveTypeNames(String value) {
15+
this.value = value;
16+
}
1817
}

‎server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88
import io.whitefox.core.types.*;
99
import java.io.IOException;
1010

11-
12-
1311
public class DataTypeDeserializer extends StdDeserializer<DataType> {
1412

1513
// needed for jackson

0 commit comments

Comments
 (0)