wip- sql predicates without error handling

Author: duhizjame

server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

@@ -31,7 +31,7 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) {
     if (request.getVersion() != null && request.getTimestamp() == null) {
       return new ReadTableRequest.ReadTableVersion(
           request.getPredicateHints(),
-          Optional.ofNullable(request.getLimitHint()),
+              jsonPredicateHints, Optional.ofNullable(request.getLimitHint()),
           request.getVersion());
     } else if (request.getVersion() == null && request.getTimestamp() != null) {
       return new ReadTableRequest.ReadTableAsOfTimestamp(

server/core/src/main/java/io/whitefox/core/ColumnRange.java

@@ -34,7 +34,7 @@ private Boolean typedContains(String point) {
       return (c1 <= 0 && c2 >= 0);
     } else if (valueType instanceof TimestampType) {
       var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point));
-      var c2 = Timestamp.valueOf(maxVal).before(Timestamp.valueOf(point));
+      var c2 = Timestamp.valueOf(maxVal).after(Timestamp.valueOf(point));
       return c1 && c2;
     } else if (valueType instanceof FloatType) {
       var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point));
@@ -46,7 +46,7 @@ private Boolean typedContains(String point) {
       return (c1 <= 0 && c2 >= 0);
     } else if (valueType instanceof DateType) {
       var c1 = Date.valueOf(minVal).before(Date.valueOf(point));
-      var c2 = Date.valueOf(maxVal).before(Date.valueOf(point));
+      var c2 = Date.valueOf(maxVal).after(Date.valueOf(point));
       return c1 && c2;
     } else if (valueType instanceof BooleanType) {
       var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point);
@@ -59,15 +59,6 @@ private Boolean typedContains(String point) {
     }
   }
 
-  public static void main(String[] args) {
-    var minVal = "4";
-    var point = "5";
-    var maxVal = "8";
-
-    var cr = new ColumnRange(minVal, maxVal, IntegerType.INTEGER);
-    cr.typedLessThan(point);
-  }
-
   private Boolean typedLessThan(String point) {
     if (valueType instanceof IntegerType) {
       var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point));
@@ -85,7 +76,6 @@ private Boolean typedLessThan(String point) {
       return (c1 < 0);
     } else if (valueType instanceof DateType) {
       return Date.valueOf(minVal).before(Date.valueOf(point));
-
     } else {
       var c = minVal.compareTo(point);
       return (c < 0);

server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java renamed to server/core/src/main/java/io/whitefox/core/PredicateUtils.java

@@ -2,24 +2,23 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.delta.standalone.Snapshot;
 import io.delta.standalone.actions.AddFile;
-import io.delta.standalone.expressions.IsNull;
 import io.whitefox.core.types.DataType;
-import io.whitefox.core.types.DateType;
 import io.whitefox.core.types.predicates.*;
 
 import java.util.List;
-import java.util.Objects;
 import java.util.Optional;
 
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.StringValue;
 import net.sf.jsqlparser.expression.operators.relational.IsNullExpression;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
-import net.sf.jsqlparser.statement.select.PlainSelect;
 import org.apache.commons.lang3.tuple.Pair;
 
-public class JsonPredicatesUtils {
+public class PredicateUtils {
 
   private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance();
 
@@ -31,29 +30,45 @@ public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsin
     }
   }
 
-  public static BaseOp parseSqlPredicate(String predicate, DataType dataType) throws JSQLParserException, PredicateException {
-    var expression = CCJSqlParserUtil.parseCondExpression(predicate);
-    if (expression instanceof IsNullExpression){
-      var isNullExpression = (IsNullExpression) expression;
-      String column = isNullExpression.getLeftExpression().getASTNode().jjtGetFirstToken().toString();
-      var colOp = new ColumnOp(column, dataType);
-      var children = List.of((LeafOp) colOp);
-      var operator = "isnull";
-      return NonLeafOp.createPartitionFilter(children, operator);
+  public static BaseOp parseSqlPredicate(String predicate, EvalContext ctx, Metadata metadata) throws PredicateException {
+    try {
+      var expression = CCJSqlParserUtil.parseCondExpression(predicate);
+      if (expression instanceof IsNullExpression){
+        var isNullExpression = (IsNullExpression) expression;
+        String column = isNullExpression.getLeftExpression().getASTNode().jjtGetFirstToken().toString();
+        var dataType = metadata.tableSchema().structType().get(column).getDataType();
+        var colOp = new ColumnOp(column, dataType);
+        var children = List.of((LeafOp) colOp);
+        var operator = "isnull";
+        return NonLeafOp.createPartitionFilter(children, operator);
+      }
+      else if (expression instanceof BinaryExpression) {
+        BinaryExpression binaryExpression = (BinaryExpression) expression;
+        String column = binaryExpression.getLeftExpression().toString();
+        String operator = binaryExpression.getStringExpression();
+        Expression value = binaryExpression.getRightExpression();
+        if (value instanceof StringValue) {
+          StringValue stringValue = (StringValue) value;
+        var dataType = metadata.tableSchema().structType().get(column).getDataType();
+        var colOp = new ColumnOp(column, dataType);
+        var litOp = new LiteralOp(stringValue.getValue(), dataType);
+        var children = List.of(colOp, litOp);
+        return NonLeafOp.createPartitionFilter(children, operator);
+        }
+        else {
+          var dataType = metadata.tableSchema().structType().get(column).getDataType();
+          var colOp = new ColumnOp(column, dataType);
+          var litOp = new LiteralOp(value.toString(), dataType);
+          var children = List.of(colOp, litOp);
+          return NonLeafOp.createPartitionFilter(children, operator);
+        }
+      }
+      else
+        throw new ExpressionNotSupportedException(predicate);
     }
-    else if (expression instanceof BinaryExpression) {
-      BinaryExpression binaryExpression = (BinaryExpression) expression;
-      String column = binaryExpression.getLeftExpression().toString();
-      String operator = binaryExpression.getStringExpression();
-      String value = binaryExpression.getRightExpression().toString();
-      var colOp = new ColumnOp(column, dataType);
-      var litOp = new LiteralOp(value, dataType);
-      var children = List.of(colOp, litOp);
-      return NonLeafOp.createPartitionFilter(children, operator);
+    catch (JSQLParserException e) {
+      throw new PredicateParsingException(e);
     }
-    // TODO: PARSING FAIL on sql;
-    else
-      throw new PredicateException();
   }
 
 

server/core/src/main/java/io/whitefox/core/ReadTableRequest.java

@@ -9,17 +9,23 @@ public interface ReadTableRequest {
 
   public static class ReadTableVersion implements ReadTableRequest {
     private final List<String> predicateHints;
+    private final List<String> jsonPredicateHints;
     private final Optional<Integer> limitHint;
 
     private final Long version;
 
     public ReadTableVersion(
-        List<String> predicateHints, Optional<Integer> limitHint, Long version) {
+            List<String> predicateHints, List<String> jsonPredicateHints, Optional<Integer> limitHint, Long version) {
       this.predicateHints = predicateHints;
+      this.jsonPredicateHints = jsonPredicateHints;
       this.limitHint = limitHint;
       this.version = version;
     }
 
+    public List<String> jsonPredicateHints() {
+      return jsonPredicateHints;
+    }
+
     public List<String> predicateHints() {
       return predicateHints;
     }
@@ -39,60 +45,70 @@ public boolean equals(Object o) {
       if (o == null || getClass() != o.getClass()) return false;
       ReadTableVersion that = (ReadTableVersion) o;
       return Objects.equals(predicateHints, that.predicateHints)
+          && Objects.equals(jsonPredicateHints, that.jsonPredicateHints)
           && Objects.equals(limitHint, that.limitHint)
           && Objects.equals(version, that.version);
     }
 
     @Override
     @SkipCoverageGenerated
     public int hashCode() {
-      return Objects.hash(predicateHints, limitHint, version);
+      return Objects.hash(predicateHints, jsonPredicateHints, limitHint, version);
     }
 
     @Override
     @SkipCoverageGenerated
     public String toString() {
       return "ReadTableVersion{" + "predicateHints="
-          + predicateHints + ", limitHint="
+          + predicateHints + "jsonPredicateHints="
+          + jsonPredicateHints + ", limitHint="
           + limitHint + ", version="
           + version + '}';
     }
   }
 
   public static class ReadTableAsOfTimestamp implements ReadTableRequest {
     private final List<String> predicateHints;
+    private final List<String> jsonPredicateHints;
     private final Optional<Integer> limitHint;
     private final Long timestamp;
 
     public ReadTableAsOfTimestamp(
-        List<String> predicateHints, Optional<Integer> limitHint, Long timestamp) {
+        List<String> predicateHints, List<String> jsonPredicateHints, Optional<Integer> limitHint, Long timestamp) {
       this.predicateHints = predicateHints;
+      this.jsonPredicateHints = jsonPredicateHints;
       this.limitHint = limitHint;
       this.timestamp = timestamp;
     }
 
+    public List<String> jsonPredicateHints() {
+      return jsonPredicateHints;
+    }
+
     @Override
     @SkipCoverageGenerated
     public boolean equals(Object o) {
       if (this == o) return true;
       if (o == null || getClass() != o.getClass()) return false;
       ReadTableAsOfTimestamp that = (ReadTableAsOfTimestamp) o;
       return Objects.equals(predicateHints, that.predicateHints)
+          && Objects.equals(jsonPredicateHints, that.jsonPredicateHints)
           && Objects.equals(limitHint, that.limitHint)
           && Objects.equals(timestamp, that.timestamp);
     }
 
     @Override
     @SkipCoverageGenerated
     public int hashCode() {
-      return Objects.hash(predicateHints, limitHint, timestamp);
+      return Objects.hash(jsonPredicateHints, predicateHints, limitHint, timestamp);
     }
 
     @Override
     @SkipCoverageGenerated
     public String toString() {
       return "ReadTableAsOfTimestamp{" + "predicateHints="
-          + predicateHints + ", limitHint="
+          + predicateHints + "jsonPredicateHints="
+          + jsonPredicateHints + ", limitHint="
           + limitHint + ", timestamp="
           + timestamp + '}';
     }
@@ -112,17 +128,23 @@ public Long timestamp() {
 
   public static class ReadTableCurrentVersion implements ReadTableRequest {
     private final List<String> predicateHints;
+    private final List<String> jsonPredicateHints;
     private final Optional<Integer> limitHint;
 
-    public ReadTableCurrentVersion(List<String> predicateHints, Optional<Integer> limitHint) {
+    public ReadTableCurrentVersion(List<String> predicateHints, List<String> jsonPredicateHints, Optional<Integer> limitHint) {
       this.predicateHints = predicateHints;
+      this.jsonPredicateHints = jsonPredicateHints;
       this.limitHint = limitHint;
     }
 
     public List<String> predicateHints() {
       return predicateHints;
     }
 
+    public List<String> jsonPredicateHints() {
+      return jsonPredicateHints;
+    }
+
     public Optional<Integer> limitHint() {
       return limitHint;
     }
@@ -131,7 +153,8 @@ public Optional<Integer> limitHint() {
     @SkipCoverageGenerated
     public String toString() {
       return "ReadTableCurrentVersion{" + "predicateHints="
-          + predicateHints + ", limitHint="
+          + predicateHints + "jsonPredicateHints="
+          + jsonPredicateHints + ", limitHint="
           + limitHint + '}';
     }
 
@@ -142,13 +165,14 @@ public boolean equals(Object o) {
       if (o == null || getClass() != o.getClass()) return false;
       ReadTableCurrentVersion that = (ReadTableCurrentVersion) o;
       return Objects.equals(predicateHints, that.predicateHints)
+          && Objects.equals(jsonPredicateHints, that.jsonPredicateHints)
           && Objects.equals(limitHint, that.limitHint);
     }
 
     @Override
     @SkipCoverageGenerated
     public int hashCode() {
-      return Objects.hash(predicateHints, limitHint);
+      return Objects.hash(jsonPredicateHints, predicateHints, limitHint);
     }
   }
 }

server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java

@@ -88,7 +88,7 @@ public Optional<Long> getTableVersion(Optional<String> startingTimestamp) {
 
   private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) {
     try {
-      var parsedPredicate = JsonPredicatesUtils.parseJsonPredicate(predicate);
+      var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate);
       return parsedPredicate.evalExpectBoolean(ctx);
     } catch (PredicateException e) {
       logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
@@ -99,13 +99,42 @@ private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile
     }
   }
 
+  private boolean evaluateSqlPredicate(String predicate, EvalContext ctx, AddFile f, Metadata metadata) {
+    try {
+      var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata);
+      return parsedPredicate.evalExpectBoolean(ctx);
+    } catch (PredicateException e) {
+      logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage());
+      logger.info("File: " + f.getPath()
+              + " will be used in processing due to failure in parsing or processing the predicate: "
+              + predicate);
+      return true;
+    }
+  }
+
+  public boolean filterFilesBasedOnSqlPredicates(List<String> predicates, AddFile f, Metadata metadata) {
+    // if there are no predicates return all possible files
+    if (predicates == null) {
+      return true;
+    }
+    try {
+      var ctx = PredicateUtils.createEvalContext(f);
+      return predicates.stream().allMatch(p -> evaluateSqlPredicate(p, ctx, f, metadata));
+    } catch (PredicateException e) {
+      logger.debug("Caught exception: " + e.getMessage());
+      logger.info("File: " + f.getPath()
+              + " will be used in processing due to failure in parsing or processing the predicate");
+      return true;
+    }
+  }
+
   public boolean filterFilesBasedOnJsonPredicates(List<String> predicates, AddFile f) {
     // if there are no predicates return all possible files
     if (predicates == null) {
       return true;
     }
     try {
-      var ctx = JsonPredicatesUtils.createEvalContext(f);
+      var ctx = PredicateUtils.createEvalContext(f);
       return predicates.stream().allMatch(p -> evaluateJsonPredicate(p, ctx, f));
     } catch (PredicateException e) {
       logger.debug("Caught exception: " + e.getMessage());
@@ -114,30 +143,35 @@ public boolean filterFilesBasedOnJsonPredicates(List<String> predicates, AddFile
       return true;
     }
   }
-  ;
 
   public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) {
     List<String> predicates;
+    List<String> sqlPredicates;
     Snapshot snapshot;
     if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) {
       snapshot = deltaLog.snapshot();
-      predicates = ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).predicateHints();
+      predicates = ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).jsonPredicateHints();
+      sqlPredicates = ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).predicateHints();
     } else if (readTableRequest instanceof ReadTableRequest.ReadTableAsOfTimestamp) {
       snapshot = deltaLog.getSnapshotForTimestampAsOf(
           ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).timestamp());
-      predicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).predicateHints();
+      predicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).jsonPredicateHints();
+      sqlPredicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).predicateHints();
     } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) {
       snapshot = deltaLog.getSnapshotForVersionAsOf(
           ((ReadTableRequest.ReadTableVersion) readTableRequest).version());
-      predicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).predicateHints();
+      predicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).jsonPredicateHints();
+      sqlPredicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).predicateHints();
     } else {
       throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest);
     }
+    var metadata = metadataFromSnapshot(snapshot);
     return new ReadTableResultToBeSigned(
         new Protocol(Optional.of(1)),
-        metadataFromSnapshot(snapshot),
+            metadata,
         snapshot.getAllFiles().stream()
             .filter(f -> filterFilesBasedOnJsonPredicates(predicates, f))
+            .filter(f -> filterFilesBasedOnSqlPredicates(sqlPredicates, f, metadata))
             .map(f -> new TableFileToBeSigned(
                 location() + "/" + f.getPath(),
                 f.getSize(),