wippp

Author: agilelab-tmnd1991

server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

@@ -5,6 +5,9 @@
 import io.whitefox.core.*;
 import io.whitefox.core.Schema;
 import io.whitefox.core.Share;
+import io.whitefox.core.delta.Metadata;
+import io.whitefox.core.delta.Protocol;
+import io.whitefox.core.results.ReadTableResult;
 import io.whitefox.core.services.DeltaSharingCapabilities;
 import java.util.*;
 import java.util.stream.Collectors;

server/app/src/main/java/io/whitefox/api/server/WhitefoxMappers.java

@@ -14,6 +14,9 @@
 import io.whitefox.core.actions.*;
 import io.whitefox.core.actions.CreateMetastore;
 import io.whitefox.core.actions.CreateStorage;
+import io.whitefox.core.delta.Metadata;
+import io.whitefox.core.delta.Protocol;
+
 import java.util.ArrayList;
 import java.util.Optional;
 import java.util.function.Function;

server/core/build.gradle.kts

@@ -2,6 +2,7 @@ plugins {
     `java-library`
     `java-test-fixtures`
     id("whitefox.java-conventions")
+    id("io.freefair.lombok") version "8.4"
 }
 
 val quarkusPlatformGroupId: String by project

server/core/src/main/java/io/whitefox/core/ReadTableResult.java

@@ -9,9 +9,7 @@ public class TableFileToBeSigned {
 
   private final String url;
   private final long size;
-
   private final long version;
-
   private final Optional<Long> timestamp;
 
   private final String stats;

server/core/src/main/java/io/whitefox/core/ReadTableResultToBeSigned.java

@@ -1,12 +1,9 @@
-package io.whitefox.core;
+package io.whitefox.core.delta;
 
 import io.whitefox.annotations.SkipCoverageGenerated;
-import io.whitefox.core.services.DeltaSharingCapabilities;
 import io.whitefox.core.types.StructType;
-import shadedelta.org.apache.parquet.hadoop.metadata.ParquetMetadata;
 
 import java.util.*;
-import java.util.stream.Collectors;
 
 public abstract class Metadata {
   Metadata() {

server/core/src/main/java/io/whitefox/core/TableFileToBeSigned.java

@@ -1,8 +1,9 @@
-package io.whitefox.core;
+package io.whitefox.core.delta;
 
 import io.whitefox.annotations.SkipCoverageGenerated;
 import io.whitefox.core.services.DeltaSharingCapabilities;
 
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 

server/core/src/main/java/io/whitefox/core/Metadata.java renamed to server/core/src/main/java/io/whitefox/core/delta/Metadata.java

@@ -0,0 +1,25 @@
+package io.whitefox.core.delta;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+import java.util.List;
+import java.util.Set;
+
+@Data
+@Builder(toBuilder = true)
+@AllArgsConstructor
+@RequiredArgsConstructor
+public class Stats {
+    private final long numRecords;
+    private final Set<ColumnStat<Double>> minValues = Set.of();
+    private final Set<ColumnStat<Double>> maxValues = Set.of();
+    private final Set<ColumnStat<Long>> nullCount = Set.of();
+    @Data
+    public static class ColumnStat<T> {
+        private final List<String> columnPath;
+        private final T value;
+    }
+}

server/core/src/main/java/io/whitefox/core/Protocol.java renamed to server/core/src/main/java/io/whitefox/core/delta/Protocol.java

@@ -0,0 +1,59 @@
+package io.whitefox.core.delta.signed;
+
+import io.whitefox.core.delta.unsigned.DeltaFileToBeSigned;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+import java.util.Optional;
+
+@Data
+@Builder(toBuilder = true)
+@AllArgsConstructor
+public class DeltaFile implements DeltaFileAction {
+
+    /**
+     * A unique string for the file in a table. The same file is guaranteed to have the same id across multiple requests. A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+     */
+    private final String id;
+
+    /**
+     * A unique string for the deletion vector file in a table. The same deletion vector file is guaranteed to have the same id across multiple requests. A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+     */
+    private final Optional<String> deletionVectorFileId;
+
+    /**
+     * The table version of the file, returned when querying a table data with a version or timestamp parameter.
+     */
+    private final Optional<Long> version;
+
+    /**
+     * The unix timestamp corresponding to the table version of the file, in milliseconds, returned when querying a table data with a version or timestamp parameter.
+     */
+    private final Optional<Long> timestamp;
+
+    /**
+     * The unix timestamp corresponding to the expiration of the url, in milliseconds, returned when the server supports the feature.
+     */
+    private final Optional<Long> expirationTimestamp;
+
+    /**
+     * Need to be parsed by a delta library as a delta single action, the path field is replaced by pr-signed url.
+     */
+    private final ParquetFileAction deltaSingleAction;
+
+    public static DeltaFile signed(DeltaFileToBeSigned tbs,
+                                   ParquetFileAction signed,
+                                   Optional<Long> newExpirationTimestamp,
+                                   String newId) {
+        return new DeltaFile(
+                newId,
+                tbs.getDeletionVectorFileId(),
+                tbs.getVersion(),
+                tbs.getTimestamp(),
+                newExpirationTimestamp,
+                signed
+        );
+    }
+}

server/core/src/main/java/io/whitefox/core/delta/Stats.java

@@ -0,0 +1,4 @@
+package io.whitefox.core.delta.signed;
+
+public interface DeltaFileAction extends FileAction {
+}

server/core/src/main/java/io/whitefox/core/delta/signed/DeltaFile.java

@@ -0,0 +1,4 @@
+package io.whitefox.core.delta.signed;
+
+public interface FileAction {
+}

server/core/src/main/java/io/whitefox/core/delta/signed/DeltaFileAction.java

@@ -0,0 +1,74 @@
+package io.whitefox.core.delta.signed;
+
+import io.whitefox.core.delta.Stats;
+import io.whitefox.core.delta.unsigned.ParquetAddFileToBeSigned;
+import io.whitefox.core.delta.unsigned.ParquetCDFFileToBeSigned;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+import java.net.URI;
+import java.util.Map;
+import java.util.Optional;
+
+@Data
+@AllArgsConstructor
+@RequiredArgsConstructor
+@Builder(toBuilder = true)
+public class ParquetAddFile implements ParquetFileAction {
+    /**
+     * An https url that a client can use to read the file directly. The same file in different responses may have different urls
+     */
+    private final String url;
+    /**
+     * A unique string for the file in a table. The same file is guaranteed to have the same id across multiple requests. A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+     */
+    private final String id;
+    /**
+     * A map from partition column to value for this file. When the table doesn’t have partition columns, this will be an empty map.
+     */
+    private final Map<String, String> partitionValues;
+    /**
+     * The size of this file in bytes.
+     */
+    private final long size;
+    /**
+     * The timestamp of the file in milliseconds from epoch.
+     */
+    private final long timestamp;
+    /**
+     * The table version of this file
+     */
+    private final int version;
+    /**
+     * Contains statistics (e.g., count, min/max values for columns) about the data in this file. This field may be missing. A file may or may not have stats. A client can decide whether to use stats or drop it.
+     */
+    private final Optional<Stats> stats;
+    /**
+     * The unix timestamp corresponding to the expiration of the url, in milliseconds, returned when the server supports the feature.
+     */
+    private final Optional<Long> expirationTimestamp;
+
+    @Override
+    public Optional<Long> getExpTs() {
+        return expirationTimestamp;
+    }
+
+    public static ParquetAddFile signed(
+            ParquetAddFileToBeSigned f,
+            URI signedUrl,
+            Optional<Long> expirationTimestamp,
+            String newId
+    ) {
+        return new ParquetAddFile(
+                signedUrl.toASCIIString(),
+                newId,
+                f.getPartitionValues(),
+                f.getSize(),
+                f.getTimestamp(),
+                f.getVersion(),
+                f.getStats(),
+                expirationTimestamp);
+    }
+}