Skip to content

Commit 5ca16d8

Browse files
agilelab-tmnd1991agilelab-tmnd1991
committed
Add Delta format and protocol. still missing files
1 parent d9cf442 commit 5ca16d8

12 files changed

+467
-240
lines changed

‎protocol/delta-sharing-protocol-api.yml

Lines changed: 0 additions & 235 deletions
Original file line numberDiff line numberDiff line change
@@ -767,241 +767,6 @@ components:
767767
type: string
768768
message:
769769
type: string
770-
771-
DeltaTableQueryResponseObject:
772-
type: object
773-
properties:
774-
protocol:
775-
# it refers to ./delta-sharing-protocol.md#protocol
776-
$ref: '#/components/schemas/DeltaProtocolObject'
777-
metadata:
778-
# it refers to ./delta-sharing-protocol.md#metadata
779-
$ref: '#/components/schemas/DeltaMetadataObject'
780-
files:
781-
type: array
782-
items:
783-
# it refers to ./delta-sharing-protocol.md#file
784-
$ref: '#/components/schemas/DeltaFileObject'
785-
DeltaProtocolObject:
786-
type: object
787-
properties:
788-
protocol:
789-
type: object
790-
properties:
791-
deltaProtocol:
792-
type: object
793-
properties:
794-
minReaderVersion:
795-
type: integer
796-
format: int32
797-
minWriterVersion:
798-
type: integer
799-
format: int32
800-
DeltaFormatObject:
801-
type: object
802-
properties:
803-
provider:
804-
type: string
805-
options:
806-
type: object
807-
additionalProperties:
808-
type: string
809-
required:
810-
- provider
811-
DeltaMetadata:
812-
type: object
813-
description: see https://github.com/delta-io/delta/blob/master/PROTOCOL.md#change-metadata
814-
required:
815-
- id
816-
- format
817-
- schemaString
818-
- partitionColumns
819-
- configuration
820-
properties:
821-
id:
822-
type: string
823-
name:
824-
type: string
825-
description:
826-
type: string
827-
format:
828-
$ref: '#/components/schemas/DeltaFormatObject'
829-
schemaString:
830-
type: string
831-
partitionColumns:
832-
type: array
833-
items:
834-
type: string
835-
createdTime:
836-
type: integer
837-
format: int64
838-
configuration:
839-
type: object
840-
additionalProperties:
841-
type: string
842-
DeltaMetadataObject:
843-
type: object
844-
properties:
845-
metaData:
846-
type: object
847-
properties:
848-
version:
849-
type: integer
850-
format: int64
851-
size:
852-
type: integer
853-
format: int64
854-
numFiles:
855-
type: integer
856-
format: int64
857-
deltaMetadata:
858-
$ref: '#/components/schemas/DeltaMetadata'
859-
required: [ deltaMetadata ]
860-
DeltaFileObject:
861-
required:
862-
- id
863-
- deltaSingleAction
864-
properties:
865-
id:
866-
type: string
867-
deletionVectorFileId:
868-
type: string
869-
version:
870-
type: integer
871-
format: int64
872-
timestamp:
873-
type: integer
874-
format: int64
875-
expirationTimestamp:
876-
type: integer
877-
format: int64
878-
deltaSingleAction:
879-
$ref: '#/components/schemas/DeltaSingleAction'
880-
DeltaSingleAction:
881-
type: object
882-
description: only one field can be not null, container of delta actions such as file, add, cdf or remove see https://github.com/delta-io/delta/tree/master/kernel/kernel-api/src/main/java/io/delta/kernel/internal/actions
883-
properties:
884-
add:
885-
$ref: '#/components/schemas/DeltaAddFileForCDFAction'
886-
cdf:
887-
$ref: '#/components/schemas/DeltaAddCDCFileAction'
888-
file:
889-
$ref: '#/components/schemas/DeltaAddFileAction'
890-
remove:
891-
$ref: '#/components/schemas/DeltaRemoveFileAction'
892-
DeltaAddFileForCDFAction:
893-
type: object
894-
description: see io.delta.sharing.server.model.AddFileForCDF
895-
properties:
896-
url:
897-
type: string
898-
id:
899-
type: string
900-
partitionValues:
901-
type: object
902-
additionalProperties:
903-
type: string
904-
size:
905-
type: integer
906-
format: int64
907-
expirationTimestamp:
908-
type: integer
909-
format: int64
910-
version:
911-
type: integer
912-
format: int64
913-
timestamp:
914-
type: integer
915-
format: int64
916-
stats:
917-
type: string
918-
DeltaAddCDCFileAction:
919-
type: object
920-
description: see io.delta.sharing.server.model.AddCDCFile
921-
properties:
922-
url:
923-
type: string
924-
id:
925-
type: string
926-
partitionValues:
927-
type: object
928-
additionalProperties:
929-
type: string
930-
size:
931-
type: integer
932-
format: int64
933-
expirationTimestamp:
934-
type: integer
935-
format: int64
936-
timestamp:
937-
type: integer
938-
format: int64
939-
version:
940-
type: integer
941-
format: int64
942-
DeltaAddFileAction:
943-
type: object
944-
description: see io.delta.sharing.server.model.AddFile
945-
properties:
946-
url:
947-
type: string
948-
id:
949-
type: string
950-
partitionValues:
951-
type: object
952-
additionalProperties:
953-
type: string
954-
size:
955-
type: integer
956-
format: int64
957-
stats:
958-
type: string
959-
expirationTimestamp:
960-
type: integer
961-
format: int64
962-
timestamp:
963-
type: integer
964-
format: int64
965-
version:
966-
type: integer
967-
format: int64
968-
DeltaRemoveFileAction:
969-
type: object
970-
properties:
971-
url:
972-
type: string
973-
id:
974-
type: string
975-
partitionValues:
976-
type: object
977-
additionalProperties:
978-
type: string
979-
size:
980-
type: integer
981-
format: int64
982-
expirationTimestamp:
983-
type: integer
984-
format: int64
985-
timestamp:
986-
type: integer
987-
format: int64
988-
version:
989-
type: integer
990-
format: int64
991-
DeltaEndStreamAction:
992-
description: An action that is returned as the last line of the streaming response. It allows the server to include additional data that might be dynamically generated while the streaming message is sent
993-
type: object
994-
properties:
995-
refreshToken:
996-
type: string
997-
description: a token used to refresh pre-signed urls for a long running query
998-
nextPageToken:
999-
type: string
1000-
description: a token used to retrieve the subsequent page of a query
1001-
minUrlExpirationTimestamp:
1002-
description: the minimum url expiration timestamp of the urls returned in current response
1003-
type: integer
1004-
format: int64
1005770
responses:
1006771
"400":
1007772
description: The request is malformed

‎server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/Format.java

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,26 @@
11
package io.whitefox.api.deltasharing.model.v1;
22

3+
import com.fasterxml.jackson.annotation.JsonCreator;
34
import com.fasterxml.jackson.annotation.JsonProperty;
4-
import lombok.Value;
5+
import lombok.EqualsAndHashCode;
56

6-
@Value
7+
@EqualsAndHashCode
78
public class Format {
9+
private static final String PARQUET = "parquet";
10+
811
@JsonProperty
9-
String provider = "parquet";
12+
public String provider() {
13+
return PARQUET;
14+
}
15+
16+
public Format() {
17+
this(PARQUET);
18+
}
19+
20+
@JsonCreator
21+
private Format(@JsonProperty("provider") String provider) {
22+
if (!"parquet".equalsIgnoreCase(provider)) {
23+
throw new IllegalArgumentException("Provider must be " + PARQUET);
24+
}
25+
}
1026
}

0 commit comments

Comments
 (0)