diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index fc052021a..7a923b181 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -23,11 +23,14 @@ public class ITDeltaSharingClient implements DatasetComparer, ScalaUtils { private final StorageManagerInitializer storageManagerInitializer; private final String deltaTablePath; + private final String icebergTablePath; public ITDeltaSharingClient() { this.storageManagerInitializer = new StorageManagerInitializer(); this.deltaTablePath = TablePath.getDeltaTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json")); + this.icebergTablePath = + TablePath.getIcebergTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json")); } @BeforeAll @@ -35,6 +38,30 @@ static void initStorageManager() { new StorageManagerInitializer().initStorageManager(); } + @Test + void showS3IcebergTable1withQueryTableApi() { + var spark = TestSparkSession.newSparkSession(); + storageManagerInitializer.createIcebergTableWithGlueMetastore(); + var ds = spark.read().format("deltaSharing").load(icebergTablePath); + var expectedSchema = new StructType(new StructField[] { + new StructField("id", DataType.fromDDL("long"), false, new Metadata(emptyScalaMap())) + }); + var expectedData = spark + .createDataFrame( + List.of( + new MrFoxDeltaTableSchema(0), + new MrFoxDeltaTableSchema(3), + new MrFoxDeltaTableSchema(2), + new MrFoxDeltaTableSchema(1), + new MrFoxDeltaTableSchema(4)), + MrFoxDeltaTableSchema.class) + .toDF(); + + assertEquals(expectedSchema, ds.schema()); + assertEquals(5, ds.count()); + assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); + } + @Test void showS3Table1withQueryTableApi() { var spark = TestSparkSession.newSparkSession(); @@ -43,6 +70,8 @@ void showS3Table1withQueryTableApi() { var expectedSchema = new StructType(new StructField[] { new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap())) }); + + ds.show(); var expectedData = spark .createDataFrame( List.of( diff --git a/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java index 3f1a15f85..72b0a59ce 100644 --- a/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/S3TestConfig.java @@ -4,6 +4,7 @@ public class S3TestConfig { private final String region; private final String accessKey; private final String secretKey; + private final String glueCatalogId; public String getRegion() { return region; @@ -17,16 +18,22 @@ public String getSecretKey() { return secretKey; } - public S3TestConfig(String region, String accessKey, String secretKey) { + public String getGlueCatalogId() { + return glueCatalogId; + } + + public S3TestConfig(String region, String accessKey, String secretKey, String glueCatalogId) { this.region = region; this.accessKey = accessKey; this.secretKey = secretKey; + this.glueCatalogId = glueCatalogId; } public static S3TestConfig loadFromEnv() { return new S3TestConfig( System.getenv().get("WHITEFOX_TEST_AWS_REGION"), System.getenv().get("WHITEFOX_TEST_AWS_ACCESS_KEY_ID"), - System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY")); + System.getenv().get("WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY"), + System.getenv().get("WHITEFOX_TEST_GLUE_CATALOG_ID")); } } diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java index 19667e9e5..8824003bd 100644 --- a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java @@ -58,10 +58,18 @@ public TableInfo createIcebergTableWithGlueMetastore() { var provider = ApiUtils.recoverConflictLazy( () -> providerV1Api.addProvider(providerRequest), () -> providerV1Api.getProvider(providerRequest.getName())); + var schemaRequest = createSchemaRequest(TableFormat.iceberg); + var shareRequest = createShareRequest(); + ignoreConflict(() -> schemaV1Api.createSchema(shareRequest.getName(), schemaRequest)); var createTableRequest = createIcebergTableRequest(); - return ApiUtils.recoverConflictLazy( + ApiUtils.recoverConflictLazy( () -> tableV1Api.createTableInProvider(provider.getName(), createTableRequest), () -> tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName())); + ignoreConflict(() -> schemaV1Api.addTableToSchema( + shareRequest.getName(), + schemaRequest, + addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName()))); + return tableV1Api.describeTableInProvider(provider.getName(), createTableRequest.getName()); } private String createSchemaRequest(TableFormat tableFormat) { @@ -122,7 +130,7 @@ private CreateMetastore createMetastoreRequest( .type(type) .skipValidation(true) .properties(new MetastoreProperties(new GlueProperties() - .catalogId("catalogId") // TODO + .catalogId(s3TestConfig.getGlueCatalogId()) .credentials(new SimpleAwsCredentials() .region(s3TestConfig.getRegion()) .awsAccessKeyId(s3TestConfig.getAccessKey()) diff --git a/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java b/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java index 63a910545..e3186f693 100644 --- a/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java @@ -4,6 +4,10 @@ public class TablePath { + public static String getIcebergTablePath(URL resource) { + return String.format("%s#%s.%s.%s", resource, "s3share", "s3schemaiceberg", "s3IcebergTable1"); + } + public static String getDeltaTablePath(URL resource) { return String.format("%s#%s.%s.%s", resource, "s3share", "s3schemadelta", "s3Table1"); }