From 811ddca45a1a80f0a937950d5c1520b69a73cc79 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sun, 1 Oct 2023 23:36:32 -0700 Subject: [PATCH 1/4] HADOOP-18850 Enable dual-layer server-side encryption with AWS KMS keys --- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 3 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 5 + .../EncryptionSecretOperations.java | 3 +- .../fs/s3a/impl/RequestFactoryImpl.java | 12 ++ .../hadoop/fs/s3a/EncryptionTestUtils.java | 9 + ...3ADSSEEncryptionWithDefaultS3Settings.java | 167 ++++++++++++++++++ ...estS3AEncryptionDSSEKMSUserDefinedKey.java | 55 ++++++ ...estS3AEncryptionWithDefaultS3Settings.java | 26 ++- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 21 ++- .../scale/ITestS3AHugeFilesEncryption.java | 13 +- 10 files changed, 296 insertions(+), 18 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java index b599790a1cf76..b57f59edd7c28 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java @@ -33,7 +33,8 @@ public enum S3AEncryptionMethods { SSE_KMS("SSE-KMS", true, false), SSE_C("SSE-C", true, true), CSE_KMS("CSE-KMS", false, true), - CSE_CUSTOM("CSE-CUSTOM", false, true); + CSE_CUSTOM("CSE-CUSTOM", false, true), + DSSE_KMS("DSSE-KMS", true, false); /** * Error string when {@link #getMethod(String)} fails. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 093608fb528c7..125e0ec73eca1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -1421,6 +1421,11 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket, diagnostics); break; + case DSSE_KMS: + LOG.debug("Using DSSE-KMS with {}", + diagnostics); + break; + case NONE: default: LOG.debug("Data is unencrypted"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java index bcd358e2d1672..8a55a970134f3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java @@ -53,7 +53,8 @@ public static Optional getSSECustomerKey(final EncryptionSecrets secrets * @return an optional key to attach to a request. */ public static Optional getSSEAwsKMSKey(final EncryptionSecrets secrets) { - if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + || secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS) && secrets.hasEncryptionKey()) { return Optional.of(secrets.getEncryptionKey()); } else { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index cacbee381bebc..f59e92218563d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -279,6 +279,10 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom, // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(kmsKey -> copyObjectRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); } else if (S3AEncryptionMethods.SSE_C == algorithm) { EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) .ifPresent(base64customerKey -> { @@ -354,6 +358,10 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(kmsKey -> putObjectRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(putObjectRequestBuilder::ssekmsKeyId); } else if (S3AEncryptionMethods.SSE_C == algorithm) { EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) .ifPresent(base64customerKey -> { @@ -415,6 +423,10 @@ private void multipartUploadEncryptionParameters( // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(kmsKey -> mpuRequestBuilder.ssekmsKeyId(kmsKey)); + } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); + EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) + .ifPresent(mpuRequestBuilder::ssekmsKeyId); } else if (S3AEncryptionMethods.SSE_C == algorithm) { EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) .ifPresent(base64customerKey -> { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java index 8d927dc957b16..188c5a2c6c5d1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java @@ -39,6 +39,8 @@ private EncryptionTestUtils() { public static final String AWS_KMS_SSE_ALGORITHM = "aws:kms"; + public static final String AWS_KMS_DSSE_ALGORITHM = "aws:kms:dsse"; + public static final String SSE_C_ALGORITHM = "AES256"; /** @@ -94,6 +96,13 @@ public static void assertEncrypted(S3AFileSystem fs, kmsKeyArn, md.ssekmsKeyId()); break; + case DSSE_KMS: + assertEquals("Wrong algorithm in " + details, + AWS_KMS_DSSE_ALGORITHM, md.serverSideEncryptionAsString()); + assertEquals("Wrong KMS key in " + details, + kmsKeyArn, + md.ssekmsKeyId()); + break; default: assertEquals("AES256", md.serverSideEncryptionAsString()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java new file mode 100644 index 0000000000000..c112a26ce9191 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; + +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.EncryptionTestUtils.AWS_KMS_DSSE_ALGORITHM; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; +import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; + +/** + * Concrete class that extends {@link AbstractTestS3AEncryption} + * and tests already configured bucket level DSSE encryption using s3 console. + */ +public class ITestS3ADSSEEncryptionWithDefaultS3Settings extends + AbstractTestS3AEncryption { + + @Override + public void setup() throws Exception { + super.setup(); + // get the KMS key for this test. + S3AFileSystem fs = getFileSystem(); + Configuration c = fs.getConf(); + skipIfEncryptionNotSet(c, getSSEAlgorithm()); + } + + @SuppressWarnings("deprecation") + @Override + protected void patchConfigurationEncryptionSettings( + final Configuration conf) { + removeBaseAndBucketOverrides(conf, + S3_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_ALGORITHM); + conf.set(S3_ENCRYPTION_ALGORITHM, + getSSEAlgorithm().getMethod()); + } + + /** + * Setting this to NONE as we don't want to overwrite + * already configured encryption settings. + * @return the algorithm + */ + @Override + protected S3AEncryptionMethods getSSEAlgorithm() { + return S3AEncryptionMethods.NONE; + } + + /** + * The check here is that the object is encrypted + * and that the encryption key is the KMS key + * provided, not any default key. + * @param path path + */ + @Override + protected void assertEncrypted(Path path) throws IOException { + S3AFileSystem fs = getFileSystem(); + Configuration c = fs.getConf(); + String kmsKey = getS3EncryptionKey(getTestBucketName(c), c); + EncryptionTestUtils.assertEncrypted(fs, path, DSSE_KMS, kmsKey); + } + + @Override + @Ignore + @Test + public void testEncryptionSettingPropagation() throws Throwable { + } + + @Override + @Ignore + @Test + public void testEncryption() throws Throwable { + } + + /** + * Skipping if the test bucket is not configured with + * aws:kms encryption algorithm. + */ + @Override + public void testEncryptionOverRename() throws Throwable { + skipIfBucketNotKmsEncrypted(); + super.testEncryptionOverRename(); + } + + /** + * If the test bucket is not configured with aws:kms encryption algorithm, + * skip the test. + * + * @throws IOException If the object creation/deletion/access fails. + */ + private void skipIfBucketNotKmsEncrypted() throws IOException { + S3AFileSystem fs = getFileSystem(); + Path path = path(getMethodName() + "find-encryption-algo"); + ContractTestUtils.touch(fs, path); + try { + String sseAlgorithm = + getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString(); + if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_DSSE_ALGORITHM)) { + skip("Test bucket is not configured with " + AWS_KMS_DSSE_ALGORITHM); + } + } finally { + ContractTestUtils.assertDeleted(fs, path, false); + } + } + + @Test + public void testEncryptionOverRename2() throws Throwable { + skipIfBucketNotKmsEncrypted(); + S3AFileSystem fs = getFileSystem(); + + // write the file with the unencrypted FS. + // this will pick up whatever defaults we have. + Path src = path(createFilename(1024)); + byte[] data = dataset(1024, 'a', 'z'); + EncryptionSecrets secrets = fs.getEncryptionSecrets(); + validateEncryptionSecrets(secrets); + writeDataset(fs, src, data, data.length, 1024 * 1024, true); + ContractTestUtils.verifyFileContents(fs, src, data); + + Configuration fs2Conf = new Configuration(fs.getConf()); + fs2Conf.set(S3_ENCRYPTION_ALGORITHM, + DSSE_KMS.getMethod()); + try (FileSystem kmsFS = FileSystem.newInstance(fs.getUri(), fs2Conf)) { + Path targetDir = path("target"); + kmsFS.mkdirs(targetDir); + ContractTestUtils.rename(kmsFS, src, targetDir); + Path renamedFile = new Path(targetDir, src.getName()); + ContractTestUtils.verifyFileContents(fs, renamedFile, data); + String kmsKey = getS3EncryptionKey(getTestBucketName(fs2Conf), fs2Conf); + // we assert that the renamed file has picked up the KMS key of our FS + EncryptionTestUtils.assertEncrypted(fs, renamedFile, DSSE_KMS, kmsKey); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java new file mode 100644 index 0000000000000..4fa12f8ffeacc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; + +/** + * Concrete class that extends {@link AbstractTestS3AEncryption} + * and tests DSSE-KMS encryption. + */ +public class ITestS3AEncryptionDSSEKMSUserDefinedKey + extends AbstractTestS3AEncryption { + + @Override + protected Configuration createConfiguration() { + // get the KMS key for this test. + Configuration c = new Configuration(); + String kmsKey = S3AUtils.getS3EncryptionKey(getTestBucketName(c), c); + // skip the test if DSSE-KMS or KMS key not set. + if (StringUtils.isBlank(kmsKey)) { + skip(S3_ENCRYPTION_KEY + " is not set for " + + DSSE_KMS.getMethod()); + } + Configuration conf = super.createConfiguration(); + conf.set(S3_ENCRYPTION_KEY, kmsKey); + return conf; + } + + @Override + protected S3AEncryptionMethods getSSEAlgorithm() { + return DSSE_KMS; + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java index 1b25846fafddf..c246161a938dd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java @@ -115,20 +115,34 @@ public void testEncryption() throws Throwable { */ @Override public void testEncryptionOverRename() throws Throwable { + skipIfBucketNotKmsEncrypted(); + super.testEncryptionOverRename(); + } + + /** + * If the test bucket is not configured with aws:kms encryption algorithm, + * skip the test. + * + * @throws IOException If the object creation/deletion/access fails. + */ + private void skipIfBucketNotKmsEncrypted() throws IOException { S3AFileSystem fs = getFileSystem(); Path path = path(getMethodName() + "find-encryption-algo"); ContractTestUtils.touch(fs, path); - String sseAlgorithm = getS3AInternals().getObjectMetadata(path) - .serverSideEncryptionAsString(); - if(StringUtils.isBlank(sseAlgorithm) || - !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) { - skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM); + try { + String sseAlgorithm = + getS3AInternals().getObjectMetadata(path).serverSideEncryptionAsString(); + if (StringUtils.isBlank(sseAlgorithm) || !sseAlgorithm.equals(AWS_KMS_SSE_ALGORITHM)) { + skip("Test bucket is not configured with " + AWS_KMS_SSE_ALGORITHM); + } + } finally { + ContractTestUtils.assertDeleted(fs, path, false); } - super.testEncryptionOverRename(); } @Test public void testEncryptionOverRename2() throws Throwable { + skipIfBucketNotKmsEncrypted(); S3AFileSystem fs = getFileSystem(); // write the file with the unencrypted FS. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 239e52a7264a0..bd5ac4bc50137 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -77,6 +77,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -1455,19 +1456,25 @@ public static S3AFileStatus innerGetFileStatus( * Skip a test if encryption algorithm or encryption key is not set. * * @param configuration configuration to probe. + * @param s3AEncryptionMethods list of encryption algorithms to probe. + * @throws IOException if the secret lookup fails. */ public static void skipIfEncryptionNotSet(Configuration configuration, - S3AEncryptionMethods s3AEncryptionMethod) throws IOException { + S3AEncryptionMethods... s3AEncryptionMethods) throws IOException { + if (s3AEncryptionMethods == null || s3AEncryptionMethods.length == 0) { + throw new IllegalArgumentException("Specify at least one encryption method"); + } // if S3 encryption algorithm is not set to desired method or AWS encryption // key is not set, then skip. String bucket = getTestBucketName(configuration); final EncryptionSecrets secrets = buildEncryptionSecrets(bucket, configuration); - if (!s3AEncryptionMethod.getMethod().equals(secrets.getEncryptionMethod().getMethod()) - || StringUtils.isBlank(secrets.getEncryptionKey())) { - skip(S3_ENCRYPTION_KEY + " is not set for " + s3AEncryptionMethod - .getMethod() + " or " + S3_ENCRYPTION_ALGORITHM + " is not set to " - + s3AEncryptionMethod.getMethod() - + " in " + secrets); + boolean encryptionMethodMatching = Arrays.stream(s3AEncryptionMethods).anyMatch( + s3AEncryptionMethod -> s3AEncryptionMethod.getMethod() + .equals(secrets.getEncryptionMethod().getMethod())); + if (!encryptionMethodMatching || StringUtils.isBlank(secrets.getEncryptionKey())) { + skip(S3_ENCRYPTION_KEY + " is not set or " + S3_ENCRYPTION_ALGORITHM + " is not set to " + + Arrays.stream(s3AEncryptionMethods).map(S3AEncryptionMethods::getMethod) + .collect(Collectors.toList()) + " in " + secrets); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java index 93242155c678a..404a9684f42ec 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesEncryption.java @@ -27,6 +27,8 @@ import org.apache.hadoop.fs.s3a.EncryptionTestUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; @@ -43,7 +45,7 @@ public class ITestS3AHugeFilesEncryption extends AbstractSTestS3AHugeFiles { @Override public void setup() throws Exception { Configuration c = new Configuration(); - skipIfEncryptionNotSet(c, SSE_KMS); + skipIfEncryptionNotSet(c, SSE_KMS, DSSE_KMS); super.setup(); } @@ -67,7 +69,12 @@ protected boolean isEncrypted(S3AFileSystem fileSystem) { protected void assertEncrypted(Path hugeFile) throws IOException { Configuration c = new Configuration(); String kmsKey = getS3EncryptionKey(getTestBucketName(c), c); - EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, - SSE_KMS, kmsKey); + if (SSE_KMS.getMethod().equals(c.get(S3_ENCRYPTION_ALGORITHM))) { + EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, SSE_KMS, kmsKey); + } else if (DSSE_KMS.getMethod().equals(c.get(S3_ENCRYPTION_ALGORITHM))) { + EncryptionTestUtils.assertEncrypted(getFileSystem(), hugeFile, DSSE_KMS, kmsKey); + } else { + throw new AssertionError("Invalid encryption configured"); + } } } From 0b5af5fdfbd9c07fad50bf79bda6e97f7e681556 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 2 Oct 2023 22:37:10 -0700 Subject: [PATCH 2/4] addendum - doc, test, source changes --- .../src/main/resources/core-default.xml | 4 +- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 2 +- .../fs/s3a/impl/RequestFactoryImpl.java | 101 ++++++++++++------ .../markdown/tools/hadoop-aws/encryption.md | 83 +++++++++++++- .../markdown/tools/hadoop-aws/performance.md | 2 +- .../site/markdown/tools/hadoop-aws/testing.md | 2 +- .../hadoop/fs/s3a/EncryptionTestUtils.java | 45 ++++---- ...3ADSSEEncryptionWithDefaultS3Settings.java | 2 +- ...estS3AEncryptionDSSEKMSUserDefinedKey.java | 13 ++- 9 files changed, 186 insertions(+), 68 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 9564a56e9e40d..0c5116575249e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1718,14 +1718,14 @@ fs.s3a.encryption.algorithm Specify a server-side encryption or client-side encryption algorithm for s3a: file system. Unset by default. It supports the - following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'SSE-C', and 'CSE-KMS' + following values: 'AES256' (for SSE-S3), 'SSE-KMS', 'DSSE-KMS', 'SSE-C', and 'CSE-KMS' fs.s3a.encryption.key Specific encryption key to use if fs.s3a.encryption.algorithm - has been set to 'SSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C + has been set to 'SSE-KMS', 'DSSE-KMS', 'SSE-C' or 'CSE-KMS'. In the case of SSE-C , the value of this property should be the Base64 encoded key. If you are using SSE-KMS and leave this property empty, you'll be using your default's S3 KMS key, otherwise you should set this property to the specific KMS key diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java index b57f59edd7c28..6cacdff6c42d8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AEncryptionMethods.java @@ -40,7 +40,7 @@ public enum S3AEncryptionMethods { * Error string when {@link #getMethod(String)} fails. * Used in tests. */ - static final String UNKNOWN_ALGORITHM + public static final String UNKNOWN_ALGORITHM = "Unknown encryption algorithm "; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index f59e92218563d..ae4cd4651104b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -60,6 +60,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import static org.apache.commons.lang3.StringUtils.isNotEmpty; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.UNKNOWN_ALGORITHM; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.util.Preconditions.checkArgument; import static org.apache.hadoop.util.Preconditions.checkNotNull; @@ -272,28 +273,38 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom, return; } - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: copyObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> copyObjectRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - copyObjectRequestBuilder.copySourceSSECustomerAlgorithm( - ServerSideEncryption.AES256.name()).copySourceSSECustomerKey(base64customerKey) - .copySourceSSECustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))) - .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> copyObjectRequestBuilder + .copySourceSSECustomerAlgorithm(ServerSideEncryption.AES256.name()) + .copySourceSSECustomerKey(base64customerKey) + .copySourceSSECustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))) + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } /** @@ -351,24 +362,35 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: putObjectRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> putObjectRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + .ifPresent(putObjectRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(putObjectRequestBuilder::ssekmsKeyId); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - putObjectRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> putObjectRequestBuilder + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5(Md5Utils.md5AsBase64( + Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } @@ -416,24 +438,35 @@ private void multipartUploadEncryptionParameters( CreateMultipartUploadRequest.Builder mpuRequestBuilder) { final S3AEncryptionMethods algorithm = getServerSideEncryptionAlgorithm(); - if (S3AEncryptionMethods.SSE_S3 == algorithm) { + switch (algorithm) { + case SSE_S3: mpuRequestBuilder.serverSideEncryption(algorithm.getMethod()); - } else if (S3AEncryptionMethods.SSE_KMS == algorithm) { + break; + case SSE_KMS: mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS); // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) - .ifPresent(kmsKey -> mpuRequestBuilder.ssekmsKeyId(kmsKey)); - } else if (S3AEncryptionMethods.DSSE_KMS == algorithm) { + .ifPresent(mpuRequestBuilder::ssekmsKeyId); + break; + case DSSE_KMS: mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(mpuRequestBuilder::ssekmsKeyId); - } else if (S3AEncryptionMethods.SSE_C == algorithm) { + break; + case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) - .ifPresent(base64customerKey -> { - mpuRequestBuilder.sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) - .sseCustomerKey(base64customerKey).sseCustomerKeyMD5( - Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey))); - }); + .ifPresent(base64customerKey -> mpuRequestBuilder + .sseCustomerAlgorithm(ServerSideEncryption.AES256.name()) + .sseCustomerKey(base64customerKey) + .sseCustomerKeyMD5( + Md5Utils.md5AsBase64(Base64.getDecoder().decode(base64customerKey)))); + break; + case CSE_KMS: + case CSE_CUSTOM: + case NONE: + break; + default: + LOG.warn(UNKNOWN_ALGORITHM + ": " + algorithm); } } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index ce1286c414a60..b1d6533f834b7 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -66,7 +66,7 @@ The server-side "SSE" encryption is performed with symmetric AES256 encryption; S3 offers different mechanisms for actually defining the key to use. -There are four key management mechanisms, which in order of simplicity of use, +There are five key management mechanisms, which in order of simplicity of use, are: * S3 Default Encryption @@ -75,6 +75,9 @@ are: by Amazon's Key Management Service, a key referenced by name in the uploading client. * SSE-C : the client specifies an actual base64 encoded AES-256 key to be used to encrypt and decrypt the data. +* DSSE-KMS: Two independent layers of encryption at server side. An AES256 key is +generated in S3, and encrypted with a secret key provided by Amazon's Key Management +Service. Encryption options @@ -84,6 +87,7 @@ Encryption options | `SSE-KMS` | server side, KMS key | key used to encrypt/decrypt | none | | `SSE-C` | server side, custom key | encryption algorithm and secret | encryption algorithm and secret | | `CSE-KMS` | client side, KMS key | encryption algorithm and key ID | encryption algorithm | +| `DSSE-KMS` | server side, KMS key | key used to encrypt/decrypt | none | With server-side encryption, the data is uploaded to S3 unencrypted (but wrapped by the HTTPS encryption channel). @@ -91,7 +95,7 @@ The data is encrypted in the S3 store and decrypted when it's being retrieved. A server side algorithm can be enabled by default for a bucket, so that whenever data is uploaded unencrypted a default encryption algorithm is added. -When data is encrypted with S3-SSE or SSE-KMS it is transparent to all clients +When data is encrypted with S3-SSE, SSE-KMS or DSSE-KMS it is transparent to all clients downloading the data. SSE-C is different in that every client must know the secret key needed to decypt the data. @@ -132,7 +136,7 @@ not explicitly declare an encryption algorithm. [S3 Default Encryption for S3 Buckets](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) -This supports SSE-S3 and SSE-KMS. +This supports SSE-S3, SSE-KMS and DSSE-KMS. There is no need to set anything up in the client: do it in the AWS console. @@ -316,6 +320,79 @@ metadata. Since only one encryption key can be provided at a time, S3A will not pass the correct encryption key to decrypt the data. +### DSSE-KMS: Dual-layer Server-Encryption with KMS Managed Encryption Keys + +By providing a dual-layer server-side encryption mechanism using AWS Key Management Service +(AWS KMS) keys, known as DSSE-KMS, two layers of encryption are applied to objects upon their +upload to Amazon S3. DSSE-KMS simplifies the process of meeting compliance requirements that +mandate the implementation of multiple layers of encryption for data while maintaining complete +control over the encryption keys. + + +When uploading data encrypted with SSE-KMS, the sequence is as follows: + +1. The S3A client must declare a specific CMK in the property `fs.s3a.encryption.key`, or leave + it blank to use the default configured for that region. + +2. The S3A client uploads all the data as normal, now including encryption information. + +3. The S3 service encrypts the data with a symmetric key unique to the new object. + +4. The S3 service retrieves the chosen CMK key from the KMS service, and, if the user has + the right to use it, uses it to provide dual-layer encryption for the data. + + +When downloading DSSE-KMS encrypted data, the sequence is as follows + +1. The S3A client issues an HTTP GET request to read the data. + +2. S3 sees that the data was encrypted with DSSE-KMS, and looks up the specific key in the + KMS service. + +3. If and only if the requesting user has been granted permission to use the CMS key does + the KMS service provide S3 with the key. + +4. As a result, S3 will only decode the data if the user has been granted access to the key. + + +### Enabling DSSE-KMS + +To enable DSSE-KMS, the property `fs.s3a.encryption.algorithm` must be set to `DSSE-KMS` in `core-site`: + +```xml + + fs.s3a.encryption.algorithm + DSSE-KMS + +``` + +The ID of the specific key used to encrypt the data should also be set in the property `fs.s3a.encryption.key`: + +```xml + + fs.s3a.encryption.key + arn:aws:kms:us-west-2:360379543683:key/071a86ff-8881-4ba0-9230-95af6d01ca01 + +``` + +Organizations may define a default key in the Amazon KMS; if a default key is set, +then it will be used whenever SSE-KMS encryption is chosen and the value of `fs.s3a.encryption.key` is empty. + +### the S3A `fs.s3a.encryption.key` key only affects created files + +With SSE-KMS, the S3A client option `fs.s3a.encryption.key` sets the +key to be used when new files are created. When reading files, this key, +and indeed the value of `fs.s3a.encryption.algorithm` is ignored: +S3 will attempt to retrieve the key and decrypt the file based on the create-time settings. + +This means that + +* There's no need to configure any client simply reading data. +* It is possible for a client to read data encrypted with one KMS key, and + write it with another. + + + ## Encryption best practises diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index e3ab79d92e5dc..2c83b063b1efc 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -447,7 +447,7 @@ and rate of requests. Spreading data across different buckets, and/or using a more balanced directory structure may be beneficial. Consult [the AWS documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html). -Reading or writing data encrypted with SSE-KMS forces S3 to make calls of +Reading or writing data encrypted with SSE-KMS or DSSE-KMS forces S3 to make calls of the AWS KMS Key Management Service, which comes with its own [Request Rate Limits](http://docs.aws.amazon.com/kms/latest/developerguide/limits.html). These default to 1200/second for an account, across all keys and all uses of diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index bfec94b19c101..3560489146717 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1033,7 +1033,7 @@ The specific tests an Assumed Role ARN is required for are To run these tests you need: 1. A role in your AWS account will full read and write access rights to -the S3 bucket used in the tests, and KMS for any SSE-KMS tests. +the S3 bucket used in the tests, and KMS for any SSE-KMS or DSSE-KMS tests. 1. Your IAM User to have the permissions to "assume" that role. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java index 188c5a2c6c5d1..7b2b1c639e3cc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/EncryptionTestUtils.java @@ -28,8 +28,7 @@ import org.apache.hadoop.fs.Path; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.assertj.core.api.Assertions.assertThat; public final class EncryptionTestUtils { @@ -79,32 +78,36 @@ public static void assertEncrypted(S3AFileSystem fs, md.ssekmsKeyId()); switch(algorithm) { case SSE_C: - assertNull("Metadata algorithm should have been null in " - + details, - md.serverSideEncryptionAsString()); - assertEquals("Wrong SSE-C algorithm in " - + details, - SSE_C_ALGORITHM, md.sseCustomerAlgorithm()); + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isNull(); + assertThat(md.sseCustomerAlgorithm()) + .describedAs("Details of SSE-C algorithm: %s", details) + .isEqualTo(SSE_C_ALGORITHM); String md5Key = convertKeyToMd5(fs); - assertEquals("getSSECustomerKeyMd5() wrong in " + details, - md5Key, md.sseCustomerKeyMD5()); + assertThat(md.sseCustomerKeyMD5()) + .describedAs("Details of the customer provided encryption key: %s", details) + .isEqualTo(md5Key); break; case SSE_KMS: - assertEquals("Wrong algorithm in " + details, - AWS_KMS_SSE_ALGORITHM, md.serverSideEncryptionAsString()); - assertEquals("Wrong KMS key in " + details, - kmsKeyArn, - md.ssekmsKeyId()); + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isEqualTo(AWS_KMS_SSE_ALGORITHM); + assertThat(md.ssekmsKeyId()) + .describedAs("Details of the KMS key: %s", details) + .isEqualTo(kmsKeyArn); break; case DSSE_KMS: - assertEquals("Wrong algorithm in " + details, - AWS_KMS_DSSE_ALGORITHM, md.serverSideEncryptionAsString()); - assertEquals("Wrong KMS key in " + details, - kmsKeyArn, - md.ssekmsKeyId()); + assertThat(md.serverSideEncryptionAsString()) + .describedAs("Details of the server-side encryption algorithm used: %s", details) + .isEqualTo(AWS_KMS_DSSE_ALGORITHM); + assertThat(md.ssekmsKeyId()) + .describedAs("Details of the KMS key: %s", details) + .isEqualTo(kmsKeyArn); break; default: - assertEquals("AES256", md.serverSideEncryptionAsString()); + assertThat(md.serverSideEncryptionAsString()) + .isEqualTo("AES256"); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java index c112a26ce9191..a39490174424c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADSSEEncryptionWithDefaultS3Settings.java @@ -123,7 +123,7 @@ public void testEncryptionOverRename() throws Throwable { */ private void skipIfBucketNotKmsEncrypted() throws IOException { S3AFileSystem fs = getFileSystem(); - Path path = path(getMethodName() + "find-encryption-algo"); + Path path = methodPath(); ContractTestUtils.touch(fs, path); try { String sseAlgorithm = diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java index 4fa12f8ffeacc..8ac2c5acc5fcc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java @@ -18,13 +18,16 @@ package org.apache.hadoop.fs.s3a; +import java.io.IOException; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; /** * Concrete class that extends {@link AbstractTestS3AEncryption} @@ -39,10 +42,12 @@ protected Configuration createConfiguration() { Configuration c = new Configuration(); String kmsKey = S3AUtils.getS3EncryptionKey(getTestBucketName(c), c); // skip the test if DSSE-KMS or KMS key not set. - if (StringUtils.isBlank(kmsKey)) { - skip(S3_ENCRYPTION_KEY + " is not set for " + - DSSE_KMS.getMethod()); + try { + skipIfEncryptionNotSet(c, DSSE_KMS); + } catch (IOException e) { + throw new RuntimeException(e); } + assume("KMS key is expected to be present", StringUtils.isNotBlank(kmsKey)); Configuration conf = super.createConfiguration(); conf.set(S3_ENCRYPTION_KEY, kmsKey); return conf; From ed0c59a28158582d655c0ab75421dc003ee87301 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 26 Oct 2023 00:20:12 -0700 Subject: [PATCH 3/4] addendum - doc links --- .../src/site/markdown/tools/hadoop-aws/encryption.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index b1d6533f834b7..11bb2937db994 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -354,6 +354,9 @@ When downloading DSSE-KMS encrypted data, the sequence is as follows 4. As a result, S3 will only decode the data if the user has been granted access to the key. +Further reading on DSSE-KMS [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingDSSEncryption.html) + +AWS Blog post [here](https://aws.amazon.com/blogs/aws/new-amazon-s3-dual-layer-server-side-encryption-with-keys-stored-in-aws-key-management-service-dsse-kms/) ### Enabling DSSE-KMS From a195f0e159c3fda2bac66fec0c41ffadf07af10d Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 26 Oct 2023 00:24:28 -0700 Subject: [PATCH 4/4] addendum --- .../hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java index 8ac2c5acc5fcc..028ba7f6e1fe0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionDSSEKMSUserDefinedKey.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import java.io.IOException; +import java.io.UncheckedIOException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -45,7 +46,7 @@ protected Configuration createConfiguration() { try { skipIfEncryptionNotSet(c, DSSE_KMS); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } assume("KMS key is expected to be present", StringUtils.isNotBlank(kmsKey)); Configuration conf = super.createConfiguration();