From 19e9aa2671344ac3c94b276e1b66d6e891b0eb54 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 8 Oct 2024 23:41:55 -0700 Subject: [PATCH 1/6] Fixing Failing Tests --- hadoop-tools/hadoop-azure/pom.xml | 28 +++++++++++++++++- .../azurebfs/ITestAbfsCustomEncryption.java | 4 ++- .../ITestAbfsRestOperationException.java | 28 ++++++++++-------- ...ITestAzureBlobFileSystemAuthorization.java | 29 +++++++++++-------- .../ITestAzureBlobFileSystemChooseSAS.java | 11 ++++++- .../ITestAbfsFileSystemContractSeek.java | 3 ++ .../services/ITestExponentialRetryPolicy.java | 12 ++++++-- 7 files changed, 85 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 460e0571a898c..f5e8e2eabf563 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -661,7 +661,6 @@ ${fs.azure.scale.test.timeout} - **/azurebfs/contract/ITest*.java **/azurebfs/ITestAzureBlobFileSystemE2EScale.java **/azurebfs/ITestAbfsReadWriteAndSeek.java **/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -673,6 +672,33 @@ + + integration-test-abfs-sequential-contract-classes + + integration-test + verify + + + + ${fs.azure.scale.test.timeout} + false + + + false + + ${fs.azure.scale.test.enabled} + ${fs.azure.scale.test.huge.filesize} + ${fs.azure.scale.test.timeout} + ${fs.azure.scale.test.list.performance.threads} + ${fs.azure.scale.test.list.performance.files} + + ${http.maxConnections} + + + **/azurebfs/contract/ITest*.java + + + diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java index b121fb9420185..837277b77b9e5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsCustomEncryption.java @@ -400,9 +400,11 @@ private AzureBlobFileSystem getAzureBlobFileSystem(final Configuration conf) { AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance( conf); fileSystemsOpenedInTest.add(fs); + // Default for this config should be true here as FNS Accounts would have failed initialization. + // This is needed to make sure test runs even if test config is missing. Assertions.assertThat( getConfiguration().getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, - false)) + true)) .describedAs("Encryption tests should run only on namespace enabled account") .isTrue(); return fs; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index ca2ab9dabcf43..2999ff57f438e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -32,9 +32,12 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -142,14 +145,16 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce AzureBlobFileSystem fs = this.getFileSystem(); Configuration config = new Configuration(this.getRawConfiguration()); - String accountName = config.get("fs.azure.abfs.account.name"); + String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); // Setup to configure custom token provider. - config.set("fs.azure.account.auth.type." + accountName, "Custom"); - config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs" - + ".azurebfs.oauth2.RetryTestTokenProvider"); - config.set("fs.azure.custom.token.fetch.retry.count", Integer.toString(numOfRetries)); + config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); + config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, + RETRY_TEST_TOKEN_PROVIDER); + config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(numOfRetries)); // Stop filesystem creation as it will lead to calls to store. - config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false"); + config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( + FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); try (final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), @@ -175,16 +180,15 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce @Test public void testAuthFailException() throws Exception { Configuration config = new Configuration(getRawConfiguration()); - String accountName = config - .get(FS_AZURE_ABFS_ACCOUNT_NAME); + String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); // Setup to configure custom token provider - config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT - + accountName, "Custom"); - config.set( - FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, + config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); + config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, RETRY_TEST_TOKEN_PROVIDER); // Stop filesystem creation as it will lead to calls to store. config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( + FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); final AzureBlobFileSystem fs = getFileSystem(config); AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java index c59009dd0feb4..99eb8c422dea2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java @@ -36,6 +36,8 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_FAIL_INIT; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN; import static org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers.aclEntry; @@ -65,7 +67,7 @@ public void setup() throws Exception { TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); Assume.assumeTrue(isHNSEnabled); loadConfiguredFileSystem(); - this.getConfiguration().set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); + this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); this.getConfiguration().set(ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); super.setup(); } @@ -75,13 +77,14 @@ public void testSASTokenProviderInitializeException() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); - Configuration testConfig = this.getConfiguration().getRawConfiguration(); - testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true"); + Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); + testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); + testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); intercept(SASTokenProviderException.class, ()-> { - testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration()); + testFs.initialize(fs.getUri(), testConfig); }); } @@ -90,12 +93,12 @@ public void testSASTokenProviderEmptySASToken() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); - Configuration testConfig = this.getConfiguration().getRawConfiguration(); - testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); + Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); + testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); - testFs.initialize(fs.getUri(), - this.getConfiguration().getRawConfiguration()); + testFs.initialize(fs.getUri(), testConfig); intercept(SASTokenProviderException.class, () -> { testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close(); @@ -107,10 +110,12 @@ public void testSASTokenProviderNullSASToken() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); - Configuration testConfig = this.getConfiguration().getRawConfiguration(); - testConfig.set(ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); + Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); + testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); + testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); - testFs.initialize(fs.getUri(), this.getConfiguration().getRawConfiguration()); + testFs.initialize(fs.getUri(), testConfig); intercept(SASTokenProviderException.class, ()-> { testFs.create(new org.apache.hadoop.fs.Path("/testFile")).close(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index d8db901151fe7..0a0f0d38d82f4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -37,6 +37,10 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_FIXED_TOKEN; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -62,8 +66,8 @@ public ITestAzureBlobFileSystemChooseSAS() throws Exception { @Override public void setup() throws Exception { - createFilesystemWithTestFileForSASTests(new Path(TEST_PATH)); super.setup(); + createFilesystemWithTestFileForSASTests(new Path(TEST_PATH)); generateAccountSAS(); } @@ -95,6 +99,11 @@ public void testBothProviderFixedTokenConfigured() throws Exception { // Configuring a SASTokenProvider class which provides a user delegation SAS. testAbfsConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, MockDelegationSASTokenProvider.class.getName()); + // Make sure test configs required by MockDelegationSASTokenProvider are set. + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_ID); + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SECRET); + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID); + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID); // configuring the Fixed SAS token which is an Account SAS. testAbfsConfig.set(FS_AZURE_SAS_FIXED_TOKEN, accountSAS); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java index f7fe5039799d7..0e230a76df5a5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java @@ -68,6 +68,9 @@ protected Configuration createConfiguration() { protected AbstractFSContract createContract(final Configuration conf) { conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE); conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE); + // Disabling cache to make sure new configs are picked up. + conf.set("fs.abfss.impl.disable.cache", "true"); + conf.set("fs.abfs.impl.disable.cache", "true"); return new AbfsFileSystemContract(conf, isSecure); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java index a3fa40a726577..25e5f8e739c21 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java @@ -20,6 +20,7 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_BACKOFF_INTERVAL; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_BACKOFF_INTERVAL; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; @@ -28,6 +29,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_AUTOTHROTTLING; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT1_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME; @@ -65,7 +67,9 @@ public class ITestExponentialRetryPolicy extends AbstractAbfsIntegrationTest { private static final String TEST_PATH = "/testfile"; private static final double MULTIPLYING_FACTOR = 1.5; private static final int ANALYSIS_PERIOD = 10000; - + private static final String dummyAccountName = "dummy.dfs.core.windows.net"; + private static final String dummyAccountName1 = "dummy1.dfs.core.windows.net"; + private static final String dummyAccountKey = "dummyKey"; public ITestExponentialRetryPolicy() throws Exception { super(); @@ -98,10 +102,12 @@ public void testThrottlingIntercept() throws Exception { final Configuration configuration = new Configuration(); configuration.addResource(TEST_CONFIGURATION_FILE_NAME); configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, false); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName, dummyAccountKey); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName1, dummyAccountKey); // On disabling throttling AbfsNoOpThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, - "dummy.dfs.core.windows.net"); + dummyAccountName); AbfsThrottlingIntercept intercept; AbfsClient abfsClient = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration); intercept = abfsClient.getIntercept(); @@ -113,7 +119,7 @@ public void testThrottlingIntercept() throws Exception { configuration.setBoolean(FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, true); // On enabling throttling AbfsClientThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration, - "dummy1.dfs.core.windows.net"); + dummyAccountName1); AbfsClient abfsClient1 = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1); intercept = abfsClient1.getIntercept(); Assertions.assertThat(intercept) From acca3fb677398e971d66abbcfc3749f0b3edde27 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 8 Oct 2024 23:49:16 -0700 Subject: [PATCH 2/6] Missed Chages --- .../fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java index 99eb8c422dea2..c95d2c2ffd032 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.MOCK_SASTOKENPROVIDER_FAIL_INIT; @@ -68,7 +69,7 @@ public void setup() throws Exception { Assume.assumeTrue(isHNSEnabled); loadConfiguredFileSystem(); this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); - this.getConfiguration().set(ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); + this.getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); super.setup(); } @@ -80,6 +81,7 @@ public void testSASTokenProviderInitializeException() throws Exception { Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); intercept(SASTokenProviderException.class, @@ -96,6 +98,7 @@ public void testSASTokenProviderEmptySASToken() throws Exception { Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); testFs.initialize(fs.getUri(), testConfig); @@ -113,6 +116,7 @@ public void testSASTokenProviderNullSASToken() throws Exception { Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); testFs.initialize(fs.getUri(), testConfig); From d7552552a8015e8c3c65fa4f46e068d824d7a7e2 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Wed, 9 Oct 2024 04:00:33 -0700 Subject: [PATCH 3/6] Follow Up Changes --- .../ITestAbfsRestOperationException.java | 44 ++++++++----------- ...ITestAzureBlobFileSystemAuthorization.java | 3 +- .../fs/azurebfs/ITestGetNameSpaceEnabled.java | 4 ++ .../services/ITestExponentialRetryPolicy.java | 16 ++++--- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index 2999ff57f438e..2ac15488d82b9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -30,12 +30,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.DOT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -143,19 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception { public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception { AzureBlobFileSystem fs = this.getFileSystem(); - - Configuration config = new Configuration(this.getRawConfiguration()); - String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); - // Setup to configure custom token provider. - config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); - config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, - RETRY_TEST_TOKEN_PROVIDER); - config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString(numOfRetries)); - // Stop filesystem creation as it will lead to calls to store. - config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); - config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( - FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); - + Configuration config = getEntries(numOfRetries); try (final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config)) { @@ -179,17 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce @Test public void testAuthFailException() throws Exception { - Configuration config = new Configuration(getRawConfiguration()); - String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); - // Setup to configure custom token provider - config.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + DOT + accountName, "Custom"); - config.set(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + DOT + accountName, - RETRY_TEST_TOKEN_PROVIDER); - // Stop filesystem creation as it will lead to calls to store. - config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); - config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( - FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); - + Configuration config = getEntries(0); final AzureBlobFileSystem fs = getFileSystem(config); AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> { fs.getFileStatus(new Path("/")); @@ -205,4 +183,20 @@ public void testAuthFailException() throws Exception { .describedAs("Incorrect error message: " + errorDesc) .contains("Auth failure: "); } + + private Configuration getEntries(final int numOfRetries) { + Configuration config = new Configuration(this.getRawConfiguration()); + String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); + // Setup to configure custom token provider. + config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom"); + config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName), + RETRY_TEST_TOKEN_PROVIDER); + config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString( + numOfRetries)); + // Stop filesystem creation as it will lead to calls to store. + config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( + FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); + return config; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java index c95d2c2ffd032..a7c6b1f76359c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; import org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider; @@ -80,7 +79,7 @@ public void testSASTokenProviderInitializeException() throws Exception { final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true"); // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index 5732ce8cfea70..058678aa4d8cd 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -44,6 +44,7 @@ import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; +import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -286,12 +287,15 @@ public void testAccountSpecificConfig() throws Exception { this.getAccountName())); String testAccountName = "testAccount.dfs.core.windows.net"; String otherAccountName = "otherAccount.dfs.core.windows.net"; + String dummyAcountKey = "dummyKey"; String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName); String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName); // Set both account specific and account agnostic config for test account rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, testAccountName), dummyAcountKey); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_KEY, otherAccountName), dummyAcountKey); // Assert that account specific config takes precedence rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java index 25e5f8e739c21..9ead6d196382e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestExponentialRetryPolicy.java @@ -67,9 +67,9 @@ public class ITestExponentialRetryPolicy extends AbstractAbfsIntegrationTest { private static final String TEST_PATH = "/testfile"; private static final double MULTIPLYING_FACTOR = 1.5; private static final int ANALYSIS_PERIOD = 10000; - private static final String dummyAccountName = "dummy.dfs.core.windows.net"; - private static final String dummyAccountName1 = "dummy1.dfs.core.windows.net"; - private static final String dummyAccountKey = "dummyKey"; + private static final String DUMMY_ACCOUNT_NAME = "dummy.dfs.core.windows.net"; + private static final String DUMMY_ACCOUNT_NAME_1 = "dummy1.dfs.core.windows.net"; + private static final String DUMMY_ACCOUNT_KEY = "dummyKey"; public ITestExponentialRetryPolicy() throws Exception { super(); @@ -102,12 +102,14 @@ public void testThrottlingIntercept() throws Exception { final Configuration configuration = new Configuration(); configuration.addResource(TEST_CONFIGURATION_FILE_NAME); configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, false); - configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName, dummyAccountKey); - configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + dummyAccountName1, dummyAccountKey); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME, + DUMMY_ACCOUNT_KEY); + configuration.set(FS_AZURE_ACCOUNT_KEY + DOT + DUMMY_ACCOUNT_NAME_1, + DUMMY_ACCOUNT_KEY); // On disabling throttling AbfsNoOpThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, - dummyAccountName); + DUMMY_ACCOUNT_NAME); AbfsThrottlingIntercept intercept; AbfsClient abfsClient = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration); intercept = abfsClient.getIntercept(); @@ -119,7 +121,7 @@ public void testThrottlingIntercept() throws Exception { configuration.setBoolean(FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, true); // On enabling throttling AbfsClientThrottlingIntercept object is returned AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration, - dummyAccountName1); + DUMMY_ACCOUNT_NAME_1); AbfsClient abfsClient1 = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1); intercept = abfsClient1.getIntercept(); Assertions.assertThat(intercept) From 0964df66711a138bbeb52799894f4388f9cd3702 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Tue, 15 Oct 2024 08:20:14 -0700 Subject: [PATCH 4/6] Resolved Commentss --- .../ITestAbfsRestOperationException.java | 20 +++++++++++-------- ...ITestAzureBlobFileSystemAuthorization.java | 18 ++++++++--------- .../ITestAbfsFileSystemContractSeek.java | 5 ++--- .../fs/azurebfs/utils/AbfsTestUtils.java | 14 +++++++++++++ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index 2ac15488d82b9..d4a068e20ae74 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -143,7 +143,7 @@ public void testCustomTokenFetchRetryCount() throws Exception { public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception { AzureBlobFileSystem fs = this.getFileSystem(); - Configuration config = getEntries(numOfRetries); + Configuration config = getCustomAuthConfiguration(numOfRetries); try (final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config)) { @@ -167,7 +167,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce @Test public void testAuthFailException() throws Exception { - Configuration config = getEntries(0); + Configuration config = getCustomAuthConfiguration(0); final AzureBlobFileSystem fs = getFileSystem(config); AbfsRestOperationException e = intercept(AbfsRestOperationException.class, () -> { fs.getFileStatus(new Path("/")); @@ -184,19 +184,23 @@ public void testAuthFailException() throws Exception { .contains("Auth failure: "); } - private Configuration getEntries(final int numOfRetries) { + /** + * Returns a configuration with a custom token provider configured. {@link RetryTestTokenProvider} + * @param numOfRetries Number of retries to be configured for token fetch. + * @return Configuration + */ + private Configuration getCustomAuthConfiguration(final int numOfRetries) { Configuration config = new Configuration(this.getRawConfiguration()); String accountName = config.get(FS_AZURE_ABFS_ACCOUNT_NAME); // Setup to configure custom token provider. config.set(accountProperty(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, accountName), "Custom"); config.set(accountProperty(FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME, accountName), RETRY_TEST_TOKEN_PROVIDER); - config.set(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, Integer.toString( - numOfRetries)); + config.setInt(AZURE_CUSTOM_TOKEN_FETCH_RETRY_COUNT, numOfRetries); // Stop filesystem creation as it will lead to calls to store. - config.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); - config.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( - FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true) + ""); + config.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, false); + config.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, config.getBoolean( + FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, true)); return config; } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java index a7c6b1f76359c..ab01b2e10c4b9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAuthorization.java @@ -63,12 +63,12 @@ public ITestAzureBlobFileSystemAuthorization() throws Exception { @Override public void setup() throws Exception { - boolean isHNSEnabled = this.getConfiguration().getBoolean( + boolean isHNSEnabled = getConfiguration().getBoolean( TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false); Assume.assumeTrue(isHNSEnabled); loadConfiguredFileSystem(); - this.getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); - this.getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, "SAS"); + getConfiguration().set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_AUTHZ_CLASS); + getConfiguration().set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.toString()); super.setup(); } @@ -79,9 +79,9 @@ public void testSASTokenProviderInitializeException() throws Exception { final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_FAIL_INIT, "true"); + testConfig.setBoolean(MOCK_SASTOKENPROVIDER_FAIL_INIT, true); // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. - testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); + testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs)); intercept(SASTokenProviderException.class, ()-> { @@ -96,9 +96,9 @@ public void testSASTokenProviderEmptySASToken() throws Exception { final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true); // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. - testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); + testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs)); testFs.initialize(fs.getUri(), testConfig); intercept(SASTokenProviderException.class, @@ -114,9 +114,9 @@ public void testSASTokenProviderNullSASToken() throws Exception { final AzureBlobFileSystem testFs = new AzureBlobFileSystem(); Configuration testConfig = new Configuration(this.getConfiguration().getRawConfiguration()); testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, TEST_ERR_AUTHZ_CLASS); - testConfig.set(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, "true"); + testConfig.setBoolean(MOCK_SASTOKENPROVIDER_RETURN_EMPTY_SAS_TOKEN, true); // Setting IS_HNS_ENABLED to avoid the exception thrown by the HNS check. - testConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs) + ""); + testConfig.setBoolean(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, this.getIsNamespaceEnabled(fs)); testFs.initialize(fs.getUri(), testConfig); intercept(SASTokenProviderException.class, diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java index 0e230a76df5a5..e8b044f92456c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.utils.AbfsTestUtils.disableFilesystemCaching; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; @@ -68,9 +69,7 @@ protected Configuration createConfiguration() { protected AbstractFSContract createContract(final Configuration conf) { conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE); conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE); - // Disabling cache to make sure new configs are picked up. - conf.set("fs.abfss.impl.disable.cache", "true"); - conf.set("fs.abfs.impl.disable.cache", "true"); + disableFilesystemCaching(conf); return new AbfsFileSystemContract(conf, isSecure); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java index 21edfe800e164..1d51229e5da47 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java @@ -21,6 +21,7 @@ import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; +import org.apache.hadoop.conf.Configuration; import org.junit.Assume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +30,8 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SCHEME; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_SECURE_SCHEME; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX; /** @@ -82,4 +85,15 @@ public void deleteContainers() throws Throwable { } LOG.info("Deleted {} test containers", count); } + + /** + * Turn off FS Caching: use if a filesystem with different options from + * the default is required. + * @param conf configuration to patch + */ + public static void disableFilesystemCaching(Configuration conf) { + // Disabling cache to make sure new configs are picked up. + conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SCHEME), true); + conf.setBoolean(String.format("fs.%s.impl.disable.cache", ABFS_SECURE_SCHEME), true); + } } From 1630291c54c0a58e92c5554b702fe53892634def Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Wed, 16 Oct 2024 07:09:56 -0700 Subject: [PATCH 5/6] Yetus Rerun --- .../hadoop/fs/azurebfs/ITestAbfsRestOperationException.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index d4a068e20ae74..f2e4f8a183be4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -144,8 +144,7 @@ public void testCustomTokenFetchRetryCount() throws Exception { public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exception { AzureBlobFileSystem fs = this.getFileSystem(); Configuration config = getCustomAuthConfiguration(numOfRetries); - try (final AzureBlobFileSystem fs1 = - (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), + try (AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config)) { RetryTestTokenProvider retryTestTokenProvider = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( From 0dc4ce008d1be3cf74af8656d638825c2435b1b5 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Fri, 18 Oct 2024 08:33:57 -0700 Subject: [PATCH 6/6] Fixed Import Ordering --- .../org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java index 1d51229e5da47..9faa54e7ba1c7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java @@ -20,12 +20,11 @@ import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; - -import org.apache.hadoop.conf.Configuration; import org.junit.Assume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.services.AuthType; @@ -37,7 +36,7 @@ /** * Some Utils for ABFS tests. */ -public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{ +public final class AbfsTestUtils extends AbstractAbfsIntegrationTest { private static final Logger LOG = LoggerFactory.getLogger(AbfsTestUtils.class);