From 00e49620049999dcf83f149100840512251d0677 Mon Sep 17 00:00:00 2001 From: Zoe Wang <33073555+zoewangg@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:25:21 -0700 Subject: [PATCH 1/4] Implement auto mode discovery --- .../AutoDefaultsModeDiscovery.java | 126 ++++++++++ .../AutoDefaultsModeDiscoveryTest.java | 222 ++++++++++++++++++ .../test/resources/jetty-logging.properties | 18 ++ .../src/test/resources/log4j.properties | 35 +++ .../defaults/sdk-default-configuration.json | 8 + .../utils/internal/SystemSettingUtils.java | 2 +- 6 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java create mode 100644 core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java create mode 100644 core/aws-core/src/test/resources/jetty-logging.properties create mode 100644 core/aws-core/src/test/resources/log4j.properties diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java new file mode 100644 index 000000000000..c0ea4b00c8d6 --- /dev/null +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java @@ -0,0 +1,126 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.awscore.internal.defaultsmode; + +import java.util.Optional; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.defaultsmode.DefaultsMode; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.internal.util.EC2MetadataUtils; +import software.amazon.awssdk.utils.JavaSystemSetting; +import software.amazon.awssdk.utils.OptionalUtils; +import software.amazon.awssdk.utils.SystemSetting; +import software.amazon.awssdk.utils.internal.SystemSettingUtils; + +/** + * This class attempts to discover the appropriate {@link DefaultsMode} the mode by inspecting the environment. It falls + * back to the standard defaults mode if the target mode cannot be determined. + */ +@SdkInternalApi +public final class AutoDefaultsModeDiscovery { + private static final DefaultsMode FALLBACK_DEFAULTS_MODE = DefaultsMode.STANDARD; + private static final String ANDROID_JAVA_VENDOR = "The Android Project"; + private static final String AWS_DEFAULT_REGION_ENV_VAR = "AWS_DEFAULT_REGION"; + + /** + * Discovers the defaultMode using the following workflow: + * + * 1. Check if it's on mobile + * 2. If it's not on mobile (best we can tell), see if we can determine whether we're an in-region or cross-region client. + * 3. If we couldn't figure out the region from environment variables. Check IMDSv2 + * 4. Finally, use fallback mode + */ + public DefaultsMode discover(Region regionResolvedFromSdkClient) { + + if (isMobile()) { + return DefaultsMode.MOBILE; + } + + if (isAwsExecutionEnvironment()) { + Optional regionStr = regionFromAwsExecutionEnvironment(); + + if (regionStr.isPresent()) { + return compareRegion(regionStr.get(), regionResolvedFromSdkClient.id()); + } + } + + Optional regionFromEc2 = queryImdsV2(); + if (regionFromEc2.isPresent()) { + return compareRegion(regionFromEc2.get(), regionResolvedFromSdkClient.id()); + } + + return FALLBACK_DEFAULTS_MODE; + } + + private static DefaultsMode compareRegion(String region, String anotherRegion) { + if (region.equalsIgnoreCase(anotherRegion)) { + return DefaultsMode.IN_REGION; + } + + return DefaultsMode.CROSS_REGION; + } + + private static Optional queryImdsV2() { + try { + String ec2InstanceRegion = EC2MetadataUtils.getEC2InstanceRegion(); + // ec2InstanceRegion could be null + return Optional.ofNullable(ec2InstanceRegion); + } catch (Exception exception) { + return Optional.empty(); + } + } + + /** + * Check to see if the application is running on a mobile device by verifying the Java + * vendor system property. Currently only checks for Android. While it's technically possible to + * use Java with iOS, it's not a common use-case. + *

+ * https://developer.android.com/reference/java/lang/System#getProperties() + */ + private static boolean isMobile() { + return JavaSystemSetting.JAVA_VENDOR.getStringValue() + .filter(o -> o.equals(ANDROID_JAVA_VENDOR)) + .isPresent(); + } + + private static boolean isAwsExecutionEnvironment() { + return SdkSystemSetting.AWS_EXECUTION_ENV.getStringValue().isPresent(); + } + + private static Optional regionFromAwsExecutionEnvironment() { + Optional regionFromRegionEnvVar = SdkSystemSetting.AWS_REGION.getStringValue(); + return OptionalUtils.firstPresent(regionFromRegionEnvVar, + () -> SystemSettingUtils.resolveEnvironmentVariable(new DefaultRegionEnvVar())); + } + + private static final class DefaultRegionEnvVar implements SystemSetting { + @Override + public String property() { + return null; + } + + @Override + public String environmentVariable() { + return AWS_DEFAULT_REGION_ENV_VAR; + } + + @Override + public String defaultValue() { + return null; + } + } +} diff --git a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java new file mode 100644 index 000000000000..be2c2ad29a9a --- /dev/null +++ b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java @@ -0,0 +1,222 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.awscore.internal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.put; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.Callable; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import software.amazon.awssdk.awscore.internal.defaultsmode.AutoDefaultsModeDiscovery; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.defaultsmode.DefaultsMode; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.testutils.EnvironmentVariableHelper; +import software.amazon.awssdk.utils.JavaSystemSetting; + +@RunWith(Parameterized.class) +public class AutoDefaultsModeDiscoveryTest { + private static final EnvironmentVariableHelper ENVIRONMENT_VARIABLE_HELPER = new EnvironmentVariableHelper(); + @Parameterized.Parameter + public TestData testData; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[] { + new TestData().clientRegion(Region.US_EAST_1) + .javaVendorProperty("The Android Project") + .awsExecutionEnvVar("AWS_Lambda_java8") + .awsRegionEnvVar("us-east-1") + .expectedResolvedMode(DefaultsMode.MOBILE), + + new TestData().clientRegion(Region.US_EAST_1) + .awsExecutionEnvVar("AWS_Lambda_java8") + .awsRegionEnvVar("us-east-1") + .expectedResolvedMode(DefaultsMode.IN_REGION), + + new TestData().clientRegion(Region.US_EAST_1) + .awsExecutionEnvVar("AWS_Lambda_java8") + .awsDefaultRegionEnvVar("us-west-2") + .expectedResolvedMode(DefaultsMode.CROSS_REGION), + + new TestData().clientRegion(Region.US_EAST_1) + .awsDefaultRegionEnvVar("us-west-2") + .ec2MetadataConfig(new Ec2MetadataConfig().region("us-east-1") + .imdsAvailable(true)) + .expectedResolvedMode(DefaultsMode.IN_REGION), + + new TestData().clientRegion(Region.US_EAST_1) + .awsDefaultRegionEnvVar("us-west-2") + .ec2MetadataConfig(new Ec2MetadataConfig().region("us-west-2") + .imdsAvailable(true) + .ec2MetadataDisabledEnvVar("false")) + .expectedResolvedMode(DefaultsMode.CROSS_REGION), + + new TestData().clientRegion(Region.US_EAST_1) + .awsDefaultRegionEnvVar("us-west-2") + .ec2MetadataConfig(new Ec2MetadataConfig().imdsAvailable(false)) + .expectedResolvedMode(DefaultsMode.STANDARD), + }); + } + + @Rule + public WireMockRule wireMock = new WireMockRule(0); + + @Before + public void methodSetup() { + System.setProperty(SdkSystemSetting.AWS_EC2_METADATA_SERVICE_ENDPOINT.property(), + "http://localhost:" + wireMock.port()); + } + + @After + public void cleanUp() { + wireMock.resetAll(); + ENVIRONMENT_VARIABLE_HELPER.reset(); + System.clearProperty(JavaSystemSetting.JAVA_VENDOR.property()); + } + + @Test + public void differentCombinationOfConfigs_shouldResolveCorrectly() throws Exception { + if (testData.javaVendorProperty != null) { + System.setProperty(JavaSystemSetting.JAVA_VENDOR.property(), testData.javaVendorProperty); + } + + if (testData.awsExecutionEnvVar != null) { + ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_EXECUTION_ENV.environmentVariable(), + testData.awsExecutionEnvVar); + } + + if (testData.awsRegionEnvVar != null) { + ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_REGION.environmentVariable(), testData.awsRegionEnvVar); + } + + if (testData.awsDefaultRegionEnvVar != null) { + ENVIRONMENT_VARIABLE_HELPER.set("AWS_DEFAULT_REGION", testData.awsDefaultRegionEnvVar); + } + + if (testData.ec2MetadataConfig != null) { + if (testData.ec2MetadataConfig.ec2MetadataDisabledEnvVar != null) { + ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_EC2_METADATA_DISABLED.environmentVariable(), + testData.ec2MetadataConfig.ec2MetadataDisabledEnvVar); + } + + if (testData.ec2MetadataConfig.imdsAvailable) { + stubSuccessfulResponse(testData.ec2MetadataConfig.region); + } + } + + Callable result = () -> new AutoDefaultsModeDiscovery().discover(testData.clientRegion); + assertThat(result.call()).isEqualTo(testData.expectedResolvedMode); + } + + public void stubSuccessfulResponse(String region) { + stubFor(put("/latest/api/token") + .willReturn(aResponse().withStatus(200).withBody("token"))); + + + stubFor(get("/latest/dynamic/instance-identity/document") + .willReturn(aResponse().withStatus(200).withBody(constructInstanceInfo(region)))); + } + + private String constructInstanceInfo(String region) { + return String.format("{" + + "\"pendingTime\":\"2014-08-07T22:07:46Z\"," + + "\"instanceType\":\"m1.small\"," + + "\"imageId\":\"ami-123456\"," + + "\"instanceId\":\"i-123456\"," + + "\"region\":\"%s\"," + + "\"version\":\"2010-08-31\"" + + "}", region); + } + + + private static final class TestData { + private Region clientRegion; + private String javaVendorProperty; + private String awsExecutionEnvVar; + private String awsRegionEnvVar; + private String awsDefaultRegionEnvVar; + private Ec2MetadataConfig ec2MetadataConfig; + private DefaultsMode expectedResolvedMode; + + public TestData clientRegion(Region clientRegion) { + this.clientRegion = clientRegion; + return this; + } + + public TestData javaVendorProperty(String javaVendorProperty) { + this.javaVendorProperty = javaVendorProperty; + return this; + } + + public TestData awsExecutionEnvVar(String awsExecutionEnvVar) { + this.awsExecutionEnvVar = awsExecutionEnvVar; + return this; + } + + public TestData awsRegionEnvVar(String awsRegionEnvVar) { + this.awsRegionEnvVar = awsRegionEnvVar; + return this; + } + + public TestData awsDefaultRegionEnvVar(String awsDefaultRegionEnvVar) { + this.awsDefaultRegionEnvVar = awsDefaultRegionEnvVar; + return this; + } + + public TestData ec2MetadataConfig(Ec2MetadataConfig ec2MetadataConfig) { + this.ec2MetadataConfig = ec2MetadataConfig; + return this; + } + + public TestData expectedResolvedMode(DefaultsMode expectedResolvedMode) { + this.expectedResolvedMode = expectedResolvedMode; + return this; + } + } + + private static final class Ec2MetadataConfig { + private boolean imdsAvailable; + private String region; + private String ec2MetadataDisabledEnvVar; + + public Ec2MetadataConfig imdsAvailable(boolean imdsAvailable) { + this.imdsAvailable = imdsAvailable; + return this; + } + + public Ec2MetadataConfig region(String region) { + this.region = region; + return this; + } + + public Ec2MetadataConfig ec2MetadataDisabledEnvVar(String ec2MetadataDisabledEnvVar) { + this.ec2MetadataDisabledEnvVar = ec2MetadataDisabledEnvVar; + return this; + } + } +} \ No newline at end of file diff --git a/core/aws-core/src/test/resources/jetty-logging.properties b/core/aws-core/src/test/resources/jetty-logging.properties new file mode 100644 index 000000000000..4ee410e7fa92 --- /dev/null +++ b/core/aws-core/src/test/resources/jetty-logging.properties @@ -0,0 +1,18 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +# Set up logging implementation +org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog +org.eclipse.jetty.LEVEL=OFF diff --git a/core/aws-core/src/test/resources/log4j.properties b/core/aws-core/src/test/resources/log4j.properties new file mode 100644 index 000000000000..391579e1fcc1 --- /dev/null +++ b/core/aws-core/src/test/resources/log4j.properties @@ -0,0 +1,35 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.appender.A1=org.apache.log4j.ConsoleAppender +log4j.appender.A1.layout=org.apache.log4j.PatternLayout + +# Print the date in ISO 8601 format +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n + +# Adjust to see more / less logging +#log4j.logger.com.amazonaws.ec2=DEBUG + +# HttpClient 3 Wire Logging +#log4j.logger.httpclient.wire=DEBUG + +# HttpClient 4 Wire Logging +# log4j.logger.org.apache.http.wire=INFO +# log4j.logger.org.apache.http=DEBUG +# log4j.logger.org.apache.http.wire=DEBUG +# log4j.logger.software.amazonaws.awssdk=DEBUG + + diff --git a/core/sdk-core/src/main/resources/software/amazon/awssdk/internal/defaults/sdk-default-configuration.json b/core/sdk-core/src/main/resources/software/amazon/awssdk/internal/defaults/sdk-default-configuration.json index e99d2b038098..f17aeb223dce 100644 --- a/core/sdk-core/src/main/resources/software/amazon/awssdk/internal/defaults/sdk-default-configuration.json +++ b/core/sdk-core/src/main/resources/software/amazon/awssdk/internal/defaults/sdk-default-configuration.json @@ -31,6 +31,14 @@ "tlsNegotiationTimeoutInMillis": { "override": 3100 } + }, + "mobile": { + "connectTimeoutInMillis": { + "override": 10000 + }, + "tlsNegotiationTimeoutInMillis": { + "override": 10000 + } } }, "documentation": { diff --git a/utils/src/main/java/software/amazon/awssdk/utils/internal/SystemSettingUtils.java b/utils/src/main/java/software/amazon/awssdk/utils/internal/SystemSettingUtils.java index 7b13ada07f33..488dd6842477 100644 --- a/utils/src/main/java/software/amazon/awssdk/utils/internal/SystemSettingUtils.java +++ b/utils/src/main/java/software/amazon/awssdk/utils/internal/SystemSettingUtils.java @@ -73,7 +73,7 @@ private static Optional resolveProperty(SystemSetting setting) { /** * Attempt to load this setting from the environment variables. */ - private static Optional resolveEnvironmentVariable(SystemSetting setting) { + public static Optional resolveEnvironmentVariable(SystemSetting setting) { try { // CHECKSTYLE:OFF - This is the only place we're allowed to use System.getenv return Optional.ofNullable(setting.environmentVariable()).map(System::getenv); From 37fd0db4ad09f2716b4d62a4d82e4cb6cd95e8fb Mon Sep 17 00:00:00 2001 From: Zoe Wang <33073555+zoewangg@users.noreply.github.com> Date: Fri, 22 Oct 2021 11:21:01 -0700 Subject: [PATCH 2/4] Fix tests on CodeBuild --- .../awscore/internal/AutoDefaultsModeDiscoveryTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java index be2c2ad29a9a..7cbd30118395 100644 --- a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java +++ b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java @@ -108,14 +108,20 @@ public void differentCombinationOfConfigs_shouldResolveCorrectly() throws Except if (testData.awsExecutionEnvVar != null) { ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_EXECUTION_ENV.environmentVariable(), testData.awsExecutionEnvVar); + } else { + ENVIRONMENT_VARIABLE_HELPER.remove(SdkSystemSetting.AWS_EXECUTION_ENV.environmentVariable()); } if (testData.awsRegionEnvVar != null) { ENVIRONMENT_VARIABLE_HELPER.set(SdkSystemSetting.AWS_REGION.environmentVariable(), testData.awsRegionEnvVar); + } else { + ENVIRONMENT_VARIABLE_HELPER.remove(SdkSystemSetting.AWS_REGION.environmentVariable()); } if (testData.awsDefaultRegionEnvVar != null) { ENVIRONMENT_VARIABLE_HELPER.set("AWS_DEFAULT_REGION", testData.awsDefaultRegionEnvVar); + } else { + ENVIRONMENT_VARIABLE_HELPER.remove("AWS_DEFAULT_REGION"); } if (testData.ec2MetadataConfig != null) { From e8931583264c62339ca5326677c1517a0f8318d7 Mon Sep 17 00:00:00 2001 From: Zoe Wang <33073555+zoewangg@users.noreply.github.com> Date: Mon, 25 Oct 2021 15:48:53 -0700 Subject: [PATCH 3/4] Address feedback --- .../AutoDefaultsModeDiscovery.java | 15 ++++---- .../AutoDefaultsModeDiscoveryTest.java | 34 +++++++++++-------- .../internal/util/EC2MetadataUtils.java | 32 ++++++++++++++--- .../internal/util/EC2MetadataUtilsTest.java | 15 ++++++++ 4 files changed, 70 insertions(+), 26 deletions(-) diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java index c0ea4b00c8d6..37b1abf5d836 100644 --- a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java @@ -27,11 +27,12 @@ import software.amazon.awssdk.utils.internal.SystemSettingUtils; /** - * This class attempts to discover the appropriate {@link DefaultsMode} the mode by inspecting the environment. It falls - * back to the standard defaults mode if the target mode cannot be determined. + * This class attempts to discover the appropriate {@link DefaultsMode} by inspecting the environment. It falls + * back to the {@link DefaultsMode#STANDARD} mode if the target mode cannot be determined. */ @SdkInternalApi public final class AutoDefaultsModeDiscovery { + private static final String EC2_METADATA_REGION_PATH = "/latest/meta-data/placement/region"; private static final DefaultsMode FALLBACK_DEFAULTS_MODE = DefaultsMode.STANDARD; private static final String ANDROID_JAVA_VENDOR = "The Android Project"; private static final String AWS_DEFAULT_REGION_ENV_VAR = "AWS_DEFAULT_REGION"; @@ -54,20 +55,20 @@ public DefaultsMode discover(Region regionResolvedFromSdkClient) { Optional regionStr = regionFromAwsExecutionEnvironment(); if (regionStr.isPresent()) { - return compareRegion(regionStr.get(), regionResolvedFromSdkClient.id()); + return compareRegion(regionStr.get(), regionResolvedFromSdkClient); } } Optional regionFromEc2 = queryImdsV2(); if (regionFromEc2.isPresent()) { - return compareRegion(regionFromEc2.get(), regionResolvedFromSdkClient.id()); + return compareRegion(regionFromEc2.get(), regionResolvedFromSdkClient); } return FALLBACK_DEFAULTS_MODE; } - private static DefaultsMode compareRegion(String region, String anotherRegion) { - if (region.equalsIgnoreCase(anotherRegion)) { + private static DefaultsMode compareRegion(String region, Region clientRegion) { + if (region.equalsIgnoreCase(clientRegion.id())) { return DefaultsMode.IN_REGION; } @@ -76,7 +77,7 @@ private static DefaultsMode compareRegion(String region, String anotherRegion) { private static Optional queryImdsV2() { try { - String ec2InstanceRegion = EC2MetadataUtils.getEC2InstanceRegion(); + String ec2InstanceRegion = EC2MetadataUtils.fetchData(EC2_METADATA_REGION_PATH, false, 1); // ec2InstanceRegion could be null return Optional.ofNullable(ec2InstanceRegion); } catch (Exception exception) { diff --git a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java index 7cbd30118395..ae3cd7bc7e12 100644 --- a/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java +++ b/core/aws-core/src/test/java/software/amazon/awssdk/awscore/internal/AutoDefaultsModeDiscoveryTest.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.defaultsmode.DefaultsMode; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.internal.util.EC2MetadataUtils; import software.amazon.awssdk.testutils.EnvironmentVariableHelper; import software.amazon.awssdk.utils.JavaSystemSetting; @@ -47,28 +48,34 @@ public class AutoDefaultsModeDiscoveryTest { @Parameterized.Parameters public static Collection data() { return Arrays.asList(new Object[] { + + // Mobile new TestData().clientRegion(Region.US_EAST_1) .javaVendorProperty("The Android Project") .awsExecutionEnvVar("AWS_Lambda_java8") .awsRegionEnvVar("us-east-1") .expectedResolvedMode(DefaultsMode.MOBILE), + // Region available from AWS execution environment new TestData().clientRegion(Region.US_EAST_1) .awsExecutionEnvVar("AWS_Lambda_java8") .awsRegionEnvVar("us-east-1") .expectedResolvedMode(DefaultsMode.IN_REGION), + // Region available from AWS execution environment new TestData().clientRegion(Region.US_EAST_1) .awsExecutionEnvVar("AWS_Lambda_java8") .awsDefaultRegionEnvVar("us-west-2") .expectedResolvedMode(DefaultsMode.CROSS_REGION), + // ImdsV2 available, in-region new TestData().clientRegion(Region.US_EAST_1) .awsDefaultRegionEnvVar("us-west-2") .ec2MetadataConfig(new Ec2MetadataConfig().region("us-east-1") .imdsAvailable(true)) .expectedResolvedMode(DefaultsMode.IN_REGION), + // ImdsV2 available, cross-region new TestData().clientRegion(Region.US_EAST_1) .awsDefaultRegionEnvVar("us-west-2") .ec2MetadataConfig(new Ec2MetadataConfig().region("us-west-2") @@ -76,6 +83,15 @@ public static Collection data() { .ec2MetadataDisabledEnvVar("false")) .expectedResolvedMode(DefaultsMode.CROSS_REGION), + // Imdsv2 disabled, should not query ImdsV2 and use fallback mode + new TestData().clientRegion(Region.US_EAST_1) + .awsDefaultRegionEnvVar("us-west-2") + .ec2MetadataConfig(new Ec2MetadataConfig().region("us-west-2") + .imdsAvailable(true) + .ec2MetadataDisabledEnvVar("true")) + .expectedResolvedMode(DefaultsMode.STANDARD), + + // Imdsv2 not available, should use fallback mode. new TestData().clientRegion(Region.US_EAST_1) .awsDefaultRegionEnvVar("us-west-2") .ec2MetadataConfig(new Ec2MetadataConfig().imdsAvailable(false)) @@ -94,6 +110,7 @@ public void methodSetup() { @After public void cleanUp() { + EC2MetadataUtils.clearCache(); wireMock.resetAll(); ENVIRONMENT_VARIABLE_HELPER.reset(); System.clearProperty(JavaSystemSetting.JAVA_VENDOR.property()); @@ -143,23 +160,10 @@ public void stubSuccessfulResponse(String region) { stubFor(put("/latest/api/token") .willReturn(aResponse().withStatus(200).withBody("token"))); - - stubFor(get("/latest/dynamic/instance-identity/document") - .willReturn(aResponse().withStatus(200).withBody(constructInstanceInfo(region)))); + stubFor(get("/latest/meta-data/placement/region") + .willReturn(aResponse().withStatus(200).withBody(region))); } - private String constructInstanceInfo(String region) { - return String.format("{" - + "\"pendingTime\":\"2014-08-07T22:07:46Z\"," - + "\"instanceType\":\"m1.small\"," - + "\"imageId\":\"ami-123456\"," - + "\"instanceId\":\"i-123456\"," - + "\"region\":\"%s\"," - + "\"version\":\"2010-08-31\"" - + "}", region); - } - - private static final class TestData { private Region clientRegion; private String javaVendorProperty; diff --git a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java index 13a2dbb245eb..edbc931f2f14 100644 --- a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java +++ b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java @@ -39,7 +39,15 @@ import software.amazon.awssdk.regions.util.ResourcesEndpointProvider; /** - * Utility class for retrieving Amazon EC2 instance metadata.
+ + * + * Utility class for retrieving Amazon EC2 instance metadata. + * + *

+ * Note: this is an internal API subject to change. Users of the SDK + * should not depend on this. + * + *

* You can use the data to build more generic AMIs that can be modified by * configuration files supplied at launch time. For example, if you run web * servers for various small businesses, they can all use the same AMI and @@ -361,7 +369,7 @@ public static List getItems(String path, int tries) { } @SdkTestInternalApi - static void clearCache() { + public static void clearCache() { CACHE.clear(); } @@ -391,6 +399,11 @@ private static List getItems(String path, int tries, boolean slurp) { log.warn("Unable to retrieve the requested metadata."); return null; } catch (IOException | URISyntaxException | RuntimeException e) { + // If there is no retry available, just throw exception instead of pausing. + if (tries - 1 == 0) { + throw SdkClientException.builder().message("Unable to contact EC2 metadata service.").cause(e).build(); + } + // Retry on any other exceptions int pause = (int) (Math.pow(2, DEFAULT_QUERY_RETRIES - tries) * MINIMUM_RETRY_WAIT_TIME_MILLISECONDS); try { @@ -427,19 +440,30 @@ public static String getToken() { } } - private static String fetchData(String path) { return fetchData(path, false); } private static String fetchData(String path, boolean force) { + return fetchData(path, force, DEFAULT_QUERY_RETRIES); + } + + /** + * Fetch data using the given path + * + * @param path the path + * @param force whether to force to override the value in the cache + * @param attempts the number of attempts that should be executed. + * @return the value retrieved from the path + */ + public static String fetchData(String path, boolean force, int attempts) { if (SdkSystemSetting.AWS_EC2_METADATA_DISABLED.getBooleanValueOrThrow()) { throw SdkClientException.builder().message("EC2 metadata usage is disabled.").build(); } try { if (force || !CACHE.containsKey(path)) { - CACHE.put(path, getData(path)); + CACHE.put(path, getData(path, attempts)); } return CACHE.get(path); } catch (SdkClientException e) { diff --git a/core/regions/src/test/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtilsTest.java b/core/regions/src/test/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtilsTest.java index e24b403e542c..4172db957e78 100644 --- a/core/regions/src/test/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtilsTest.java +++ b/core/regions/src/test/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtilsTest.java @@ -24,7 +24,9 @@ import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.assertj.core.api.Assertions.assertThat; + import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.http.Fault; import com.github.tomakehurst.wiremock.junit.WireMockRule; import org.junit.Before; import org.junit.Rule; @@ -131,4 +133,17 @@ public void getAmiId_queriesTokenResource_400Error_throws() { EC2MetadataUtils.getAmiId(); } + + @Test + public void fetchDataWithAttemptNumber_ioError_shouldHonor() { + int attempts = 1; + thrown.expect(SdkClientException.class); + thrown.expectMessage("Unable to contact EC2 metadata service"); + + stubFor(put(urlPathEqualTo(TOKEN_RESOURCE_PATH)).willReturn(aResponse().withBody("some-token")));; + stubFor(get(urlPathEqualTo(AMI_ID_RESOURCE)).willReturn(aResponse().withFault(Fault.CONNECTION_RESET_BY_PEER))); + + EC2MetadataUtils.fetchData(AMI_ID_RESOURCE, false, attempts); + WireMock.verify(attempts, getRequestedFor(urlPathEqualTo(AMI_ID_RESOURCE))); + } } From 43839be91a462853a13999699842e3f87f3fdcdb Mon Sep 17 00:00:00 2001 From: Zoe Wang <33073555+zoewangg@users.noreply.github.com> Date: Mon, 25 Oct 2021 16:30:40 -0700 Subject: [PATCH 4/4] Add comment and rename misnamed constant --- .../defaultsmode/AutoDefaultsModeDiscovery.java | 3 ++- .../awssdk/regions/internal/util/EC2MetadataUtils.java | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java index 37b1abf5d836..868da56e1dd6 100644 --- a/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java +++ b/core/aws-core/src/main/java/software/amazon/awssdk/awscore/internal/defaultsmode/AutoDefaultsModeDiscovery.java @@ -42,7 +42,8 @@ public final class AutoDefaultsModeDiscovery { * * 1. Check if it's on mobile * 2. If it's not on mobile (best we can tell), see if we can determine whether we're an in-region or cross-region client. - * 3. If we couldn't figure out the region from environment variables. Check IMDSv2 + * 3. If we couldn't figure out the region from environment variables. Check IMDSv2. This step might take up to 1 second + * (default connect timeout) * 4. Finally, use fallback mode */ public DefaultsMode discover(Region regionResolvedFromSdkClient) { diff --git a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java index edbc931f2f14..4280003676aa 100644 --- a/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java +++ b/core/regions/src/main/java/software/amazon/awssdk/regions/internal/util/EC2MetadataUtils.java @@ -81,7 +81,7 @@ public final class EC2MetadataUtils { private static final String EC2_METADATA_TOKEN_HEADER = "x-aws-ec2-metadata-token"; - private static final int DEFAULT_QUERY_RETRIES = 3; + private static final int DEFAULT_QUERY_ATTEMPTS = 3; private static final int MINIMUM_RETRY_WAIT_TIME_MILLISECONDS = 250; private static final Logger log = LoggerFactory.getLogger(EC2MetadataUtils.class); private static final Map CACHE = new ConcurrentHashMap<>(); @@ -349,7 +349,7 @@ private static String[] stringArrayValue(JsonNode jsonNode) { } public static String getData(String path) { - return getData(path, DEFAULT_QUERY_RETRIES); + return getData(path, DEFAULT_QUERY_ATTEMPTS); } public static String getData(String path, int tries) { @@ -361,7 +361,7 @@ public static String getData(String path, int tries) { } public static List getItems(String path) { - return getItems(path, DEFAULT_QUERY_RETRIES, false); + return getItems(path, DEFAULT_QUERY_ATTEMPTS, false); } public static List getItems(String path, int tries) { @@ -405,7 +405,7 @@ private static List getItems(String path, int tries, boolean slurp) { } // Retry on any other exceptions - int pause = (int) (Math.pow(2, DEFAULT_QUERY_RETRIES - tries) * MINIMUM_RETRY_WAIT_TIME_MILLISECONDS); + int pause = (int) (Math.pow(2, DEFAULT_QUERY_ATTEMPTS - tries) * MINIMUM_RETRY_WAIT_TIME_MILLISECONDS); try { Thread.sleep(pause < MINIMUM_RETRY_WAIT_TIME_MILLISECONDS ? MINIMUM_RETRY_WAIT_TIME_MILLISECONDS : pause); @@ -445,7 +445,7 @@ private static String fetchData(String path) { } private static String fetchData(String path, boolean force) { - return fetchData(path, force, DEFAULT_QUERY_RETRIES); + return fetchData(path, force, DEFAULT_QUERY_ATTEMPTS); } /**