Use AWS SDK, if available, for AWS credential fetching (#1017)

* Add optional dependencies for both AWS SDK v2 and v1
* For credential fetching, use AWS SDK v2 if available. Otherwise use AWS SDK v1. Otherwise, use the existing driver implementations

JAVA-4718

Author: jyemin

.evergreen/.evg.yml

@@ -371,7 +371,9 @@ functions:
             PASS=$(urlencode ${iam_auth_ecs_secret_access_key})
             MONGODB_URI="mongodb://$USER:$PASS@localhost"
           EOF
-          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
+          AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
+          .evergreen/run-mongodb-aws-test.sh
 
   "run aws auth test with assume role credentials":
     - command: shell.exec
@@ -400,7 +402,9 @@ functions:
             SESSION_TOKEN=$(urlencode $SESSION_TOKEN)
             MONGODB_URI="mongodb://$USER:$PASS@localhost"
           EOF
-          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} DRIVERS_TOOLS=${DRIVERS_TOOLS} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} DRIVERS_TOOLS=${DRIVERS_TOOLS} \
+          AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
+          .evergreen/run-mongodb-aws-test.sh
 
   "run aws auth test with aws EC2 credentials":
     - command: shell.exec
@@ -420,7 +424,7 @@ functions:
           ${PREPARE_SHELL}
           # Write an empty prepare_mongodb_aws so no auth environment variables are set.
           echo "" > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
-          JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
 
   "run aws auth test with aws credentials as environment variables":
     - command: shell.exec
@@ -449,7 +453,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
 
   "run aws auth test with aws credentials and session token as environment variables":
     - command: shell.exec
@@ -479,7 +483,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
+          JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh
 
   "run aws ECS auth test":
     - command: shell.exec
@@ -1637,6 +1641,22 @@ axes:
         variables:
           LOGIN_CONTEXT_NAME: "com.sun.security.jgss.initiate"
 
+  - id: aws-credential-provider
+    display_name: AWS Credential Provider
+    values:
+      - id: aws_sdk_v2
+        display_name: AWS SDK V2
+        variables:
+          AWS_CREDENTIAL_PROVIDER: "awsSdkV2"
+      - id: aws_sdk_v1
+        display_name: AWS SDK V1
+        variables:
+          AWS_CREDENTIAL_PROVIDER: "awsSdkV1"
+      - id: built_in
+        display_name: Built-In
+        variables:
+          AWS_CREDENTIAL_PROVIDER: "builtIn"
+
 task_groups:
   - name: testgcpkms_task_group
     setup_group_can_fail_task: true
@@ -1813,15 +1833,22 @@ buildvariants:
     - name: "plain-auth-test"
 
 - matrix_name: "aws-auth-test"
-  matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu" }
-  display_name: "MONGODB-AWS Auth test ${version} ${jdk}"
+  matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu",
+                 aws-credential-provider: "*" }
+  display_name: "MONGODB-AWS Auth test ${version} ${jdk} ${aws-credential-provider}"
   run_on: ubuntu1804-test
   tasks:
     - name: "aws-auth-test-with-regular-aws-credentials"
     - name: "aws-auth-test-with-assume-role-credentials"
     - name: "aws-auth-test-with-aws-credentials-as-environment-variables"
     - name: "aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables"
     - name: "aws-auth-test-with-aws-EC2-credentials"
+
+- matrix_name: "aws-ecs-auth-test"
+  matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu" }
+  display_name: "MONGODB-AWS ECS Auth test ${version} ${jdk}"
+  run_on: ubuntu1804-test
+  tasks:
     - name: "aws-ECS-auth-test"
 
 - matrix_name: "accept-api-version-2-test"

.evergreen/run-mongodb-aws-ecs-test.sh

@@ -44,5 +44,30 @@ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")"
 ./gradlew -version
 
 echo "Running tests..."
-./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --debug --info driver-core:test --tests AwsAuthenticationSpecification
+./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=awsSdkV2 --stacktrace --debug --info \
+           driver-core:test --tests AwsAuthenticationSpecification
+first=$?
+echo $first
+
+./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=awsSdkV1 --stacktrace --debug --info \
+           driver-core:test --tests AwsAuthenticationSpecification
+second=$?
+echo $second
+
+./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=builtIn --stacktrace --debug --info \
+           driver-core:test --tests AwsAuthenticationSpecification
+third=$?
+echo $third
+
+if [ $first -ne 0 ]; then
+   exit $first
+elif [ $second -ne 0 ]; then
+   exit $second
+elif [ $third -ne 0 ]; then
+   exit $third
+else
+   exit 0
+fi
+
+
 cd -

.evergreen/run-mongodb-aws-test.sh

@@ -4,9 +4,9 @@ set -o xtrace
 set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
-#       JDK                     Set the version of java to be used.  Java versions can be set from the java toolchain /opt/java
-#                               "jdk5", "jdk6", "jdk7", "jdk8", "jdk9", "jdk11"
-
+#  JDK                               Set the version of java to be used.  Java versions can be set from the java toolchain /opt/java
+#                                    "jdk5", "jdk6", "jdk7", "jdk8", "jdk9", "jdk11"
+#  AWS_CREDENTIAL_PROVIDER           "builtIn", 'awsSdkV1', 'awsSdkV2'
 ############################################
 #            Main Program                  #
 ############################################
@@ -37,5 +37,6 @@ echo "Running tests with Java ${JAVA_VERSION}"
 
 # As this script may be executed multiple times in a single task, with different values for MONGODB_URI, it's necessary
 # to run cleanTest to ensure that the test actually executes each run
-./gradlew -PjavaVersion=${JAVA_VERSION} -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --debug --info --no-build-cache \
-driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification
+./gradlew -PjavaVersion="${JAVA_VERSION}" -Dorg.mongodb.test.uri="${MONGODB_URI}" \
+-Dorg.mongodb.test.aws.credential.provider="${AWS_CREDENTIAL_PROVIDER}" \
+--stacktrace --debug --info --no-build-cache driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification

build.gradle

@@ -47,6 +47,8 @@ ext {
     nettyVersion = '4.1.79.Final'
     snappyVersion = '1.1.8.4'
     zstdVersion = '1.5.2-3'
+    awsSdkV2Version = '2.17.293'
+    awsSdkV1Version = '1.12.291'
     mongoCryptVersion = '1.6.0-alpha0'
     projectReactorVersion = '2020.0.22'
     junitBomVersion = '5.8.1'

driver-core/build.gradle

@@ -47,6 +47,12 @@ dependencies {
     api "io.netty:netty-buffer", optional
     api "io.netty:netty-transport", optional
     api "io.netty:netty-handler", optional
+
+    // Optionally depend on both AWS SDK v2 and v1.  The driver will use v2 is present, v1 if present, or built-in functionality if
+    // neither are present
+    implementation "software.amazon.awssdk:auth:$awsSdkV2Version", optional
+    implementation "com.amazonaws:aws-java-sdk-core:$awsSdkV1Version", optional
+
     implementation "org.xerial.snappy:snappy-java:$snappyVersion", optional
     implementation "com.github.luben:zstd-jni:$zstdVersion", optional
     implementation "org.mongodb:mongodb-crypt:$mongoCryptVersion", optional

driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java

@@ -17,61 +17,82 @@
 package com.mongodb.internal.authentication;
 
 import com.mongodb.AwsCredential;
-import org.bson.BsonDocument;
+import com.mongodb.diagnostics.logging.Logger;
+import com.mongodb.diagnostics.logging.Loggers;
+import com.mongodb.internal.VisibleForTesting;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.function.Supplier;
 
-import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;
+import static com.mongodb.internal.VisibleForTesting.AccessModifier.PRIVATE;
 
 /**
  * Utility class for working with AWS authentication.
  *
  * <p>This class should not be considered a part of the public API.</p>
  */
 public final class AwsCredentialHelper {
+    public static final Logger LOGGER = Loggers.getLogger("authenticator");
 
-    public static AwsCredential obtainFromEnvironment() {
-        if (System.getenv("AWS_ACCESS_KEY_ID") != null) {
-            return obtainFromEnvironmentVariables();
+    private static volatile Supplier<AwsCredential> awsCredentialSupplier;
+
+    static {
+        if (isClassAvailable("software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider")) {
+            awsCredentialSupplier = new AwsSdkV2CredentialSupplier();
+            LOGGER.info("Using DefaultCredentialsProvider from AWS SDK v2 to retrieve AWS credentials. This is the recommended "
+                    + "configuration");
+        } else if (isClassAvailable("com.amazonaws.auth.DefaultAWSCredentialsProviderChain")) {
+            awsCredentialSupplier = new AwsSdkV1CredentialSupplier();
+            LOGGER.info("Using DefaultAWSCredentialsProviderChain from AWS SDK v1 to retrieve AWS credentials. Consider adding a "
+                    + "dependency to AWS SDK v2's software.amazon.awssdk:auth artifact to get access to additional AWS authentication "
+                    + "functionality.");
         } else {
-            return obtainFromEc2OrEcsResponse();
+            awsCredentialSupplier = new BuiltInAwsCredentialSupplier();
+            LOGGER.info("Using built-in driver implementation to retrieve AWS credentials. Consider adding a dependency to AWS SDK "
+                    + "v2's software.amazon.awssdk:auth artifact to get access to additional AWS authentication functionality.");
         }
     }
 
-    private static AwsCredential obtainFromEnvironmentVariables() {
-        return new AwsCredential(
-                System.getenv("AWS_ACCESS_KEY_ID"),
-                System.getenv("AWS_SECRET_ACCESS_KEY"),
-                System.getenv("AWS_SESSION_TOKEN"));
+    private static boolean isClassAvailable(final String className) {
+        try {
+            Class.forName(className);
+            return true;
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
     }
 
-    private static AwsCredential obtainFromEc2OrEcsResponse() {
-        String path = System.getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
-        BsonDocument ec2OrEcsResponse = path == null ? BsonDocument.parse(getEc2Response()) : BsonDocument.parse(getEcsResponse(path));
-
-        return new AwsCredential(
-                ec2OrEcsResponse.getString("AccessKeyId").getValue(),
-                ec2OrEcsResponse.getString("SecretAccessKey").getValue(),
-                ec2OrEcsResponse.getString("Token").getValue());
+    /**
+     * This method is visible to allow tests to require the built-in provider rather than rely on the fixed checks for classes on the
+     * classpath.  It allows us to easily write tests of the built-in implementation without resorting to runtime classpath shenanigans.
+     */
+    @VisibleForTesting(otherwise = PRIVATE)
+    public static void requireBuiltInProvider() {
+        LOGGER.info("Using built-in driver implementation to retrieve AWS credentials");
+        awsCredentialSupplier = new BuiltInAwsCredentialSupplier();
     }
 
-    private static String getEcsResponse(final String path) {
-        return getHttpContents("GET", "http://169.254.170.2" + path, null);
+    /**
+     * This method is visible to allow tests to require the AWS SDK v1 provider rather than rely on the fixed checks for classes on the
+     * classpath.  It allows us to easily write tests of the AWS SDK v1 implementation without resorting to runtime classpath shenanigans.
+     */
+    @VisibleForTesting(otherwise = PRIVATE)
+    public static void requireAwsSdkV1Provider() {
+        LOGGER.info("Using AWS SDK v1 to retrieve AWS credentials");
+        awsCredentialSupplier = new AwsSdkV1CredentialSupplier();
     }
 
-    private static String getEc2Response() {
-        final String endpoint = "http://169.254.169.254";
-        final String path = "/latest/meta-data/iam/security-credentials/";
-
-        Map<String, String> header = new HashMap<>();
-        header.put("X-aws-ec2-metadata-token-ttl-seconds", "30");
-        String token = getHttpContents("PUT", endpoint + "/latest/api/token", header);
+    /**
+     * This method is visible to allow tests to require the AWS SDK v2 provider rather than rely on the fixed checks for classes on the
+     * classpath.  It allows us to easily write tests of the AWS SDK v2 implementation without resorting to runtime classpath shenanigans.
+     */
+    @VisibleForTesting(otherwise = PRIVATE)
+    public static void requireAwsSdkV2Provider() {
+        LOGGER.info("Using AWS SDK v2 to retrieve AWS credentials");
+        awsCredentialSupplier = new AwsSdkV2CredentialSupplier();
+    }
 
-        header.clear();
-        header.put("X-aws-ec2-metadata-token", token);
-        String role = getHttpContents("GET", endpoint + path, header);
-        return getHttpContents("GET", endpoint + path + role, header);
+    public static AwsCredential obtainFromEnvironment() {
+        return awsCredentialSupplier.get();
     }
 
     private AwsCredentialHelper() {

driver-core/src/main/com/mongodb/internal/authentication/AwsSdkV1CredentialSupplier.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2008-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.mongodb.internal.authentication;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.mongodb.AwsCredential;
+
+import java.util.function.Supplier;
+
+public final class AwsSdkV1CredentialSupplier implements Supplier<AwsCredential> {
+
+    private final AWSCredentialsProvider provider = DefaultAWSCredentialsProviderChain.getInstance();
+
+    @Override
+    public AwsCredential get() {
+        AWSCredentials credentials = provider.getCredentials();
+        if (credentials instanceof AWSSessionCredentials) {
+            AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials;
+            return new AwsCredential(sessionCredentials.getAWSAccessKeyId(), sessionCredentials.getAWSSecretKey(),
+                    sessionCredentials.getSessionToken());
+        } else {
+            return new AwsCredential(credentials.getAWSAccessKeyId(), credentials.getAWSSecretKey(), null);
+        }
+    }
+}

driver-core/src/main/com/mongodb/internal/authentication/AwsSdkV2CredentialSupplier.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2008-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.mongodb.internal.authentication;
+
+import com.mongodb.AwsCredential;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+
+import java.util.function.Supplier;
+
+public final class AwsSdkV2CredentialSupplier implements Supplier<AwsCredential> {
+
+    private final AwsCredentialsProvider provider = DefaultCredentialsProvider.create();
+
+    @Override
+    public AwsCredential get() {
+        AwsCredentials credentials = provider.resolveCredentials();
+        if (credentials instanceof AwsSessionCredentials) {
+            AwsSessionCredentials sessionCredentials = (AwsSessionCredentials) credentials;
+            return new AwsCredential(sessionCredentials.accessKeyId(), sessionCredentials.secretAccessKey(),
+                    sessionCredentials.sessionToken());
+        } else {
+            return new AwsCredential(credentials.accessKeyId(), credentials.secretAccessKey(), null);
+        }
+    }
+}