Skip to content

Use AWS SDK v2/v1 if available for AWS credential fetching #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions .evergreen/.evg.yml
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,9 @@ functions:
PASS=$(urlencode ${iam_auth_ecs_secret_access_key})
MONGODB_URI="mongodb://$USER:$PASS@localhost"
EOF
JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} .evergreen/run-mongodb-aws-test.sh
JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \
AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
.evergreen/run-mongodb-aws-test.sh

"run aws auth test with assume role credentials":
- command: shell.exec
Expand Down Expand Up @@ -400,7 +402,9 @@ functions:
SESSION_TOKEN=$(urlencode $SESSION_TOKEN)
MONGODB_URI="mongodb://$USER:$PASS@localhost"
EOF
JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} DRIVERS_TOOLS=${DRIVERS_TOOLS} .evergreen/run-mongodb-aws-test.sh
JAVA_VERSION=${JAVA_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} DRIVERS_TOOLS=${DRIVERS_TOOLS} \
AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} \
.evergreen/run-mongodb-aws-test.sh

"run aws auth test with aws EC2 credentials":
- command: shell.exec
Expand All @@ -420,7 +424,7 @@ functions:
${PREPARE_SHELL}
# Write an empty prepare_mongodb_aws so no auth environment variables are set.
echo "" > "${PROJECT_DIRECTORY}/prepare_mongodb_aws.sh"
JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh

"run aws auth test with aws credentials as environment variables":
- command: shell.exec
Expand Down Expand Up @@ -449,7 +453,7 @@ functions:
working_dir: "src"
script: |
${PREPARE_SHELL}
JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh

"run aws auth test with aws credentials and session token as environment variables":
- command: shell.exec
Expand Down Expand Up @@ -479,7 +483,7 @@ functions:
working_dir: "src"
script: |
${PREPARE_SHELL}
JAVA_VERSION=${JAVA_VERSION} .evergreen/run-mongodb-aws-test.sh
JAVA_VERSION=${JAVA_VERSION} AWS_CREDENTIAL_PROVIDER=${AWS_CREDENTIAL_PROVIDER} .evergreen/run-mongodb-aws-test.sh

"run aws ECS auth test":
- command: shell.exec
Expand Down Expand Up @@ -1637,6 +1641,22 @@ axes:
variables:
LOGIN_CONTEXT_NAME: "com.sun.security.jgss.initiate"

- id: aws-credential-provider
display_name: AWS Credential Provider
values:
- id: aws_sdk_v2
display_name: AWS SDK V2
variables:
AWS_CREDENTIAL_PROVIDER: "awsSdkV2"
- id: aws_sdk_v1
display_name: AWS SDK V1
variables:
AWS_CREDENTIAL_PROVIDER: "awsSdkV1"
- id: built_in
display_name: Built-In
variables:
AWS_CREDENTIAL_PROVIDER: "builtIn"

task_groups:
- name: testgcpkms_task_group
setup_group_can_fail_task: true
Expand Down Expand Up @@ -1813,15 +1833,22 @@ buildvariants:
- name: "plain-auth-test"

- matrix_name: "aws-auth-test"
matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu" }
display_name: "MONGODB-AWS Auth test ${version} ${jdk}"
matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu",
aws-credential-provider: "*" }
display_name: "MONGODB-AWS Auth test ${version} ${jdk} ${aws-credential-provider}"
run_on: ubuntu1804-test
tasks:
- name: "aws-auth-test-with-regular-aws-credentials"
- name: "aws-auth-test-with-assume-role-credentials"
- name: "aws-auth-test-with-aws-credentials-as-environment-variables"
- name: "aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables"
- name: "aws-auth-test-with-aws-EC2-credentials"

- matrix_name: "aws-ecs-auth-test"
matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17"], version: ["4.4", "5.0", "6.0", "latest"], os: "ubuntu" }
display_name: "MONGODB-AWS ECS Auth test ${version} ${jdk}"
run_on: ubuntu1804-test
tasks:
- name: "aws-ECS-auth-test"

- matrix_name: "accept-api-version-2-test"
Expand Down
27 changes: 26 additions & 1 deletion .evergreen/run-mongodb-aws-ecs-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,30 @@ RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE:-$0}")"
./gradlew -version

echo "Running tests..."
./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --debug --info driver-core:test --tests AwsAuthenticationSpecification
./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=awsSdkV2 --stacktrace --debug --info \
driver-core:test --tests AwsAuthenticationSpecification
first=$?
echo $first

./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=awsSdkV1 --stacktrace --debug --info \
driver-core:test --tests AwsAuthenticationSpecification
second=$?
echo $second

./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} -Dorg.mongodb.test.aws.credential.provider=builtIn --stacktrace --debug --info \
driver-core:test --tests AwsAuthenticationSpecification
third=$?
echo $third

if [ $first -ne 0 ]; then
exit $first
elif [ $second -ne 0 ]; then
exit $second
elif [ $third -ne 0 ]; then
exit $third
else
exit 0
fi


cd -
11 changes: 6 additions & 5 deletions .evergreen/run-mongodb-aws-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ set -o xtrace
set -o errexit # Exit the script with error if any of the commands fail

# Supported/used environment variables:
# JDK Set the version of java to be used. Java versions can be set from the java toolchain /opt/java
# "jdk5", "jdk6", "jdk7", "jdk8", "jdk9", "jdk11"

# JDK Set the version of java to be used. Java versions can be set from the java toolchain /opt/java
# "jdk5", "jdk6", "jdk7", "jdk8", "jdk9", "jdk11"
# AWS_CREDENTIAL_PROVIDER "builtIn", 'awsSdkV1', 'awsSdkV2'
############################################
# Main Program #
############################################
Expand Down Expand Up @@ -37,5 +37,6 @@ echo "Running tests with Java ${JAVA_VERSION}"

# As this script may be executed multiple times in a single task, with different values for MONGODB_URI, it's necessary
# to run cleanTest to ensure that the test actually executes each run
./gradlew -PjavaVersion=${JAVA_VERSION} -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --debug --info --no-build-cache \
driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification
./gradlew -PjavaVersion="${JAVA_VERSION}" -Dorg.mongodb.test.uri="${MONGODB_URI}" \
-Dorg.mongodb.test.aws.credential.provider="${AWS_CREDENTIAL_PROVIDER}" \
--stacktrace --debug --info --no-build-cache driver-core:cleanTest driver-core:test --tests AwsAuthenticationSpecification
2 changes: 2 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ ext {
nettyVersion = '4.1.79.Final'
snappyVersion = '1.1.8.4'
zstdVersion = '1.5.2-3'
awsSdkV2Version = '2.17.293'
awsSdkV1Version = '1.12.291'
mongoCryptVersion = '1.6.0-alpha0'
projectReactorVersion = '2020.0.22'
junitBomVersion = '5.8.1'
Expand Down
6 changes: 6 additions & 0 deletions driver-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ dependencies {
api "io.netty:netty-buffer", optional
api "io.netty:netty-transport", optional
api "io.netty:netty-handler", optional

// Optionally depend on both AWS SDK v2 and v1. The driver will use v2 is present, v1 if present, or built-in functionality if
// neither are present
implementation "software.amazon.awssdk:auth:$awsSdkV2Version", optional
implementation "com.amazonaws:aws-java-sdk-core:$awsSdkV1Version", optional

implementation "org.xerial.snappy:snappy-java:$snappyVersion", optional
implementation "com.github.luben:zstd-jni:$zstdVersion", optional
implementation "org.mongodb:mongodb-crypt:$mongoCryptVersion", optional
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,61 +17,82 @@
package com.mongodb.internal.authentication;

import com.mongodb.AwsCredential;
import org.bson.BsonDocument;
import com.mongodb.diagnostics.logging.Logger;
import com.mongodb.diagnostics.logging.Loggers;
import com.mongodb.internal.VisibleForTesting;

import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;

import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;
import static com.mongodb.internal.VisibleForTesting.AccessModifier.PRIVATE;

/**
* Utility class for working with AWS authentication.
*
* <p>This class should not be considered a part of the public API.</p>
*/
public final class AwsCredentialHelper {
public static final Logger LOGGER = Loggers.getLogger("authenticator");

public static AwsCredential obtainFromEnvironment() {
if (System.getenv("AWS_ACCESS_KEY_ID") != null) {
return obtainFromEnvironmentVariables();
private static volatile Supplier<AwsCredential> awsCredentialSupplier;

static {
if (isClassAvailable("software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider")) {
awsCredentialSupplier = new AwsSdkV2CredentialSupplier();
LOGGER.info("Using DefaultCredentialsProvider from AWS SDK v2 to retrieve AWS credentials. This is the recommended "
+ "configuration");
} else if (isClassAvailable("com.amazonaws.auth.DefaultAWSCredentialsProviderChain")) {
awsCredentialSupplier = new AwsSdkV1CredentialSupplier();
LOGGER.info("Using DefaultAWSCredentialsProviderChain from AWS SDK v1 to retrieve AWS credentials. Consider adding a "
+ "dependency to AWS SDK v2's software.amazon.awssdk:auth artifact to get access to additional AWS authentication "
+ "functionality.");
} else {
return obtainFromEc2OrEcsResponse();
awsCredentialSupplier = new BuiltInAwsCredentialSupplier();
LOGGER.info("Using built-in driver implementation to retrieve AWS credentials. Consider adding a dependency to AWS SDK "
+ "v2's software.amazon.awssdk:auth artifact to get access to additional AWS authentication functionality.");
}
}

private static AwsCredential obtainFromEnvironmentVariables() {
return new AwsCredential(
System.getenv("AWS_ACCESS_KEY_ID"),
System.getenv("AWS_SECRET_ACCESS_KEY"),
System.getenv("AWS_SESSION_TOKEN"));
private static boolean isClassAvailable(final String className) {
try {
Class.forName(className);
return true;
} catch (ClassNotFoundException e) {
return false;
}
}

private static AwsCredential obtainFromEc2OrEcsResponse() {
String path = System.getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
BsonDocument ec2OrEcsResponse = path == null ? BsonDocument.parse(getEc2Response()) : BsonDocument.parse(getEcsResponse(path));

return new AwsCredential(
ec2OrEcsResponse.getString("AccessKeyId").getValue(),
ec2OrEcsResponse.getString("SecretAccessKey").getValue(),
ec2OrEcsResponse.getString("Token").getValue());
/**
* This method is visible to allow tests to require the built-in provider rather than rely on the fixed checks for classes on the
* classpath. It allows us to easily write tests of the built-in implementation without resorting to runtime classpath shenanigans.
*/
@VisibleForTesting(otherwise = PRIVATE)
public static void requireBuiltInProvider() {
LOGGER.info("Using built-in driver implementation to retrieve AWS credentials");
awsCredentialSupplier = new BuiltInAwsCredentialSupplier();
}

private static String getEcsResponse(final String path) {
return getHttpContents("GET", "http://169.254.170.2" + path, null);
/**
* This method is visible to allow tests to require the AWS SDK v1 provider rather than rely on the fixed checks for classes on the
* classpath. It allows us to easily write tests of the AWS SDK v1 implementation without resorting to runtime classpath shenanigans.
*/
@VisibleForTesting(otherwise = PRIVATE)
public static void requireAwsSdkV1Provider() {
LOGGER.info("Using AWS SDK v1 to retrieve AWS credentials");
awsCredentialSupplier = new AwsSdkV1CredentialSupplier();
}

private static String getEc2Response() {
final String endpoint = "http://169.254.169.254";
final String path = "/latest/meta-data/iam/security-credentials/";

Map<String, String> header = new HashMap<>();
header.put("X-aws-ec2-metadata-token-ttl-seconds", "30");
String token = getHttpContents("PUT", endpoint + "/latest/api/token", header);
/**
* This method is visible to allow tests to require the AWS SDK v2 provider rather than rely on the fixed checks for classes on the
* classpath. It allows us to easily write tests of the AWS SDK v2 implementation without resorting to runtime classpath shenanigans.
*/
@VisibleForTesting(otherwise = PRIVATE)
public static void requireAwsSdkV2Provider() {
LOGGER.info("Using AWS SDK v2 to retrieve AWS credentials");
awsCredentialSupplier = new AwsSdkV2CredentialSupplier();
}

header.clear();
header.put("X-aws-ec2-metadata-token", token);
String role = getHttpContents("GET", endpoint + path, header);
return getHttpContents("GET", endpoint + path + role, header);
public static AwsCredential obtainFromEnvironment() {
return awsCredentialSupplier.get();
}

private AwsCredentialHelper() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.mongodb.internal.authentication;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.mongodb.AwsCredential;

import java.util.function.Supplier;

public final class AwsSdkV1CredentialSupplier implements Supplier<AwsCredential> {

private final AWSCredentialsProvider provider = DefaultAWSCredentialsProviderChain.getInstance();

@Override
public AwsCredential get() {
AWSCredentials credentials = provider.getCredentials();
if (credentials instanceof AWSSessionCredentials) {
AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials;
return new AwsCredential(sessionCredentials.getAWSAccessKeyId(), sessionCredentials.getAWSSecretKey(),
sessionCredentials.getSessionToken());
} else {
return new AwsCredential(credentials.getAWSAccessKeyId(), credentials.getAWSSecretKey(), null);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.mongodb.internal.authentication;

import com.mongodb.AwsCredential;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;

import java.util.function.Supplier;

public final class AwsSdkV2CredentialSupplier implements Supplier<AwsCredential> {

private final AwsCredentialsProvider provider = DefaultCredentialsProvider.create();

@Override
public AwsCredential get() {
AwsCredentials credentials = provider.resolveCredentials();
if (credentials instanceof AwsSessionCredentials) {
AwsSessionCredentials sessionCredentials = (AwsSessionCredentials) credentials;
return new AwsCredential(sessionCredentials.accessKeyId(), sessionCredentials.secretAccessKey(),
sessionCredentials.sessionToken());
} else {
return new AwsCredential(credentials.accessKeyId(), credentials.secretAccessKey(), null);
}
}
}
Loading