Skip to content

Support attached service accounts for GCP KMS #987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 16, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions .evergreen/.evg.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1407,6 +1407,53 @@ tasks:
SSL: "nossl"
- func: run-csfle-aws-from-environment-test

- name: "testgcpkms-task"
commands:
- command: shell.exec
type: setup
params:
working_dir: "src"
shell: "bash"
script: |
${PREPARE_SHELL}
echo "Copying files ... begin"
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
tar czf /tmp/mongo-java-driver.tgz .
GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
echo "Untarring file ... end"

- command: shell.exec
type: test
params:
working_dir: "src"
shell: "bash"
script: |
${PREPARE_SHELL}
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh

- name: "testgcpkms-fail-task"
# testgcpkms-fail-task runs in a non-GCE environment.
# It is expected to fail to obtain GCE credentials.
commands:
- command: shell.exec
type: test
params:
working_dir: "src"
shell: "bash"
script: |
${PREPARE_SHELL}
MONGODB_URI='mongodb://localhost:27017' SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh

axes:
- id: version
display_name: MongoDB Version
Expand Down Expand Up @@ -1604,6 +1651,44 @@ axes:
variables:
LOGIN_CONTEXT_NAME: "com.sun.security.jgss.initiate"

task_groups:
- name: testgcpkms_task_group
setup_group_can_fail_task: true
setup_group_timeout_secs: 1800 # 30 minutes
setup_group:
- func: fetch source
- func: prepare resources
- func: fix absolute paths
- func: make files executable
- command: shell.exec
params:
shell: "bash"
script: |
${PREPARE_SHELL}
echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
export GCPKMS_MACHINETYPE="e2-standard-4"
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
# Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions.
- command: expansions.update
params:
file: testgcpkms-expansions.yml
teardown_group:
- command: shell.exec
params:
shell: "bash"
script: |
${PREPARE_SHELL}
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
tasks:
- testgcpkms-task

buildvariants:

# Test packaging and other release related routines
Expand Down Expand Up @@ -1837,3 +1922,12 @@ buildvariants:
display_name: "CSFLE AWS From Environment"
tasks:
- name: ".csfle-aws-from-environment"

- name: testgcpkms-variant
display_name: "GCP KMS"
run_on:
- debian11-small
tasks:
- name: testgcpkms_task_group
batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
- testgcpkms-fail-task
37 changes: 37 additions & 0 deletions .evergreen/run-mongodb-fle-gcp-auto.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

set -o xtrace
set -o errexit # Exit the script with error if any of the commands fail

# Supported/used environment variables:
# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server via MONGODB-AWS
# authentication mechanism
# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false"
############################################
# Main Program #
############################################

echo "Running GCP Credential Acquisition Test"

if ! which java ; then
echo "Installing java..."
sudo apt install openjdk-17-jdk -y
fi

./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-sync:test \
--tests ClientSideEncryptionOnDemandGcpCredentialsTest
first=$?
echo $first

./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-reactive-streams:test \
--tests ClientSideEncryptionOnDemandGcpCredentialsTest
second=$?
echo $second

if [ $first -ne 0 ]; then
exit $first
elif [ $second -ne 0 ]; then
exit $second
else
exit 0
fi
10 changes: 9 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ ext {
nettyTcnativeBoringsslVersion = '2.0.48.Final'
snappyVersion = '1.1.8.4'
zstdVersion = '1.5.0-4'
mongoCryptVersion = '1.5.2'
mongoCryptVersion = '1.6.0-SNAPSHOT'
projectReactorVersion = 'Californium-SR23'
junitBomVersion = '5.8.1'
gitVersion = getGitVersion()
Expand Down Expand Up @@ -81,6 +81,14 @@ configure(coreProjects) {
includeGroup "org.mongodb"
}
}

maven {
url 'https://oss.sonatype.org/content/repositories/snapshots'
content {
// Only include mongodb-crypt snapshots
includeGroup "org.mongodb"
}
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,18 @@
package com.mongodb.internal.authentication;

import com.mongodb.AwsCredential;
import com.mongodb.MongoInternalException;
import com.mongodb.lang.NonNull;
import org.bson.BsonDocument;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;

import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;

/**
* Utility class for working with AWS authentication.
*
* <p>This class should not be considered a part of the public API.</p>
*/
public final class AwsCredentialHelper {

public static AwsCredential obtainFromEnvironment() {
Expand Down Expand Up @@ -75,41 +74,6 @@ private static String getEc2Response() {
return getHttpContents("GET", endpoint + path + role, header);
}

@NonNull
private static String getHttpContents(final String method, final String endpoint, final Map<String, String> headers) {
StringBuilder content = new StringBuilder();
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) new URL(endpoint).openConnection();
conn.setRequestMethod(method);
conn.setReadTimeout(10000);
if (headers != null) {
for (Map.Entry<String, String> kvp : headers.entrySet()) {
conn.setRequestProperty(kvp.getKey(), kvp.getValue());
}
}

int status = conn.getResponseCode();
if (status != HttpURLConnection.HTTP_OK) {
throw new IOException(String.format("%d %s", status, conn.getResponseMessage()));
}

try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
content.append(inputLine);
}
}
} catch (IOException e) {
throw new MongoInternalException("Unexpected IOException", e);
} finally {
if (conn != null) {
conn.disconnect();
}
}
return content.toString();
}

private AwsCredentialHelper() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.mongodb.internal.authentication;

import com.mongodb.MongoClientException;
import org.bson.BsonDocument;

import java.util.HashMap;
import java.util.Map;

import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;

/**
* Utility class for working with GCP authentication.
*
* <p>This class should not be considered a part of the public API.</p>
*/
public final class GcpCredentialHelper {
public static BsonDocument obtainFromEnvironment() {
String alternateHost = System.getenv("GCE_METADATA_HOST");
String host = alternateHost == null ? "metadata.google.internal" : alternateHost;
String endpoint = "http://" + host;
String path = "/computeMetadata/v1/instance/service-accounts/default/token";

Map<String, String> header = new HashMap<>();
header.put("Metadata-Flavor", "Google");
String response = getHttpContents("GET", endpoint + path, header);
BsonDocument responseDocument = BsonDocument.parse(response);
if (responseDocument.containsKey("access_token")) {
return new BsonDocument("accessToken", responseDocument.get("access_token"));
} else {
throw new MongoClientException("access_token is missing from GCE metadata response. Full response is ''" + response);
}
}

private GcpCredentialHelper() {
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright 2008-present MongoDB, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.mongodb.internal.authentication;

import com.mongodb.MongoInternalException;
import com.mongodb.lang.NonNull;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Map;

/**
* Utility class for working with HTTP servers.
*
* <p>This class should not be considered a part of the public API.</p>
*/
public final class HttpHelper {

private HttpHelper() {
}

@NonNull
public static String getHttpContents(final String method, final String endpoint, final Map<String, String> headers) {
StringBuilder content = new StringBuilder();
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) new URL(endpoint).openConnection();
conn.setRequestMethod(method);
conn.setReadTimeout(10000);
if (headers != null) {
for (Map.Entry<String, String> kvp : headers.entrySet()) {
conn.setRequestProperty(kvp.getKey(), kvp.getValue());
}
}

int status = conn.getResponseCode();
if (status != HttpURLConnection.HTTP_OK) {
throw new IOException(String.format("%d %s", status, conn.getResponseMessage()));
}

try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
content.append(inputLine);
}
}
} catch (IOException e) {
throw new MongoInternalException("Unexpected IOException", e);
} finally {
if (conn != null) {
conn.disconnect();
}
}
return content.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.mongodb.MongoConfigurationException;
import com.mongodb.crypt.capi.MongoCryptOptions;
import com.mongodb.internal.authentication.AwsCredentialHelper;
import com.mongodb.internal.authentication.GcpCredentialHelper;
import com.mongodb.lang.Nullable;
import org.bson.BsonDocument;
import org.bson.BsonDocumentWrapper;
Expand Down Expand Up @@ -111,6 +112,9 @@ public static BsonDocument fetchCredentials(final Map<String, Map<String, Object
kmsProvidersDocument.put("aws", awsCredentialDocument);
}
}
if (kmsProvidersDocument.containsKey("gcp") && kmsProvidersDocument.get("gcp").asDocument().isEmpty()) {
kmsProvidersDocument.put("gcp", GcpCredentialHelper.obtainFromEnvironment());
}
return kmsProvidersDocument;
}

Expand Down
Loading