From 094150faea65a0656f60240146b201f986e8214d Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Mon, 8 Aug 2022 17:46:13 -0400 Subject: [PATCH 1/4] Support attached service accounts for GCP KMS JAVA-4685 --- .evergreen/.evg.yml | 94 +++++++++++++++++++ .evergreen/run-mongodb-fle-gcp-auto.sh | 37 ++++++++ build.gradle | 10 +- .../authentication/AwsCredentialHelper.java | 50 ++-------- .../authentication/GcpCredentialHelper.java | 52 ++++++++++ .../internal/authentication/HttpHelper.java | 74 +++++++++++++++ .../internal/capi/MongoCryptHelper.java | 4 + ...eEncryptionOnDemandGcpCredentialsTest.java | 33 +++++++ ...eEncryptionOnDemandGcpCredentialsTest.java | 81 ++++++++++++++++ ...eEncryptionOnDemandGcpCredentialsTest.java | 31 ++++++ 10 files changed, 422 insertions(+), 44 deletions(-) create mode 100755 .evergreen/run-mongodb-fle-gcp-auto.sh create mode 100644 driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java create mode 100644 driver-core/src/main/com/mongodb/internal/authentication/HttpHelper.java create mode 100644 driver-reactive-streams/src/test/functional/com/mongodb/reactivestreams/client/ClientSideEncryptionOnDemandGcpCredentialsTest.java create mode 100644 driver-sync/src/test/functional/com/mongodb/client/AbstractClientSideEncryptionOnDemandGcpCredentialsTest.java create mode 100644 driver-sync/src/test/functional/com/mongodb/client/ClientSideEncryptionOnDemandGcpCredentialsTest.java diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index e79b1539371..e9a0790a204 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -1407,6 +1407,53 @@ tasks: SSL: "nossl" - func: run-csfle-aws-from-environment-test + - name: "testgcpkms-task" + commands: + - command: shell.exec + type: setup + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + echo "Copying files ... begin" + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + tar czf /tmp/mongo-java-driver.tgz . + GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh + echo "Copying files ... end" + echo "Untarring file ... begin" + GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + echo "Untarring file ... end" + + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + + - name: "testgcpkms-fail-task" + # testgcpkms-fail-task runs in a non-GCE environment. + # It is expected to fail to obtain GCE credentials. + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI='mongodb://localhost:27017' SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh + axes: - id: version display_name: MongoDB Version @@ -1604,6 +1651,44 @@ axes: variables: LOGIN_CONTEXT_NAME: "com.sun.security.jgss.initiate" +task_groups: + - name: testgcpkms_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: prepare resources + - func: fix absolute paths + - func: make files executable + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json + export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json + export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS + export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}" + export GCPKMS_MACHINETYPE="e2-standard-4" + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh + # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. + - command: expansions.update + params: + file: testgcpkms-expansions.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + script: | + ${PREPARE_SHELL} + export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} + export GCPKMS_PROJECT=${GCPKMS_PROJECT} + export GCPKMS_ZONE=${GCPKMS_ZONE} + export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} + $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + tasks: + - testgcpkms-task + buildvariants: # Test packaging and other release related routines @@ -1837,3 +1922,12 @@ buildvariants: display_name: "CSFLE AWS From Environment" tasks: - name: ".csfle-aws-from-environment" + +- name: testgcpkms-variant + display_name: "GCP KMS" + run_on: + - debian11-small + tasks: + - name: testgcpkms_task_group + batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README + - testgcpkms-fail-task diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh new file mode 100755 index 00000000000..ff04f26861a --- /dev/null +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -o xtrace +set -o errexit # Exit the script with error if any of the commands fail + +# Supported/used environment variables: +# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server via MONGODB-AWS +# authentication mechanism +# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" +############################################ +# Main Program # +############################################ + +echo "Running GCP Credential Acquisition Test" + +if ! which java ; then + echo "Installing java..." + sudo apt install openjdk-17-jdk -y +fi + +./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-sync:test \ + --tests ClientSideEncryptionOnDemandGcpCredentialsTest +first=$? +echo $first + +./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-reactive-streams:test \ + --tests ClientSideEncryptionOnDemandGcpCredentialsTest +second=$? +echo $second + +if [ $first -ne 0 ]; then + exit $first +elif [ $second -ne 0 ]; then + exit $second +else + exit 0 +fi diff --git a/build.gradle b/build.gradle index ec5a0b15f5f..9c0fe3e9420 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ ext { nettyTcnativeBoringsslVersion = '2.0.48.Final' snappyVersion = '1.1.8.4' zstdVersion = '1.5.0-4' - mongoCryptVersion = '1.5.2' + mongoCryptVersion = '1.6.0-SNAPSHOT' projectReactorVersion = 'Californium-SR23' junitBomVersion = '5.8.1' gitVersion = getGitVersion() @@ -81,6 +81,14 @@ configure(coreProjects) { includeGroup "org.mongodb" } } + + maven { + url 'https://oss.sonatype.org/content/repositories/snapshots' + content { + // Only include mongodb-crypt snapshots + includeGroup "org.mongodb" + } + } } } diff --git a/driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java b/driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java index 3ba1d5a408b..28ff12ae09c 100644 --- a/driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java +++ b/driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java @@ -17,19 +17,18 @@ package com.mongodb.internal.authentication; import com.mongodb.AwsCredential; -import com.mongodb.MongoInternalException; -import com.mongodb.lang.NonNull; import org.bson.BsonDocument; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import static com.mongodb.internal.authentication.HttpHelper.getHttpContents; + +/** + * Utility class for working with AWS authentication. + * + *

This class should not be considered a part of the public API.

+ */ public final class AwsCredentialHelper { public static AwsCredential obtainFromEnvironment() { @@ -75,41 +74,6 @@ private static String getEc2Response() { return getHttpContents("GET", endpoint + path + role, header); } - @NonNull - private static String getHttpContents(final String method, final String endpoint, final Map headers) { - StringBuilder content = new StringBuilder(); - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(endpoint).openConnection(); - conn.setRequestMethod(method); - conn.setReadTimeout(10000); - if (headers != null) { - for (Map.Entry kvp : headers.entrySet()) { - conn.setRequestProperty(kvp.getKey(), kvp.getValue()); - } - } - - int status = conn.getResponseCode(); - if (status != HttpURLConnection.HTTP_OK) { - throw new IOException(String.format("%d %s", status, conn.getResponseMessage())); - } - - try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { - String inputLine; - while ((inputLine = in.readLine()) != null) { - content.append(inputLine); - } - } - } catch (IOException e) { - throw new MongoInternalException("Unexpected IOException", e); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - return content.toString(); - } - private AwsCredentialHelper() { } } diff --git a/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java b/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java new file mode 100644 index 00000000000..dbceecb9af4 --- /dev/null +++ b/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java @@ -0,0 +1,52 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.internal.authentication; + +import com.mongodb.MongoClientException; +import org.bson.BsonDocument; + +import java.util.HashMap; +import java.util.Map; + +import static com.mongodb.internal.authentication.HttpHelper.getHttpContents; + +/** + * Utility class for working with GCP authentication. + * + *

This class should not be considered a part of the public API.

+ */ +public final class GcpCredentialHelper { + public static BsonDocument obtainFromEnvironment() { + String alternateHost = System.getenv("GCE_METADATA_HOST"); + String host = alternateHost == null ? "metadata.google.internal" : alternateHost; + String endpoint = "http://" + host; + String path = "/computeMetadata/v1/instance/service-accounts/default/token"; + + Map header = new HashMap<>(); + header.put("Metadata-Flavor", "Google"); + String response = getHttpContents("GET", endpoint + path, header); + BsonDocument responseDocument = BsonDocument.parse(response); + if (responseDocument.containsKey("access_token")) { + return new BsonDocument("accessToken", responseDocument.get("access_token")); + } else { + throw new MongoClientException("access_token is missing from GCE metadata response. Full response is ''" + response); + } + } + + private GcpCredentialHelper() { + } +} diff --git a/driver-core/src/main/com/mongodb/internal/authentication/HttpHelper.java b/driver-core/src/main/com/mongodb/internal/authentication/HttpHelper.java new file mode 100644 index 00000000000..f98831f79ca --- /dev/null +++ b/driver-core/src/main/com/mongodb/internal/authentication/HttpHelper.java @@ -0,0 +1,74 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.internal.authentication; + +import com.mongodb.MongoInternalException; +import com.mongodb.lang.NonNull; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * Utility class for working with HTTP servers. + * + *

This class should not be considered a part of the public API.

+ */ +public final class HttpHelper { + + private HttpHelper() { + } + + @NonNull + public static String getHttpContents(final String method, final String endpoint, final Map headers) { + StringBuilder content = new StringBuilder(); + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(endpoint).openConnection(); + conn.setRequestMethod(method); + conn.setReadTimeout(10000); + if (headers != null) { + for (Map.Entry kvp : headers.entrySet()) { + conn.setRequestProperty(kvp.getKey(), kvp.getValue()); + } + } + + int status = conn.getResponseCode(); + if (status != HttpURLConnection.HTTP_OK) { + throw new IOException(String.format("%d %s", status, conn.getResponseMessage())); + } + + try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String inputLine; + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + } + } catch (IOException e) { + throw new MongoInternalException("Unexpected IOException", e); + } finally { + if (conn != null) { + conn.disconnect(); + } + } + return content.toString(); + } +} diff --git a/driver-core/src/main/com/mongodb/internal/capi/MongoCryptHelper.java b/driver-core/src/main/com/mongodb/internal/capi/MongoCryptHelper.java index f63ab898ff9..dc98c741bdf 100644 --- a/driver-core/src/main/com/mongodb/internal/capi/MongoCryptHelper.java +++ b/driver-core/src/main/com/mongodb/internal/capi/MongoCryptHelper.java @@ -25,6 +25,7 @@ import com.mongodb.MongoConfigurationException; import com.mongodb.crypt.capi.MongoCryptOptions; import com.mongodb.internal.authentication.AwsCredentialHelper; +import com.mongodb.internal.authentication.GcpCredentialHelper; import com.mongodb.lang.Nullable; import org.bson.BsonDocument; import org.bson.BsonDocumentWrapper; @@ -111,6 +112,9 @@ public static BsonDocument fetchCredentials(final Map> kmsProviders = new HashMap<>(); + kmsProviders.put("gcp", new HashMap<>()); + clientEncryption = getClientEncryption(ClientEncryptionSettings.builder() + .keyVaultMongoClientSettings(Fixture.getMongoClientSettings()) + .keyVaultNamespace("keyvault.datakeys") + .kmsProviders(kmsProviders) + .build()); + } + + @AfterEach + public void tearDown() { + clientEncryption.close(); + } + + @Test + @EnabledIfSystemProperty(named = "org.mongodb.test.gcp.success", matches = "true") + public void testSuccess() { + clientEncryption.createDataKey("gcp", getDataKeyOptions()); + } + + @Test + @EnabledIfSystemProperty(named = "org.mongodb.test.gcp.success", matches = "false") + public void testFailure() { + MongoInternalException thrown = assertThrows( + MongoInternalException.class, + () -> clientEncryption.createDataKey("gcp", getDataKeyOptions())); + assertTrue(thrown.getCause() instanceof IOException); + } + + @NotNull + private DataKeyOptions getDataKeyOptions() { + return new DataKeyOptions().masterKey(BsonDocument.parse( + "{projectId: \"devprod-drivers\", location: \"global\", keyRing: \"key-ring-csfle\", keyName: \"key-name-csfle\"}")); + } +} diff --git a/driver-sync/src/test/functional/com/mongodb/client/ClientSideEncryptionOnDemandGcpCredentialsTest.java b/driver-sync/src/test/functional/com/mongodb/client/ClientSideEncryptionOnDemandGcpCredentialsTest.java new file mode 100644 index 00000000000..f971b934309 --- /dev/null +++ b/driver-sync/src/test/functional/com/mongodb/client/ClientSideEncryptionOnDemandGcpCredentialsTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.client; + +import com.mongodb.ClientEncryptionSettings; +import com.mongodb.client.vault.ClientEncryption; +import com.mongodb.client.vault.ClientEncryptions; +import com.mongodb.lang.NonNull; + +public class ClientSideEncryptionOnDemandGcpCredentialsTest extends AbstractClientSideEncryptionOnDemandGcpCredentialsTest { + + @NonNull + @Override + public ClientEncryption getClientEncryption(final ClientEncryptionSettings settings) { + return ClientEncryptions.create(settings); + } +} From 5d7c9069bc6cc09251dcead348e40b2164e1b182 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Wed, 10 Aug 2022 12:54:53 -0400 Subject: [PATCH 2/4] Remove GCE_METADATA_HOST env var support --- .../internal/authentication/GcpCredentialHelper.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java b/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java index dbceecb9af4..3c576552c73 100644 --- a/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java +++ b/driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java @@ -31,14 +31,11 @@ */ public final class GcpCredentialHelper { public static BsonDocument obtainFromEnvironment() { - String alternateHost = System.getenv("GCE_METADATA_HOST"); - String host = alternateHost == null ? "metadata.google.internal" : alternateHost; - String endpoint = "http://" + host; - String path = "/computeMetadata/v1/instance/service-accounts/default/token"; + String endpoint = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; Map header = new HashMap<>(); header.put("Metadata-Flavor", "Google"); - String response = getHttpContents("GET", endpoint + path, header); + String response = getHttpContents("GET", endpoint, header); BsonDocument responseDocument = BsonDocument.parse(response); if (responseDocument.containsKey("access_token")) { return new BsonDocument("accessToken", responseDocument.get("access_token")); From ded723af4af16aa331cb49ff251354f87d03fb85 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Thu, 11 Aug 2022 19:10:27 -0400 Subject: [PATCH 3/4] Update .evergreen/run-mongodb-fle-gcp-auto.sh Co-authored-by: Kevin Albertson --- .evergreen/run-mongodb-fle-gcp-auto.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/run-mongodb-fle-gcp-auto.sh b/.evergreen/run-mongodb-fle-gcp-auto.sh index ff04f26861a..27d2ac107ea 100755 --- a/.evergreen/run-mongodb-fle-gcp-auto.sh +++ b/.evergreen/run-mongodb-fle-gcp-auto.sh @@ -4,8 +4,7 @@ set -o xtrace set -o errexit # Exit the script with error if any of the commands fail # Supported/used environment variables: -# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server via MONGODB-AWS -# authentication mechanism +# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server # SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false" ############################################ # Main Program # From c9ee1084ed14184e3c90cd9620356f5d977ae201 Mon Sep 17 00:00:00 2001 From: Jeff Yemin Date: Tue, 16 Aug 2022 10:11:58 -0400 Subject: [PATCH 4/4] Update mongodb-crypt dependency --- build.gradle | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/build.gradle b/build.gradle index 9c0fe3e9420..253e8c3e1f7 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ ext { nettyTcnativeBoringsslVersion = '2.0.48.Final' snappyVersion = '1.1.8.4' zstdVersion = '1.5.0-4' - mongoCryptVersion = '1.6.0-SNAPSHOT' + mongoCryptVersion = '1.6.0-alpha0' projectReactorVersion = 'Californium-SR23' junitBomVersion = '5.8.1' gitVersion = getGitVersion() @@ -74,21 +74,13 @@ configure(coreProjects) { google() mavenCentral() - maven { - url 'https://oss.sonatype.org/content/repositories/staging' - content { - // Only include mongodb-crypt snapshots - includeGroup "org.mongodb" - } - } - - maven { - url 'https://oss.sonatype.org/content/repositories/snapshots' - content { - // Only include mongodb-crypt snapshots - includeGroup "org.mongodb" - } - } + // Uncomment this to test with a snapshot build of mongodb-crypt +// maven { +// url 'https://oss.sonatype.org/content/repositories/snapshots' +// content { +// includeGroup "org.mongodb" +// } +// } } }