Skip to content

Commit adc9c8c

Browse files
jyeminjyemin
authored
Support attached service accounts for GCP KMS (#987)
JAVA-4685
1 parent 6a23890 commit adc9c8c

File tree

10 files changed

+417
-51
lines changed

10 files changed

+417
-51
lines changed

‎.evergreen/.evg.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1407,6 +1407,53 @@ tasks:
14071407
SSL: "nossl"
14081408
- func: run-csfle-aws-from-environment-test
14091409

1410+
- name: "testgcpkms-task"
1411+
commands:
1412+
- command: shell.exec
1413+
type: setup
1414+
params:
1415+
working_dir: "src"
1416+
shell: "bash"
1417+
script: |
1418+
${PREPARE_SHELL}
1419+
echo "Copying files ... begin"
1420+
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
1421+
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
1422+
export GCPKMS_ZONE=${GCPKMS_ZONE}
1423+
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
1424+
tar czf /tmp/mongo-java-driver.tgz .
1425+
GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
1426+
echo "Copying files ... end"
1427+
echo "Untarring file ... begin"
1428+
GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
1429+
echo "Untarring file ... end"
1430+
1431+
- command: shell.exec
1432+
type: test
1433+
params:
1434+
working_dir: "src"
1435+
shell: "bash"
1436+
script: |
1437+
${PREPARE_SHELL}
1438+
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
1439+
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
1440+
export GCPKMS_ZONE=${GCPKMS_ZONE}
1441+
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
1442+
GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' SUCCESS=true ./.evergreen/run-mongodb-fle-gcp-auto.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
1443+
1444+
- name: "testgcpkms-fail-task"
1445+
# testgcpkms-fail-task runs in a non-GCE environment.
1446+
# It is expected to fail to obtain GCE credentials.
1447+
commands:
1448+
- command: shell.exec
1449+
type: test
1450+
params:
1451+
working_dir: "src"
1452+
shell: "bash"
1453+
script: |
1454+
${PREPARE_SHELL}
1455+
MONGODB_URI='mongodb://localhost:27017' SUCCESS=false ./.evergreen/run-mongodb-fle-gcp-auto.sh
1456+
14101457
axes:
14111458
- id: version
14121459
display_name: MongoDB Version
@@ -1604,6 +1651,44 @@ axes:
16041651
variables:
16051652
LOGIN_CONTEXT_NAME: "com.sun.security.jgss.initiate"
16061653

1654+
task_groups:
1655+
- name: testgcpkms_task_group
1656+
setup_group_can_fail_task: true
1657+
setup_group_timeout_secs: 1800 # 30 minutes
1658+
setup_group:
1659+
- func: fetch source
1660+
- func: prepare resources
1661+
- func: fix absolute paths
1662+
- func: make files executable
1663+
- command: shell.exec
1664+
params:
1665+
shell: "bash"
1666+
script: |
1667+
${PREPARE_SHELL}
1668+
echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
1669+
export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
1670+
export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
1671+
export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
1672+
export GCPKMS_MACHINETYPE="e2-standard-4"
1673+
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
1674+
# Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions.
1675+
- command: expansions.update
1676+
params:
1677+
file: testgcpkms-expansions.yml
1678+
teardown_group:
1679+
- command: shell.exec
1680+
params:
1681+
shell: "bash"
1682+
script: |
1683+
${PREPARE_SHELL}
1684+
export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
1685+
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
1686+
export GCPKMS_ZONE=${GCPKMS_ZONE}
1687+
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
1688+
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
1689+
tasks:
1690+
- testgcpkms-task
1691+
16071692
buildvariants:
16081693

16091694
# Test packaging and other release related routines
@@ -1837,3 +1922,12 @@ buildvariants:
18371922
display_name: "CSFLE AWS From Environment"
18381923
tasks:
18391924
- name: ".csfle-aws-from-environment"
1925+
1926+
- name: testgcpkms-variant
1927+
display_name: "GCP KMS"
1928+
run_on:
1929+
- debian11-small
1930+
tasks:
1931+
- name: testgcpkms_task_group
1932+
batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
1933+
- testgcpkms-fail-task

‎.evergreen/run-mongodb-fle-gcp-auto.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
set -o xtrace
4+
set -o errexit # Exit the script with error if any of the commands fail
5+
6+
# Supported/used environment variables:
7+
# MONGODB_URI Set the URI, including an optional username/password to use to connect to the server
8+
# SUCCESS Whether the authentication is expected to succeed or fail. One of "true" or "false"
9+
############################################
10+
# Main Program #
11+
############################################
12+
13+
echo "Running GCP Credential Acquisition Test"
14+
15+
if ! which java ; then
16+
echo "Installing java..."
17+
sudo apt install openjdk-17-jdk -y
18+
fi
19+
20+
./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-sync:test \
21+
--tests ClientSideEncryptionOnDemandGcpCredentialsTest
22+
first=$?
23+
echo $first
24+
25+
./gradlew -Dorg.mongodb.test.uri="${MONGODB_URI}" -Dorg.mongodb.test.gcp.success="${SUCCESS}" --stacktrace --debug --info driver-reactive-streams:test \
26+
--tests ClientSideEncryptionOnDemandGcpCredentialsTest
27+
second=$?
28+
echo $second
29+
30+
if [ $first -ne 0 ]; then
31+
exit $first
32+
elif [ $second -ne 0 ]; then
33+
exit $second
34+
else
35+
exit 0
36+
fi

‎build.gradle

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ ext {
4848
nettyTcnativeBoringsslVersion = '2.0.48.Final'
4949
snappyVersion = '1.1.8.4'
5050
zstdVersion = '1.5.0-4'
51-
mongoCryptVersion = '1.5.2'
51+
mongoCryptVersion = '1.6.0-alpha0'
5252
projectReactorVersion = 'Californium-SR23'
5353
junitBomVersion = '5.8.1'
5454
gitVersion = getGitVersion()
@@ -74,13 +74,13 @@ configure(coreProjects) {
7474
google()
7575
mavenCentral()
7676

77-
maven {
78-
url 'https://oss.sonatype.org/content/repositories/staging'
79-
content {
80-
// Only include mongodb-crypt snapshots
81-
includeGroup "org.mongodb"
82-
}
83-
}
77+
// Uncomment this to test with a snapshot build of mongodb-crypt
78+
// maven {
79+
// url 'https://oss.sonatype.org/content/repositories/snapshots'
80+
// content {
81+
// includeGroup "org.mongodb"
82+
// }
83+
// }
8484
}
8585
}
8686

‎driver-core/src/main/com/mongodb/internal/authentication/AwsCredentialHelper.java

Lines changed: 7 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,18 @@
1717
package com.mongodb.internal.authentication;
1818

1919
import com.mongodb.AwsCredential;
20-
import com.mongodb.MongoInternalException;
21-
import com.mongodb.lang.NonNull;
2220
import org.bson.BsonDocument;
2321

24-
import java.io.BufferedReader;
25-
import java.io.IOException;
26-
import java.io.InputStreamReader;
27-
import java.net.HttpURLConnection;
28-
import java.net.URL;
29-
import java.nio.charset.StandardCharsets;
3022
import java.util.HashMap;
3123
import java.util.Map;
3224

25+
import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;
26+
27+
/**
28+
* Utility class for working with AWS authentication.
29+
*
30+
* <p>This class should not be considered a part of the public API.</p>
31+
*/
3332
public final class AwsCredentialHelper {
3433

3534
public static AwsCredential obtainFromEnvironment() {
@@ -75,41 +74,6 @@ private static String getEc2Response() {
7574
return getHttpContents("GET", endpoint + path + role, header);
7675
}
7776

78-
@NonNull
79-
private static String getHttpContents(final String method, final String endpoint, final Map<String, String> headers) {
80-
StringBuilder content = new StringBuilder();
81-
HttpURLConnection conn = null;
82-
try {
83-
conn = (HttpURLConnection) new URL(endpoint).openConnection();
84-
conn.setRequestMethod(method);
85-
conn.setReadTimeout(10000);
86-
if (headers != null) {
87-
for (Map.Entry<String, String> kvp : headers.entrySet()) {
88-
conn.setRequestProperty(kvp.getKey(), kvp.getValue());
89-
}
90-
}
91-
92-
int status = conn.getResponseCode();
93-
if (status != HttpURLConnection.HTTP_OK) {
94-
throw new IOException(String.format("%d %s", status, conn.getResponseMessage()));
95-
}
96-
97-
try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
98-
String inputLine;
99-
while ((inputLine = in.readLine()) != null) {
100-
content.append(inputLine);
101-
}
102-
}
103-
} catch (IOException e) {
104-
throw new MongoInternalException("Unexpected IOException", e);
105-
} finally {
106-
if (conn != null) {
107-
conn.disconnect();
108-
}
109-
}
110-
return content.toString();
111-
}
112-
11377
private AwsCredentialHelper() {
11478
}
11579
}

‎driver-core/src/main/com/mongodb/internal/authentication/GcpCredentialHelper.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright 2008-present MongoDB, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.mongodb.internal.authentication;
18+
19+
import com.mongodb.MongoClientException;
20+
import org.bson.BsonDocument;
21+
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
25+
import static com.mongodb.internal.authentication.HttpHelper.getHttpContents;
26+
27+
/**
28+
* Utility class for working with GCP authentication.
29+
*
30+
* <p>This class should not be considered a part of the public API.</p>
31+
*/
32+
public final class GcpCredentialHelper {
33+
public static BsonDocument obtainFromEnvironment() {
34+
String endpoint = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token";
35+
36+
Map<String, String> header = new HashMap<>();
37+
header.put("Metadata-Flavor", "Google");
38+
String response = getHttpContents("GET", endpoint, header);
39+
BsonDocument responseDocument = BsonDocument.parse(response);
40+
if (responseDocument.containsKey("access_token")) {
41+
return new BsonDocument("accessToken", responseDocument.get("access_token"));
42+
} else {
43+
throw new MongoClientException("access_token is missing from GCE metadata response. Full response is ''" + response);
44+
}
45+
}
46+
47+
private GcpCredentialHelper() {
48+
}
49+
}

‎driver-core/src/main/com/mongodb/internal/authentication/HttpHelper.java

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright 2008-present MongoDB, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.mongodb.internal.authentication;
18+
19+
import com.mongodb.MongoInternalException;
20+
import com.mongodb.lang.NonNull;
21+
22+
import java.io.BufferedReader;
23+
import java.io.IOException;
24+
import java.io.InputStreamReader;
25+
import java.net.HttpURLConnection;
26+
import java.net.URL;
27+
import java.nio.charset.StandardCharsets;
28+
import java.util.Map;
29+
30+
/**
31+
* Utility class for working with HTTP servers.
32+
*
33+
* <p>This class should not be considered a part of the public API.</p>
34+
*/
35+
public final class HttpHelper {
36+
37+
private HttpHelper() {
38+
}
39+
40+
@NonNull
41+
public static String getHttpContents(final String method, final String endpoint, final Map<String, String> headers) {
42+
StringBuilder content = new StringBuilder();
43+
HttpURLConnection conn = null;
44+
try {
45+
conn = (HttpURLConnection) new URL(endpoint).openConnection();
46+
conn.setRequestMethod(method);
47+
conn.setReadTimeout(10000);
48+
if (headers != null) {
49+
for (Map.Entry<String, String> kvp : headers.entrySet()) {
50+
conn.setRequestProperty(kvp.getKey(), kvp.getValue());
51+
}
52+
}
53+
54+
int status = conn.getResponseCode();
55+
if (status != HttpURLConnection.HTTP_OK) {
56+
throw new IOException(String.format("%d %s", status, conn.getResponseMessage()));
57+
}
58+
59+
try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
60+
String inputLine;
61+
while ((inputLine = in.readLine()) != null) {
62+
content.append(inputLine);
63+
}
64+
}
65+
} catch (IOException e) {
66+
throw new MongoInternalException("Unexpected IOException", e);
67+
} finally {
68+
if (conn != null) {
69+
conn.disconnect();
70+
}
71+
}
72+
return content.toString();
73+
}
74+
}

‎driver-core/src/main/com/mongodb/internal/capi/MongoCryptHelper.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.mongodb.MongoConfigurationException;
2626
import com.mongodb.crypt.capi.MongoCryptOptions;
2727
import com.mongodb.internal.authentication.AwsCredentialHelper;
28+
import com.mongodb.internal.authentication.GcpCredentialHelper;
2829
import com.mongodb.lang.Nullable;
2930
import org.bson.BsonDocument;
3031
import org.bson.BsonDocumentWrapper;
@@ -111,6 +112,9 @@ public static BsonDocument fetchCredentials(final Map<String, Map<String, Object
111112
kmsProvidersDocument.put("aws", awsCredentialDocument);
112113
}
113114
}
115+
if (kmsProvidersDocument.containsKey("gcp") && kmsProvidersDocument.get("gcp").asDocument().isEmpty()) {
116+
kmsProvidersDocument.put("gcp", GcpCredentialHelper.obtainFromEnvironment());
117+
}
114118
return kmsProvidersDocument;
115119
}
116120

0 commit comments

Comments
 (0)