diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 302be327499..074682d906f 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -1484,6 +1484,23 @@ tasks: - func: "run perf tests" - func: "send dashboard data" + - name: "test-aws-lambda-deployed" + commands: + - command: ec2.assume_role + params: + role_arn: ${LAMBDA_AWS_ROLE_ARN} + duration_seconds: 3600 + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - .evergreen/run-deployed-lambda-aws-tests.sh + env: + TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/driver-lambda/ + AWS_REGION: us-east-1 + - name: "mmapv1-storage-test" commands: - func: "bootstrap mongo-orchestration" @@ -1891,6 +1908,32 @@ task_groups: $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh tasks: - testazurekms-task + - name: test_atlas_task_group + setup_group: + - func: fetch source + - func: prepare resources + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: subprocess.exec + params: + working_dir: src + binary: bash + add_expansions_to_env: true + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - test-aws-lambda-deployed buildvariants: @@ -2023,6 +2066,12 @@ buildvariants: tasks: - name: "perf" +- name: rhel8-test-atlas + display_name: Atlas Cluster Tests + run_on: rhel80-large + tasks: + - test_atlas_task_group + - name: plain-auth-test display_name: "PLAIN (LDAP) Auth test" run_on: rhel80-small diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh new file mode 100644 index 00000000000..42010aad659 --- /dev/null +++ b/.evergreen/run-deployed-lambda-aws-tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail + +RELATIVE_DIR_PATH="$(dirname "${BASH_SOURCE[0]:-$0}")" +. "${RELATIVE_DIR_PATH}/javaConfig.bash" + +# compiled outside of lambda workflow. Note "SkipBuild: True" in template.yaml +./gradlew -version +./gradlew --info driver-lambda:shadowJar + +. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh diff --git a/build.gradle b/build.gradle index 10add6b07ab..7e756d1430d 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ def coreProjects = subprojects - utilProjects def scalaProjects = subprojects.findAll { it.name.contains('scala') } def javaProjects = subprojects - scalaProjects def javaMainProjects = javaProjects - utilProjects -def javaCodeCheckedProjects = javaMainProjects.findAll { !['driver-benchmarks', 'driver-workload-executor'].contains(it.name) } +def javaCodeCheckedProjects = javaMainProjects.findAll { !['driver-benchmarks', 'driver-workload-executor', 'driver-lambda'].contains(it.name) } def javaAndScalaTestedProjects = javaCodeCheckedProjects + scalaProjects configure(coreProjects) { diff --git a/driver-lambda/build.gradle b/driver-lambda/build.gradle new file mode 100644 index 00000000000..76e293c9c1b --- /dev/null +++ b/driver-lambda/build.gradle @@ -0,0 +1,74 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +buildscript { + repositories { + maven { url "https://plugins.gradle.org/m2/" } + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:6.1.0' + } +} + +plugins { + id("application") +} + +apply plugin: 'application' +apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'java' + +compileJava { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +mainClassName = "com.mongodb.workload.WorkloadExecutor" + +sourceSets { + main { + java { + srcDir 'src/main' + } + resources { + srcDir 'src/resources' + } + } +} + +dependencies { + implementation project(':driver-sync') + implementation project(':bson') + + implementation('com.amazonaws:aws-lambda-java-core:1.2.2') + implementation('com.amazonaws:aws-lambda-java-events:3.11.1') +} + + +javadoc { + enabled = false +} + +jar { + manifest { + attributes "Main-Class": "com.mongodb.lambdatest.LambdaTestApp" + } +} + +shadowJar { + archiveBaseName.set('lambdatest') + archiveVersion.set('') +} diff --git a/driver-lambda/samconfig.toml b/driver-lambda/samconfig.toml new file mode 100644 index 00000000000..332bf47b0b0 --- /dev/null +++ b/driver-lambda/samconfig.toml @@ -0,0 +1,31 @@ +# More information about the configuration file can be found here: +# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html +version = 0.1 + +[default] +[default.global.parameters] +stack_name = "lambdatest" + +[default.build.parameters] +cached = true +parallel = false + +[default.validate.parameters] +lint = true + +[default.deploy.parameters] +capabilities = "CAPABILITY_IAM" +confirm_changeset = false # headless +resolve_s3 = true + +[default.package.parameters] +resolve_s3 = true + +[default.sync.parameters] +watch = true + +[default.local_start_api.parameters] +warm_containers = "EAGER" + +[default.local_start_lambda.parameters] +warm_containers = "EAGER" diff --git a/driver-lambda/src/main/com/mongodb/lambdatest/LambdaTestApp.java b/driver-lambda/src/main/com/mongodb/lambdatest/LambdaTestApp.java new file mode 100644 index 00000000000..c9a0d2ca990 --- /dev/null +++ b/driver-lambda/src/main/com/mongodb/lambdatest/LambdaTestApp.java @@ -0,0 +1,154 @@ +/* + * Copyright 2008-present MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.mongodb.lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.event.CommandFailedEvent; +import com.mongodb.event.CommandListener; +import com.mongodb.event.CommandSucceededEvent; +import com.mongodb.event.ConnectionClosedEvent; +import com.mongodb.event.ConnectionCreatedEvent; +import com.mongodb.event.ConnectionPoolListener; +import com.mongodb.event.ServerHeartbeatFailedEvent; +import com.mongodb.event.ServerHeartbeatSucceededEvent; +import com.mongodb.event.ServerMonitorListener; +import com.mongodb.lang.NonNull; +import org.bson.BsonDocument; +import org.bson.BsonInt64; +import org.bson.BsonString; +import org.bson.BsonValue; +import org.bson.Document; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +/** + * Test App for AWS lambda functions + */ +public class LambdaTestApp implements RequestHandler { + private final MongoClient mongoClient; + private long openConnections = 0; + private long totalHeartbeatCount = 0; + private long totalHeartbeatDurationMs = 0; + private long totalCommandCount = 0; + private long totalCommandDurationMs = 0; + + public LambdaTestApp() { + String connectionString = System.getenv("MONGODB_URI"); + + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(connectionString)) + .addCommandListener(new CommandListener() { + @Override + public void commandSucceeded(@NonNull final CommandSucceededEvent event) { + totalCommandCount++; + totalCommandDurationMs += event.getElapsedTime(MILLISECONDS); + } + @Override + public void commandFailed(@NonNull final CommandFailedEvent event) { + totalCommandCount++; + totalCommandDurationMs += event.getElapsedTime(MILLISECONDS); + } + }) + .applyToServerSettings(builder -> builder.addServerMonitorListener(new ServerMonitorListener() { + @Override + public void serverHeartbeatSucceeded(@NonNull final ServerHeartbeatSucceededEvent event) { + totalHeartbeatCount++; + totalHeartbeatDurationMs += event.getElapsedTime(MILLISECONDS); + } + @Override + public void serverHeartbeatFailed(@NonNull final ServerHeartbeatFailedEvent event) { + totalHeartbeatCount++; + totalHeartbeatDurationMs += event.getElapsedTime(MILLISECONDS); + } + })) + .applyToConnectionPoolSettings(builder -> builder.addConnectionPoolListener(new ConnectionPoolListener() { + @Override + public void connectionCreated(@NonNull final ConnectionCreatedEvent event) { + openConnections++; + } + @Override + public void connectionClosed(@NonNull final ConnectionClosedEvent event) { + openConnections--; + } + })) + .build(); + mongoClient = MongoClients.create(settings); + } + + public APIGatewayProxyResponseEvent handleRequest(final APIGatewayProxyRequestEvent input, final Context context) { + try { + MongoCollection collection = mongoClient + .getDatabase("lambdaTest") + .getCollection("test"); + BsonValue id = collection.insertOne(new Document("n", 1)).getInsertedId(); + collection.deleteOne(new Document("_id", id)); + + BsonDocument responseBody = getBsonDocument(); + + return templateResponse() + .withStatusCode(200) + .withBody(responseBody.toJson()); + + } catch (Throwable e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + BsonDocument responseBody = new BsonDocument() + .append("throwable", new BsonString(e.getMessage())) + .append("stacktrace", new BsonString(sw.toString())); + return templateResponse() + .withBody(responseBody.toJson()) + .withStatusCode(500); + } + } + + private BsonDocument getBsonDocument() { + BsonDocument responseBody = new BsonDocument() + .append("totalCommandDurationMs", new BsonInt64(totalCommandDurationMs)) + .append("totalCommandCount", new BsonInt64(totalCommandCount)) + .append("totalHeartbeatDurationMs", new BsonInt64(totalHeartbeatDurationMs)) + .append("totalHeartbeatCount", new BsonInt64(totalHeartbeatCount)) + .append("openConnections", new BsonInt64(openConnections)); + + totalCommandDurationMs = 0; + totalCommandCount = 0; + totalHeartbeatCount = 0; + totalHeartbeatDurationMs = 0; + + return responseBody; + } + + private APIGatewayProxyResponseEvent templateResponse() { + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/json"); + headers.put("X-Custom-Header", "application/json"); + return new APIGatewayProxyResponseEvent() + .withHeaders(headers); + } +} diff --git a/driver-lambda/template.yaml b/driver-lambda/template.yaml new file mode 100644 index 00000000000..7a53bb20272 --- /dev/null +++ b/driver-lambda/template.yaml @@ -0,0 +1,69 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + Java driver lambda function test +Parameters: + MongoDbUri: + Type: String + Description: The MongoDB connection string. + +Globals: + Function: + Timeout: 30 + MemorySize: 128 + Tracing: Active + Api: + TracingEnabled: false + +Resources: + MongoDBFunction: + Type: AWS::Serverless::Function + Metadata: + SkipBuild: True + Properties: + CodeUri: build/libs/lambdatest-all.jar + Handler: com.mongodb.lambdatest.LambdaTestApp::handleRequest + Runtime: java11 + Environment: + Variables: + MONGODB_URI: !Ref MongoDbUri + JAVA_TOOL_OPTIONS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1 + Architectures: + - x86_64 + MemorySize: 512 + Events: + LambdaTest: + Type: Api + Properties: + Path: /mongodb + Method: get + ApplicationResourceGroup: + Type: AWS::ResourceGroups::Group + Properties: + Name: + Fn::Join: + - '' + - - ApplicationInsights-SAM- + - Ref: AWS::StackName + ResourceQuery: + Type: CLOUDFORMATION_STACK_1_0 + ApplicationInsightsMonitoring: + Type: AWS::ApplicationInsights::Application + Properties: + ResourceGroupName: + Fn::Join: + - '' + - - ApplicationInsights-SAM- + - Ref: AWS::StackName + AutoConfigurationEnabled: true + DependsOn: ApplicationResourceGroup +Outputs: + LambdaTestApi: + Description: API Gateway endpoint URL for Prod stage for Lambda Test function + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/mongodb/" + MongoDBFunction: + Description: Lambda Test Lambda Function ARN + Value: !GetAtt MongoDBFunction.Arn + MongoDBFunctionIamRole: + Description: Implicit IAM Role created for Lambda Test function + Value: !GetAtt MongoDBFunctionRole.Arn diff --git a/gradle/javadoc.gradle b/gradle/javadoc.gradle index 890207a15b5..469ca31253d 100644 --- a/gradle/javadoc.gradle +++ b/gradle/javadoc.gradle @@ -17,7 +17,7 @@ import static org.gradle.util.CollectionUtils.single */ -def projectsThatDoNotPublishJavaDocs = project(":util").allprojects + project(":driver-benchmarks") + project("driver-workload-executor") +def projectsThatDoNotPublishJavaDocs = project(":util").allprojects + project(":driver-benchmarks") + project("driver-workload-executor") + project("driver-lambda") def javaMainProjects = subprojects - projectsThatDoNotPublishJavaDocs task docs { diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 1c012a63a5a..561e76957a5 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -69,7 +69,7 @@ ext { } } -def projectsNotPublishedToMaven = project(":util").allprojects + project(":driver-benchmarks") + project("driver-workload-executor") +def projectsNotPublishedToMaven = project(":util").allprojects + project(":driver-benchmarks") + project("driver-workload-executor") + project("driver-lambda") def publishedProjects = subprojects - projectsNotPublishedToMaven def scalaProjects = publishedProjects.findAll { it.name.contains('scala') } def javaProjects = publishedProjects - scalaProjects diff --git a/settings.gradle b/settings.gradle index 9691c80cc5c..22ac7c67bfd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ include ':bson' include ':bson-record-codec' include ':driver-benchmarks' include ':driver-workload-executor' +include ':driver-lambda' include ':driver-core' include ':driver-legacy' include ':driver-sync'