From 930ff3239a51f110e7c5c3718b3bfd04803fad62 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 08:44:56 -0600 Subject: [PATCH 01/21] refactor dockerfile --- .evergreen/config.yml | 1 + .evergreen/run-tests.sh | 1 - Dockerfile | 10 +++++++--- etc/docker_entry.sh | 7 +------ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 490cf91311..19380322ac 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1528,6 +1528,7 @@ tasks: - name: test-docker-runner tags: ["pullrequest"] commands: + - func: bootstrap-mongo-orchestration - func: run-docker-test - name: test-load-balancer-noauth-nossl diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index d4abde9b3b..4b558bcd7d 100644 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -18,7 +18,6 @@ fi export GOROOT="${GOROOT}" export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH" -export PATH="${MONGODB_BINARIES:-$DRIVERS_TOOLS/mongodb/bin}:$PATH" export PROJECT="${project}" export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64 diff --git a/Dockerfile b/Dockerfile index d49c7ca1a7..6201c5d9f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,15 +16,19 @@ COPY etc/install-libmongocrypt.sh /root/install-libmongocrypt.sh RUN cd /root && bash ./install-libmongocrypt.sh -# Inherit from the drivers-evergreen-tools image and copy in the files -# from the libmongocrypt build stage. -FROM drivers-evergreen-tools +# Copy in the files from the libmongocrypt build stage. +FROM ubuntu:20.04 # Install common deps. RUN export DEBIAN_FRONTEND=noninteractive && \ export TZ=Etc/UTC && \ apt-get -qq update && \ apt-get -qqy install --reinstall --no-install-recommends \ + git \ + ca-certificates \ + curl \ + wget \ + sudo \ tzdata \ ca-certificates \ pkg-config \ diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh index 51f31217f1..de619df73e 100644 --- a/etc/docker_entry.sh +++ b/etc/docker_entry.sh @@ -1,18 +1,13 @@ #!/usr/bin/env bash # -# Entry point for Dockerfile for launching a server and running a go test. +# Entry point for Dockerfile for launching running a go test. # set -eux -# Start the server. -bash /root/base-entrypoint.sh -source $DRIVERS_TOOLS/.evergreen/mo-expansion.sh - # Prep files. cd /src rm -f test.suite cp -r $HOME/install ./install -export PATH="$MONGODB_BINARIES:$PATH" # Run the test. bash ./.evergreen/run-tests.sh From e87b4256c4f494c1124467c1fb3d88f9cdb5562d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 11:08:33 -0600 Subject: [PATCH 02/21] add skip --- mongo/client_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongo/client_test.go b/mongo/client_test.go index d4aeaeacd3..e16a50fd67 100644 --- a/mongo/client_test.go +++ b/mongo/client_test.go @@ -318,6 +318,9 @@ func TestClient(t *testing.T) { }) }) t.Run("endSessions", func(t *testing.T) { + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("This test does not work properly in docker.") + } cs := integtest.ConnString(t) originalBatchSize := endSessionsBatchSize endSessionsBatchSize = 2 From e3138a9c0a8158866af57bb0820351b32916fff4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 11:20:42 -0600 Subject: [PATCH 03/21] fix skip condition --- mongo/client_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongo/client_test.go b/mongo/client_test.go index e16a50fd67..2aa3f5c8b2 100644 --- a/mongo/client_test.go +++ b/mongo/client_test.go @@ -318,9 +318,6 @@ func TestClient(t *testing.T) { }) }) t.Run("endSessions", func(t *testing.T) { - if len(os.Getenv("DOCKER_RUNNING")) > 0 { - t.Skip("This test does not work properly in docker.") - } cs := integtest.ConnString(t) originalBatchSize := endSessionsBatchSize endSessionsBatchSize = 2 @@ -340,6 +337,9 @@ func TestClient(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("skipping test in docker environment") + } t.Run(tc.name, func(t *testing.T) { // Setup a client and skip the test based on server version. From e515e84c27301228ea7691622ddafaf649783fce Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 11:31:10 -0600 Subject: [PATCH 04/21] add docker running env var --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6201c5d9f7..cd56254a5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,4 +54,6 @@ COPY ./etc/docker_entry.sh /root/docker_entry.sh COPY --from=libmongocrypt /root/install /root/install +ENV DOCKER_RUNNING=true + ENTRYPOINT ["/bin/bash", "/root/docker_entry.sh"] From fced28395effc76c77866b7d36a3a1197ef963aa Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 12:45:40 -0600 Subject: [PATCH 05/21] skip another test --- mongo/with_transactions_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongo/with_transactions_test.go b/mongo/with_transactions_test.go index 9a387264f9..f2d0333fc9 100644 --- a/mongo/with_transactions_test.go +++ b/mongo/with_transactions_test.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "math" + "os" "strconv" "strings" "testing" @@ -37,6 +38,9 @@ func TestConvenientTransactions(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("skipping test in docker environment") + } client := setupConvenientTransactions(t) db := client.Database("TestConvenientTransactions") From 0a6c7088f7677dfd67b1f4ef93535b8e608177d8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 5 Dec 2023 17:20:50 -0600 Subject: [PATCH 06/21] try with 127 --- mongo/integration/mtest/opmsg_deployment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongo/integration/mtest/opmsg_deployment.go b/mongo/integration/mtest/opmsg_deployment.go index ae4e359380..2215f84b38 100644 --- a/mongo/integration/mtest/opmsg_deployment.go +++ b/mongo/integration/mtest/opmsg_deployment.go @@ -21,7 +21,7 @@ import ( ) const ( - serverAddress = address.Address("localhost:27017") + serverAddress = address.Address("127.0.0.1:27017") maxDocumentSize uint32 = 16777216 maxMessageSize uint32 = 48000000 maxBatchCount uint32 = 100000 From d91c9dfb24eb501ad529e209f1ef52e03aa9c972 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 7 Dec 2023 07:19:08 -0600 Subject: [PATCH 07/21] clean up docker entry --- etc/docker_entry.sh | 3 +++ etc/run_docker.sh | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh index de619df73e..bbbc5f896b 100644 --- a/etc/docker_entry.sh +++ b/etc/docker_entry.sh @@ -9,5 +9,8 @@ cd /src rm -f test.suite cp -r $HOME/install ./install +export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" +export PATH="$MONGODB_BINARIES:$PATH" + # Run the test. bash ./.evergreen/run-tests.sh diff --git a/etc/run_docker.sh b/etc/run_docker.sh index a4b400035b..a62c63ba8f 100644 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -28,13 +28,21 @@ ENV="-e MONGODB_VERSION=$MONGODB_VERSION -e TOPOLOGY=$TOPOLOGY" ENV="$ENV -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" ENV="$ENV -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" ENV="$ENV -e GO_BUILD_TAGS=$GO_BUILD_TAGS" +ENV="$ENV -e MONGODB_URI=mongodb://host.docker.internal" + +# Ensure host.docker.internal is available on Linux. +EXTRA_ARGS="" +if [ "$(uname -s)" = "Linux" ]; then + EXTRA_ARGS="--add-host" +fi + +# If there is a tty, add the -t arg. +test -t 1 && EXTRA_ARGS="-t $EXTRA_ARGS" VOL="-v `pwd`:/src" VOL="$VOL -v $DRIVERS_TOOLS:/root/drivers-evergreen-tools" -USE_TTY="" -test -t 1 && USE_TTY="-t" -docker run $PLATFORM --rm $VOL $ENV -i $USE_TTY go-test +docker run $PLATFORM --rm $VOL $ENV $EXTRA_ARGS -i go-test if [ -f "test.suite" ]; then tail test.suite fi From 632ec27bb1bff6b6daf253713852dd8012f0e10d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 7 Dec 2023 09:27:46 -0600 Subject: [PATCH 08/21] clean up docker runner --- etc/run_docker.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/etc/run_docker.sh b/etc/run_docker.sh index a62c63ba8f..cdb2d79eee 100644 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -24,25 +24,26 @@ AUTH=${AUTH:-""} SSL=${SSL:=""} GO_BUILD_TAGS=${GO_BUILD_TAGS:-""} -ENV="-e MONGODB_VERSION=$MONGODB_VERSION -e TOPOLOGY=$TOPOLOGY" -ENV="$ENV -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" -ENV="$ENV -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" -ENV="$ENV -e GO_BUILD_TAGS=$GO_BUILD_TAGS" -ENV="$ENV -e MONGODB_URI=mongodb://host.docker.internal" +ARGS="$PLATFORM --rm -i" +ARGS="$ARGS -e MONGODB_VERSION=$MONGODB_VERSION -e TOPOLOGY=$TOPOLOGY" +ARGS="$ARGS -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" +ARGS="$ARGS -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" +ARGS="$ARGS -e GO_BUILD_TAGS=$GO_BUILD_TAGS" +ARGS="$ARGS -e DRIVERS_TOOLS=/root/drivers-evergeen-tools" +ARGS="$ARGS -e MONGODB_URI=mongodb://host.docker.internal" # Ensure host.docker.internal is available on Linux. -EXTRA_ARGS="" if [ "$(uname -s)" = "Linux" ]; then - EXTRA_ARGS="--add-host" + ARGS="$ARGS --add-host host.docker.internal:127.0.0.1" fi # If there is a tty, add the -t arg. -test -t 1 && EXTRA_ARGS="-t $EXTRA_ARGS" +test -t 1 && ARGS="-t $ARGS" -VOL="-v `pwd`:/src" -VOL="$VOL -v $DRIVERS_TOOLS:/root/drivers-evergreen-tools" +ARGS="$ARGS -v `pwd`:/src" +ARGS="$ARGS -v $DRIVERS_TOOLS:/root/drivers-evergreen-tools" -docker run $PLATFORM --rm $VOL $ENV $EXTRA_ARGS -i go-test +docker run $ARGS go-test if [ -f "test.suite" ]; then tail test.suite fi From 680a71acd0c49f246d31018d65667e5e07d2263c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 7 Dec 2023 12:30:19 -0600 Subject: [PATCH 09/21] change host order --- etc/run_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/run_docker.sh b/etc/run_docker.sh index cdb2d79eee..8c17075ed0 100644 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -34,7 +34,7 @@ ARGS="$ARGS -e MONGODB_URI=mongodb://host.docker.internal" # Ensure host.docker.internal is available on Linux. if [ "$(uname -s)" = "Linux" ]; then - ARGS="$ARGS --add-host host.docker.internal:127.0.0.1" + ARGS="$ARGS --add-host 127.0.0.1:host.docker.internal" fi # If there is a tty, add the -t arg. From 0688114882ded40a4491327946334fb53ef19956 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 7 Dec 2023 12:58:49 -0600 Subject: [PATCH 10/21] fix host handling --- etc/run_docker.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/etc/run_docker.sh b/etc/run_docker.sh index 8c17075ed0..b3ddc646fb 100644 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -30,11 +30,15 @@ ARGS="$ARGS -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" ARGS="$ARGS -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" ARGS="$ARGS -e GO_BUILD_TAGS=$GO_BUILD_TAGS" ARGS="$ARGS -e DRIVERS_TOOLS=/root/drivers-evergeen-tools" -ARGS="$ARGS -e MONGODB_URI=mongodb://host.docker.internal" -# Ensure host.docker.internal is available on Linux. +# Ensure host.docker.internal is available on MacOS. +if [ "$(uname -s)" = "Darwin" ]; then + ARGS="$ARGS -e MONGODB_URI=mongodb://host.docker.internal" +fi + +# Ensure host network is available on Linux. if [ "$(uname -s)" = "Linux" ]; then - ARGS="$ARGS --add-host 127.0.0.1:host.docker.internal" + ARGS="$ARGS --network=host" fi # If there is a tty, add the -t arg. From 05105182ef9c8b3784f951ccc2e6bf7410498267 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 8 Dec 2023 07:31:58 -0600 Subject: [PATCH 11/21] add test skip --- x/mongo/driver/operation/hello_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x/mongo/driver/operation/hello_test.go b/x/mongo/driver/operation/hello_test.go index 114f53b617..31575faff1 100644 --- a/x/mongo/driver/operation/hello_test.go +++ b/x/mongo/driver/operation/hello_test.go @@ -8,6 +8,7 @@ package operation import ( "fmt" + "os" "runtime" "testing" @@ -142,6 +143,10 @@ func TestAppendClientDriver(t *testing.T) { func TestAppendClientEnv(t *testing.T) { clearTestEnv(t) + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("These tests gives different results when run in Docker due to extra environment data.") + } + tests := []struct { name string omitEnvFields bool From 4562e8a0365d25c379afc0f544bb0a4439bced2f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 11 Dec 2023 12:49:17 -0600 Subject: [PATCH 12/21] clean up and use updated drivers-tools --- .evergreen/config.yml | 4 +-- etc/docker_entry.sh | 1 - etc/run_docker.sh | 35 +++----------------------- x/mongo/driver/operation/hello_test.go | 4 +++ x/mongo/driver/topology/server_test.go | 3 +++ 5 files changed, 12 insertions(+), 35 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 19380322ac..3dfb40a43c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -27,7 +27,7 @@ timeout: ls -la functions: fetch-source: - # Executes git clone and applies the submitted patch, if any + # Executes clone and applies the submitted patch, if any - command: git.get_project type: system params: @@ -145,7 +145,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone -b docker-update https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - command: shell.exec diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh index bbbc5f896b..5db1311e55 100644 --- a/etc/docker_entry.sh +++ b/etc/docker_entry.sh @@ -9,7 +9,6 @@ cd /src rm -f test.suite cp -r $HOME/install ./install -export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export PATH="$MONGODB_BINARIES:$PATH" # Run the test. diff --git a/etc/run_docker.sh b/etc/run_docker.sh index b3ddc646fb..b6d0fc9ae5 100644 --- a/etc/run_docker.sh +++ b/etc/run_docker.sh @@ -8,46 +8,17 @@ if [ -z "$DRIVERS_TOOLS" ]; then exit 1 fi PLATFORM=${DOCKER_PLATFORM:-} - -pushd $DRIVERS_TOOLS/.evergreen/docker/ubuntu20.04 -docker build $PLATFORM -t drivers-evergreen-tools . -popd docker build $PLATFORM -t go-test . # Handle environment variables and optional positional arg for the makefile target. - MAKEFILE_TARGET=${1:-evg-test-versioned-api} -MONGODB_VERSION=${MONGODB_VERSION:-latest} -TOPOLOGY=${TOPOLOGY:-replica_set} -ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-basic.json} -AUTH=${AUTH:-""} -SSL=${SSL:=""} GO_BUILD_TAGS=${GO_BUILD_TAGS:-""} -ARGS="$PLATFORM --rm -i" -ARGS="$ARGS -e MONGODB_VERSION=$MONGODB_VERSION -e TOPOLOGY=$TOPOLOGY" -ARGS="$ARGS -e MAKEFILE_TARGET=$MAKEFILE_TARGET -e AUTH=$AUTH" -ARGS="$ARGS -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE -e SSL=$SSL" +ARGS=" -e MAKEFILE_TARGET=$MAKEFILE_TARGET" ARGS="$ARGS -e GO_BUILD_TAGS=$GO_BUILD_TAGS" -ARGS="$ARGS -e DRIVERS_TOOLS=/root/drivers-evergeen-tools" - -# Ensure host.docker.internal is available on MacOS. -if [ "$(uname -s)" = "Darwin" ]; then - ARGS="$ARGS -e MONGODB_URI=mongodb://host.docker.internal" -fi - -# Ensure host network is available on Linux. -if [ "$(uname -s)" = "Linux" ]; then - ARGS="$ARGS --network=host" -fi - -# If there is a tty, add the -t arg. -test -t 1 && ARGS="-t $ARGS" - -ARGS="$ARGS -v `pwd`:/src" -ARGS="$ARGS -v $DRIVERS_TOOLS:/root/drivers-evergreen-tools" +ARGS="$ARGS go-test" -docker run $ARGS go-test +$DRIVERS_TOOLS/.evergreen/docker/run-client.sh $ARGS if [ -f "test.suite" ]; then tail test.suite fi diff --git a/x/mongo/driver/operation/hello_test.go b/x/mongo/driver/operation/hello_test.go index 31575faff1..7185d08ae3 100644 --- a/x/mongo/driver/operation/hello_test.go +++ b/x/mongo/driver/operation/hello_test.go @@ -382,6 +382,10 @@ func TestAppendClientPlatform(t *testing.T) { func TestEncodeClientMetadata(t *testing.T) { clearTestEnv(t) + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("These tests gives different results when run in Docker due to extra environment data.") + } + type application struct { Name string `bson:"name"` } diff --git a/x/mongo/driver/topology/server_test.go b/x/mongo/driver/topology/server_test.go index 4a9ffb10a1..b9101ac032 100644 --- a/x/mongo/driver/topology/server_test.go +++ b/x/mongo/driver/topology/server_test.go @@ -131,6 +131,9 @@ func TestServerHeartbeatTimeout(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } + if len(os.Getenv("DOCKER_RUNNING")) > 0 { + t.Skip("Skipping this test in docker.") + } networkTimeoutError := &net.DNSError{ IsTimeout: true, From 4fa794df4fc2f908ef9ea1f56e03b2706051413c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 11 Dec 2023 19:39:36 -0600 Subject: [PATCH 13/21] Update contributing guide --- docs/CONTRIBUTING.md | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index ec628ae0b2..f6d629c6d2 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -152,19 +152,56 @@ The usage of host.docker.internal comes from the [Docker networking documentatio There is currently no arm64 support for the go1.x runtime, see [here](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html). Known issues running on linux/arm64 include the inability to network with the localhost from the public.ecr.aws/lambda/go Docker image. +### Load Balancer + +To launch the load balancer on MacOS, run the following. + +- `brew install haproxy` +- Clone drivers-evergreen-tools and save the path as `DRIVERS_TOOLS`. +- Start the servers using (or use the docker-based method below): + +```bash +LOAD_BALANCER=true TOPOLOGY=sharded_cluster AUTH=noauth SSL=nossl MONGODB_VERSION=6.0 DRIVERS_TOOLS=$PWD/drivers-evergreen-tools MONGO_ORCHESTRATION_HOME=$PWD/drivers-evergreen-tools/.evergreen/orchestration $PWD/drivers-evergreen-tools/.evergreen/run-orchestration.sh +``` + +- Start the load balancer using: + +```bash +MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start +``` + +- Run the load balancer tests (or use the docker runner below with `evg-test-load-balancers`): + +```bash +make evg-test-load-balancers +``` + ### Testing in Docker We support local testing in Docker. To test using docker, you will need to set the `DRIVERS_TOOLs` environment variable to point to a local clone of the drivers-evergreen-tools repository. This is essential for running the testing matrix in a container. You can set the `DRIVERS_TOOLS` variable in your shell profile or in your project-specific environment. +1. First, start the drivers-tools server docker container, as: + +```bash +bash $DRIVERS_TOOLS/.evergreen/docker/start-server.sh +``` + +See the readme in `$DRIVERS_TOOLS/.evergreen/docker` for more information on usage. + +2. Next, start any other required services in another terminal, like a load balancer. + +1. Finally, run the Go Driver tests using the following script in this repo: + ```bash bash etc/run_docker.sh ``` The script takes an optional argument for the `MAKEFILE_TARGET` and allows for some environment variable overrides. The docker container has the required binaries, including libmongocrypt. -The entry script starts a MongoDB topology, and then executes the desired `MAKEFILE_TARGET`. +The entry script executes the desired `MAKEFILE_TARGET`. -For example, to test against a sharded cluster, using enterprise auth, run: +For example, to test against a sharded cluster (make sure you started the server with a sharded_cluster), +using enterprise auth, run: ```bash TOPOLOGY=sharded_cluster bash etc/run_docker.sh evg-test-enterprise-auth From 888ae782cde6d0af60e411a4bda0104f54af1eb7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 11 Dec 2023 19:45:41 -0600 Subject: [PATCH 14/21] fix tag --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3dfb40a43c..d9fcad3aae 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -1526,7 +1526,6 @@ tasks: - func: run-atlas-data-lake-test - name: test-docker-runner - tags: ["pullrequest"] commands: - func: bootstrap-mongo-orchestration - func: run-docker-test @@ -2586,6 +2585,7 @@ buildvariants: - name: "test-atlas-data-lake" - name: docker-runner-test + tags: ["pullrequest"] display_name: "Docker Runner Test" run_on: - ubuntu2204-large From 776379a08b7117af8c4c632cdc39827b8102d8e5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:02:30 -0600 Subject: [PATCH 15/21] Update mongo/client_test.go Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- mongo/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongo/client_test.go b/mongo/client_test.go index 2aa3f5c8b2..013c1ae6bb 100644 --- a/mongo/client_test.go +++ b/mongo/client_test.go @@ -337,7 +337,7 @@ func TestClient(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } - if len(os.Getenv("DOCKER_RUNNING")) > 0 { + if os.Getenv("DOCKER_RUNNING") != "" { t.Skip("skipping test in docker environment") } From 33f52126fedfdf65957abeaae815457fa60ae727 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:02:38 -0600 Subject: [PATCH 16/21] Update mongo/with_transactions_test.go Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- mongo/with_transactions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongo/with_transactions_test.go b/mongo/with_transactions_test.go index f2d0333fc9..af7ce98b0c 100644 --- a/mongo/with_transactions_test.go +++ b/mongo/with_transactions_test.go @@ -38,7 +38,7 @@ func TestConvenientTransactions(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } - if len(os.Getenv("DOCKER_RUNNING")) > 0 { + if os.Getenv("DOCKER_RUNNING") != "" { t.Skip("skipping test in docker environment") } From c01cf6ee6bb33e1483df26fb287c12c50499a941 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:02:47 -0600 Subject: [PATCH 17/21] Update x/mongo/driver/operation/hello_test.go Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- x/mongo/driver/operation/hello_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/mongo/driver/operation/hello_test.go b/x/mongo/driver/operation/hello_test.go index 7185d08ae3..97b4e324ec 100644 --- a/x/mongo/driver/operation/hello_test.go +++ b/x/mongo/driver/operation/hello_test.go @@ -143,7 +143,7 @@ func TestAppendClientDriver(t *testing.T) { func TestAppendClientEnv(t *testing.T) { clearTestEnv(t) - if len(os.Getenv("DOCKER_RUNNING")) > 0 { + if os.Getenv("DOCKER_RUNNING") != "" { t.Skip("These tests gives different results when run in Docker due to extra environment data.") } From c9d8da53c40424e74475b465e60097c0e64b5b94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:02:55 -0600 Subject: [PATCH 18/21] Update x/mongo/driver/operation/hello_test.go Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- x/mongo/driver/operation/hello_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/mongo/driver/operation/hello_test.go b/x/mongo/driver/operation/hello_test.go index 97b4e324ec..6bfaf00f6d 100644 --- a/x/mongo/driver/operation/hello_test.go +++ b/x/mongo/driver/operation/hello_test.go @@ -382,7 +382,7 @@ func TestAppendClientPlatform(t *testing.T) { func TestEncodeClientMetadata(t *testing.T) { clearTestEnv(t) - if len(os.Getenv("DOCKER_RUNNING")) > 0 { + if os.Getenv("DOCKER_RUNNING") != "" { t.Skip("These tests gives different results when run in Docker due to extra environment data.") } From 3c36cd9b02a373e1f38af41be999a5024bd8a933 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:03:03 -0600 Subject: [PATCH 19/21] Update x/mongo/driver/topology/server_test.go Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- x/mongo/driver/topology/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/mongo/driver/topology/server_test.go b/x/mongo/driver/topology/server_test.go index b9101ac032..a04f5ed7c0 100644 --- a/x/mongo/driver/topology/server_test.go +++ b/x/mongo/driver/topology/server_test.go @@ -131,7 +131,7 @@ func TestServerHeartbeatTimeout(t *testing.T) { if testing.Short() { t.Skip("skipping integration test in short mode") } - if len(os.Getenv("DOCKER_RUNNING")) > 0 { + if os.Getenv("DOCKER_RUNNING") != "" { t.Skip("Skipping this test in docker.") } From 887fa7b51f5bd6d3dc98cc74b1cb5b11da206829 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 14 Dec 2023 17:03:11 -0600 Subject: [PATCH 20/21] Update etc/docker_entry.sh Co-authored-by: Qingyang Hu <103950869+qingyang-hu@users.noreply.github.com> --- etc/docker_entry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/docker_entry.sh b/etc/docker_entry.sh index 5db1311e55..4e3e325970 100644 --- a/etc/docker_entry.sh +++ b/etc/docker_entry.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Entry point for Dockerfile for launching running a go test. +# Entry point for Dockerfile for running a go test. # set -eux From 154c039cde3ee6a31c9b3c579bf52d2ab3431fad Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 18 Dec 2023 13:30:40 -0600 Subject: [PATCH 21/21] use main branch --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d9fcad3aae..54ad119ac9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -145,7 +145,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone -b docker-update https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - command: shell.exec