CSOT: Master into csot merge

.evergreen/.evg.yml

@@ -12,9 +12,8 @@ stepback: true
 # Actual testing tasks are marked with `type: test`
 command_type: system
 
-# Protect ourself against rogue test case, or curl gone wild, that runs forever
-# 12 minutes is the longest we'll ever run
-exec_timeout_secs: 3600 # 12 minutes is the longest we'll ever run
+# Protect ourselves against rogue test case, or curl gone wild, that runs forever
+exec_timeout_secs: 3600
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -968,6 +967,60 @@ tasks:
         - func: "run load-balancer"
         - func: "run load-balancer tests"
 
+    - name: "oidc-auth-test"
+      commands:
+        - command: subprocess.exec
+          type: test
+          params:
+            working_dir: "src"
+            binary: bash
+            include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+            env:
+              OIDC_ENV: "test"
+            args:
+              - .evergreen/run-mongodb-oidc-test.sh
+
+    - name: "oidc-auth-test-azure"
+      commands:
+        - command: shell.exec
+          params:
+            shell: bash
+            env:
+              JAVA_HOME: ${JAVA_HOME}
+            script: |-
+              set -o errexit
+              ${PREPARE_SHELL}
+              cd src
+              git add .
+              git commit --allow-empty -m "add files"
+              # uncompressed tar used to allow appending .git folder
+              export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tar
+              git archive -o $AZUREOIDC_DRIVERS_TAR_FILE HEAD
+              tar -rf $AZUREOIDC_DRIVERS_TAR_FILE .git
+              export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh"
+              bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh
+
+    - name: "oidc-auth-test-gcp"
+      commands:
+        - command: shell.exec
+          params:
+            shell: bash
+            script: |-
+              set -o errexit
+              ${PREPARE_SHELL}
+              cd src
+              git add .
+              git commit --allow-empty -m "add files"
+              # uncompressed tar used to allow appending .git folder
+              export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tar
+              git archive -o $GCPOIDC_DRIVERS_TAR_FILE HEAD
+              tar -rf $GCPOIDC_DRIVERS_TAR_FILE .git
+              # Define the command to run on the VM.
+              # Ensure that we source the environment file created for us, set up any other variables we need,
+              # and then run our test suite on the vm.
+              export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh"
+              bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh
+
     - name: serverless-test
       commands:
         - func: "run serverless"
@@ -1710,6 +1763,13 @@ axes:
            # Multiple mongos instances can be specified in the connection string
            # for this version.
            SAFE_FOR_MULTI_MONGOS: true
+      - id: "8.0"
+        display_name: "8.0"
+        variables:
+          VERSION: "8.0"
+          # Multiple mongos instances can be specified in the connection string
+          # for this version.
+          SAFE_FOR_MULTI_MONGOS: true
       - id: "7.0"
         display_name: "7.0"
         variables:
@@ -1749,10 +1809,6 @@ axes:
         display_name: "4.0"
         variables:
            VERSION: "4.0"
-      - id: "3.6"
-        display_name: "3.6"
-        variables:
-           VERSION: "3.6"
   - id: os
     display_name: OS
     values:
@@ -2065,6 +2121,78 @@ task_groups:
     tasks:
       - test-aws-lambda-deployed
 
+  - name: testoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: ec2.assume_role
+        params:
+          role_arn: ${aws_test_secrets_role}
+      - command: subprocess.exec
+        params:
+          binary: bash
+          include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test
+
+  - name: testazureoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: subprocess.exec
+        params:
+          binary: bash
+          env:
+            AZUREOIDC_VMNAME_PREFIX: "JAVA_DRIVER"
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test-azure
+
+  - name: testgcpoidc_task_group
+    setup_group:
+      - func: fetch source
+      - func: prepare resources
+      - func: fix absolute paths
+      - command: subprocess.exec
+        params:
+          binary: bash
+          env:
+            GCPOIDC_VMNAME_PREFIX: "JAVA_DRIVER"
+            GCPKMS_MACHINETYPE: "e2-medium" # comparable elapsed time to Azure; default was starved, caused timeouts
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh
+    teardown_task:
+      - command: subprocess.exec
+        params:
+          binary: bash
+          args:
+            - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800
+    tasks:
+      - oidc-auth-test-gcp
+
 buildvariants:
 
 # Test packaging and other release related routines
@@ -2090,15 +2218,15 @@ buildvariants:
 
 - matrix_name: "tests-zstd-compression"
   matrix_spec: { compressor : "zstd", auth: "noauth", ssl: "nossl", jdk: "jdk8",
-                 version: ["4.2", "4.4", "5.0", "6.0", "7.0", "latest"],
+                 version: ["4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "latest"],
                  topology: "standalone", os: "linux" }
   display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${jdk} ${os} "
   tags: ["tests-variant"]
   tasks:
     - name: "test"
 
 - matrix_name: "tests-jdk8-unsecure"
-  matrix_spec: { auth: "noauth", ssl: "nossl", jdk: "jdk8", version: ["3.6", "4.0", "4.2", "4.4", "5.0", "6.0", "7.0", "latest"],
+  matrix_spec: { auth: "noauth", ssl: "nossl", jdk: "jdk8", version: ["4.0", "4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "latest"],
                  topology: "*", os: "linux" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} "
   tags: ["tests-variant"]
@@ -2107,7 +2235,7 @@ buildvariants:
 
 - matrix_name: "tests-jdk-secure"
   matrix_spec: { auth: "auth", ssl: "ssl", jdk: [ "jdk8", "jdk17", "jdk21"],
-                 version: [ "3.6", "4.0", "4.2", "4.4", "5.0", "6.0", "7.0", "latest" ],
+                 version: ["4.0", "4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "latest" ],
                  topology: "*", os: "linux" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${jdk} ${os} "
   tags: ["tests-variant"]
@@ -2122,15 +2250,15 @@ buildvariants:
     - name: "test"
 
 - matrix_name: "tests-require-api-version"
-  matrix_spec: { api-version: "required", auth: "auth", ssl: "nossl", jdk: ["jdk21"], version: ["5.0", "6.0", "7.0", "latest"],
+  matrix_spec: { api-version: "required", auth: "auth", ssl: "nossl", jdk: ["jdk21"], version: ["5.0", "6.0", "7.0", "8.0", "latest"],
                  topology: "standalone", os: "linux" }
   display_name: "${version} ${topology} ${api-version} "
   tags: ["tests-variant"]
   tasks:
     - name: "test"
 
 - matrix_name: "tests-load-balancer-secure"
-  matrix_spec: { auth: "auth", ssl: "ssl", jdk: ["jdk21"], version: ["5.0", "6.0", "7.0", "latest"], topology: "sharded-cluster",
+  matrix_spec: { auth: "auth", ssl: "ssl", jdk: ["jdk21"], version: ["5.0", "6.0", "7.0", "8.0", "latest"], topology: "sharded-cluster",
                  os: "ubuntu" }
   display_name: "Load Balancer ${version} ${auth} ${ssl} ${jdk} ${os}"
   tasks:
@@ -2216,8 +2344,29 @@ buildvariants:
   tasks:
     - name: "test_atlas_task_group_search_indexes"
 
+- name: "oidc-auth-test"
+  display_name: "OIDC Auth"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testoidc_task_group
+      batchtime: 20160 # 14 days
+
+- name: testazureoidc-variant
+  display_name: "OIDC Auth Azure"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testazureoidc_task_group
+      batchtime: 20160 # 14 days
+
+- name: testgcpoidc-variant
+  display_name: "OIDC Auth GCP"
+  run_on: ubuntu2204-small
+  tasks:
+    - name: testgcpoidc_task_group
+      batchtime: 20160 # 14 days
+
 - matrix_name: "aws-auth-test"
-  matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17", "jdk21"], version: ["4.4", "5.0", "6.0", "7.0", "latest"], os: "ubuntu",
+  matrix_spec: { ssl: "nossl", jdk: ["jdk8", "jdk17", "jdk21"], version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], os: "ubuntu",
                  aws-credential-provider: "*" }
   display_name: "MONGODB-AWS Basic Auth test ${version} ${jdk} ${aws-credential-provider}"
   run_on: ubuntu2004-small
@@ -2236,14 +2385,15 @@ buildvariants:
     - name: "aws-auth-test-with-web-identity-credentials"
 
 - matrix_name: "accept-api-version-2-test"
-  matrix_spec: { ssl: "nossl", auth: "noauth", jdk: "jdk21", version: ["5.0", "6.0", "7.0", "latest"], topology: "standalone", os: "linux" }
+  matrix_spec: { ssl: "nossl", auth: "noauth", jdk: "jdk21", version: ["5.0", "6.0", "7.0", "8.0", "latest"], topology: "standalone",
+                 os: "linux" }
   display_name: "Accept API Version 2 ${version}"
   run_on: ubuntu2004-small
   tasks:
     - name: "accept-api-version-2-test"
 
 - matrix_name: "ocsp-test"
-  matrix_spec: { auth: "noauth", ssl: "ssl", jdk: "jdk21", version: ["4.4", "5.0", "6.0", "7.0", "latest"], os: "ubuntu" }
+  matrix_spec: { auth: "noauth", ssl: "ssl", jdk: "jdk21", version: ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], os: "ubuntu" }
   display_name: "OCSP test ${version} ${os}"
   tasks:
     - name: ".ocsp"
@@ -2318,7 +2468,7 @@ buildvariants:
     - name: ".csfle-aws-from-environment"
 
 - matrix_name: "csfle-tests-with-mongocryptd"
-  matrix_spec: { os: "linux", version: [ "4.2", "4.4", "5.0", "6.0", "7.0", "latest" ], topology: ["replicaset"] }
+  matrix_spec: { os: "linux", version: [ "4.2", "4.4", "5.0", "6.0", "7.0", "8.0", "latest" ], topology: ["replicaset"] }
   display_name: "CSFLE with mongocryptd: ${version}"
   tasks:
     - name: "csfle-tests-with-mongocryptd"

.evergreen/prepare-oidc-get-tokens-docker.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit  # Exit the script with error if any of the commands fail
+
+############################################
+#            Main Program                  #
+############################################
+
+# Supported/used environment variables:
+#       DRIVERS_TOOLS                                          The path to evergreeen tools
+#       OIDC_AWS_*                                             Required OIDC_AWS_* env variables must be configured
+#
+# Environment variables used as output:
+#       OIDC_TESTS_ENABLED                                     Allows running OIDC tests
+#       OIDC_TOKEN_DIR                                         The path to generated OIDC AWS tokens
+#       AWS_WEB_IDENTITY_TOKEN_FILE                            The path to AWS token for device workflow
+
+if [ -z ${DRIVERS_TOOLS+x} ]; then
+    echo "DRIVERS_TOOLS. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_ROLE_ARN+x} ]; then
+    echo "OIDC_AWS_ROLE_ARN. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_SECRET_ACCESS_KEY+x} ]; then
+    echo "OIDC_AWS_SECRET_ACCESS_KEY. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_ACCESS_KEY_ID+x} ]; then
+    echo "OIDC_AWS_ACCESS_KEY_ID. is not set";
+    exit 1
+fi
+
+export AWS_ROLE_ARN=${OIDC_AWS_ROLE_ARN}
+export AWS_SECRET_ACCESS_KEY=${OIDC_AWS_SECRET_ACCESS_KEY}
+export AWS_ACCESS_KEY_ID=${OIDC_AWS_ACCESS_KEY_ID}
+export OIDC_FOLDER=${DRIVERS_TOOLS}/.evergreen/auth_oidc
+export OIDC_TOKEN_DIR=${OIDC_FOLDER}/test_tokens
+export AWS_WEB_IDENTITY_TOKEN_FILE=${OIDC_TOKEN_DIR}/test1
+export OIDC_TESTS_ENABLED=true
+
+echo "Configuring OIDC server for local authentication tests"
+
+cd ${OIDC_FOLDER}
+DRIVERS_TOOLS=${DRIVERS_TOOLS} ./oidc_get_tokens.sh

.evergreen/prepare-oidc-server-docker.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit  # Exit the script with error if any of the commands fail
+
+############################################
+#            Main Program                  #
+############################################
+
+# Supported/used environment variables:
+#       DRIVERS_TOOLS                                          The path to evergreeen tools
+#       OIDC_AWS_*                                             OIDC_AWS_* env variables must be configured
+#
+# Environment variables used as output:
+#       OIDC_TESTS_ENABLED                                     Allows running OIDC tests
+#       OIDC_TOKEN_DIR                                         The path to generated tokens
+#       AWS_WEB_IDENTITY_TOKEN_FILE                            The path to AWS token for device workflow
+
+if [ -z ${DRIVERS_TOOLS+x} ]; then
+    echo "DRIVERS_TOOLS. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_ROLE_ARN+x} ]; then
+    echo "OIDC_AWS_ROLE_ARN. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_SECRET_ACCESS_KEY+x} ]; then
+    echo "OIDC_AWS_SECRET_ACCESS_KEY. is not set";
+    exit 1
+fi
+
+if [ -z ${OIDC_AWS_ACCESS_KEY_ID+x} ]; then
+    echo "OIDC_AWS_ACCESS_KEY_ID. is not set";
+    exit 1
+fi
+
+export AWS_ROLE_ARN=${OIDC_AWS_ROLE_ARN}
+export AWS_SECRET_ACCESS_KEY=${OIDC_AWS_SECRET_ACCESS_KEY}
+export AWS_ACCESS_KEY_ID=${OIDC_AWS_ACCESS_KEY_ID}
+export OIDC_FOLDER=${DRIVERS_TOOLS}/.evergreen/auth_oidc
+export OIDC_TOKEN_DIR=${OIDC_FOLDER}/test_tokens
+export AWS_WEB_IDENTITY_TOKEN_FILE=${OIDC_TOKEN_DIR}/test1
+export OIDC_TESTS_ENABLED=true
+
+echo "Configuring OIDC server for local authentication tests"
+
+cd ${OIDC_FOLDER}
+DRIVERS_TOOLS=${DRIVERS_TOOLS} ./start_local_server.sh