diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 338139f77b..7623c3b443 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -115,11 +115,6 @@ functions: export OCSP_CONNECTIVITY="${OCSP_CONNECTIVITY}" export OCSP_VERIFIER="${OCSP_VERIFIER}" - export ATLAS_REPLICA_SET_URI="${atlas_replica_set_uri}" - export ATLAS_SHARDED_URI="${atlas_sharded_uri}" - export ATLAS_FREE_TIER_URI="${atlas_free_tier_uri}" - export ATLAS_TLS11_URI="${atlas_tls11_uri}" - export ATLAS_TLS12_URI="${atlas_tls12_uri}" export RVM_RUBY="${RVM_RUBY}" EOT @@ -132,36 +127,111 @@ functions: params: file: src/expansion.yml - "export AWS auth credentials": + bootstrap-mongo-orchestration: - command: shell.exec - type: test params: - silent: true - working_dir: "src" + shell: "bash" script: | - cat < .env.private - IAM_AUTH_ASSUME_AWS_ACCOUNT="${iam_auth_assume_aws_account}" - IAM_AUTH_ASSUME_AWS_SECRET_ACCESS_KEY="${iam_auth_assume_aws_secret_access_key}" - IAM_AUTH_ASSUME_ROLE_NAME="${iam_auth_assume_role_name}" - IAM_AUTH_EC2_INSTANCE_ACCOUNT="${iam_auth_ec2_instance_account}" - IAM_AUTH_EC2_INSTANCE_PROFILE="${iam_auth_ec2_instance_profile}" - IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY="${iam_auth_ec2_instance_secret_access_key}" - IAM_AUTH_ECS_ACCOUNT="${iam_auth_ecs_account}" - IAM_AUTH_ECS_ACCOUNT_ARN="${iam_auth_ecs_account_arn}" - IAM_AUTH_ECS_CLUSTER="${iam_auth_ecs_cluster}" - IAM_AUTH_ECS_SECRET_ACCESS_KEY="${iam_auth_ecs_secret_access_key}" - IAM_AUTH_ECS_SECURITY_GROUP="${iam_auth_ecs_security_group}" - IAM_AUTH_ECS_SUBNET_A="${iam_auth_ecs_subnet_a}" - IAM_AUTH_ECS_SUBNET_B="${iam_auth_ecs_subnet_b}" - IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition_ubuntu2004}" - - IAM_WEB_IDENTITY_ISSUER="${iam_web_identity_issuer}" - IAM_WEB_IDENTITY_JWKS_URI="${iam_web_identity_jwks_uri}" - IAM_WEB_IDENTITY_RSA_KEY="${iam_web_identity_rsa_key}" - IAM_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" - IAM_AUTH_ASSUME_WEB_ROLE_NAME="${iam_auth_assume_web_role_name}" + set -x + ${PREPARE_SHELL} - EOT + MONGODB_VERSION=${VERSION} \ + TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ + LOAD_BALANCER=${LOAD_BALANCER} \ + sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + - command: expansions.update + params: + file: mo-expansion.yml + + run-valid-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ + -p 8100 -v + + run-revoked-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ + -p 8100 \ + -v \ + --fault revoked + + run-valid-delegate-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ + -p 8100 -v + + run-revoked-delegate-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ + -p 8100 \ + -v \ + --fault revoked + + run-load-balancer: + - command: shell.exec + params: + shell: "bash" + script: | + DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + - command: expansions.update + params: + file: lb-expansion.yml + + "export AWS auth credentials": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: subprocess.exec + type: test + params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run CSOT tests": - command: shell.exec @@ -171,46 +241,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi export CSOT_SPEC_TESTS=1 TEST_CMD="bundle exec rspec spec/spec_tests/client_side_operations_timeout_spec.rb" \ .evergreen/run-tests.sh - "export FLE credentials": - - command: shell.exec - type: test - params: - silent: true - working_dir: "src" - script: | - cat < .env.private - MONGO_RUBY_DRIVER_AWS_KEY="${fle_aws_key}" - MONGO_RUBY_DRIVER_AWS_SECRET="${fle_aws_secret}" - MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" - MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" - - MONGO_RUBY_DRIVER_AZURE_TENANT_ID="${fle_azure_tenant_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_ID="${fle_azure_client_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET="${fle_azure_client_secret}" - MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" - - MONGO_RUBY_DRIVER_GCP_EMAIL="${fle_gcp_email}" - MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY="${fle_gcp_private_key}" - MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" - MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" - MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" - MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" - MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" - EOT - "export Kerberos credentials": - command: shell.exec type: test @@ -341,7 +375,7 @@ functions: "upload test results": - command: attach.xunit_results params: - file: ./src/rspec.xml + file: ./src/tmp/*.xml "delete private environment": - command: shell.exec @@ -373,13 +407,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi .evergreen/run-tests.sh "run AWS auth tests": @@ -411,14 +438,6 @@ functions: script: | ${PREPARE_SHELL} AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" \ - ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} \ - ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} \ - ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} \ - ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} \ - ATLAS_X509_CERT_BASE64="${atlas_x509_cert_base64}" \ - ATLAS_X509_URI="${atlas_x509}" \ - ATLAS_X509_DEV_CERT_BASE64="${atlas_x509_dev_cert_base64}" \ - ATLAS_X509_DEV_URI="${atlas_x509_dev}" \ .evergreen/run-tests-atlas.sh pre: @@ -426,11 +445,11 @@ pre: - func: "create expansions" post: + - func: "upload test results" - func: "delete private environment" # Removed, causing timeouts # - func: "upload working dir" - func: "upload mo artifacts" - # - func: "upload test results" - func: "upload test results to s3" task_groups: @@ -619,24 +638,29 @@ tasks: - name: "test-atlas" commands: - func: "run Atlas tests" - - name: "test-mlaunch" + - name: "test-main" commands: + - func: bootstrap-mongo-orchestration - func: "run tests" - name: "driver-bench" commands: + - func: bootstrap-mongo-orchestration - func: "run benchmarks" - name: "test-kerberos" commands: + - func: bootstrap-mongo-orchestration - func: "run Kerberos unit tests" - name: "test-csot" commands: + - func: bootstrap-mongo-orchestration - func: "run CSOT tests" - name: "test-fle" commands: - - func: "export FLE credentials" + - func: bootstrap-mongo-orchestration - func: "run tests" - name: "test-aws-auth" commands: + - func: bootstrap-mongo-orchestration - func: "export AWS auth credentials" - func: "run AWS auth tests" - name: "test-full-atlas-task" @@ -813,26 +837,18 @@ axes: - id: "topology" display_name: Topology values: - - id: "standalone" - display_name: Standalone + - id: "server" + display_name: Server variables: - TOPOLOGY: standalone - - id: "replica-set" + TOPOLOGY: server + - id: "replica_set" display_name: Replica Set variables: - TOPOLOGY: replica-set - - id: "replica-set-single-node" - display_name: Replica Set (Single Node) - variables: - TOPOLOGY: replica-set-single-node - - id: "sharded-cluster" - display_name: Sharded - variables: - TOPOLOGY: sharded-cluster - - id: "load-balanced" - display_name: Load Balanced + TOPOLOGY: replica_set + - id: "sharded_cluster" + display_name: Sharded Cluster variables: - TOPOLOGY: load-balanced + TOPOLOGY: sharded_cluster - id: "single-mongos" display_name: Single Mongos @@ -1169,7 +1185,7 @@ buildvariants: matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: standalone + topology: server os: ubuntu2204 display_name: DriverBench tasks: @@ -1180,100 +1196,100 @@ buildvariants: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] ruby: "ruby-3.3" mongodb-version: ["latest", "8.0", "7.0"] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: ${auth-and-ssl} ${ruby} db-${mongodb-version} ${topology} tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-recent" matrix_spec: ruby: ["ruby-3.3", "ruby-3.2", "jruby-9.4"] mongodb-version: ["latest", "8.0", "7.0"] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${os} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-8-arm" matrix_spec: ruby: "ruby-3.3" mongodb-version: [ '8.0' ] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2404-arm display_name: "${mongodb-version} ${os} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-5.x" matrix_spec: ruby: ["ruby-3.3", "ruby-3.2", "jruby-9.4"] mongodb-version: ['5.0'] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-4.x" matrix_spec: ruby: ["ruby-3.0", "ruby-2.7"] mongodb-version: ['4.4', '4.2', '4.0'] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-3.6" matrix_spec: ruby: "ruby-2.7" mongodb-version: ['3.6'] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "single-lb" matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: load-balanced + topology: sharded_cluster single-mongos: single-mongos os: ubuntu2204 display_name: "${mongodb-version} ${topology} single-lb ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-api-version" matrix_spec: ruby: "ruby-3.3" mongodb-version: '7.0' - topology: standalone + topology: server api-version-required: yes os: ubuntu2204 display_name: "${mongodb-version} api-version-required ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "single-mongos" matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: "sharded-cluster" + topology: "sharded_cluster" single-mongos: single-mongos os: ubuntu2204 display_name: "${mongodb-version} ${topology} single-mongos ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: CSOT matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: replica-set-single-node + topology: replica_set os: ubuntu2204 display_name: "CSOT - ${mongodb-version}" tasks: @@ -1284,134 +1300,134 @@ buildvariants: retry-reads: no-retry-reads ruby: "ruby-3.3" mongodb-version: "8.0" - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${retry-reads} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "no-retry-writes" matrix_spec: retry-writes: no-retry-writes ruby: "ruby-3.3" mongodb-version: "8.0" - topology: [replica-set, sharded-cluster] + topology: [replica_set, sharded_cluster] os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${retry-writes} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: mmapv1 matrix_spec: ruby: "ruby-2.7" mongodb-version: ['3.6', '4.0'] - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] storage-engine: mmapv1 os: ubuntu1804 display_name: "${mongodb-version} ${topology} mmapv1 ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "lint" matrix_spec: lint: on ruby: "ruby-3.3" mongodb-version: "8.0" - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${lint} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "fork" matrix_spec: fork: on ruby: "ruby-3.3" mongodb-version: "8.0" - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${topology} fork ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "solo" matrix_spec: solo: on ruby: ["ruby-3.3", "ruby-3.2", "ruby-3.1"] mongodb-version: "8.0" - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${topology} solo ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "stress older" matrix_spec: stress: on ruby: "ruby-2.7" mongodb-version: ['4.2', '4.0', '3.6'] - topology: replica-set + topology: replica_set os: ubuntu1804 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "stress" matrix_spec: stress: on ruby: "ruby-3.3" mongodb-version: ["8.0", "7.0"] - topology: replica-set + topology: replica_set os: ubuntu2204 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "x509-tests" matrix_spec: auth-and-ssl: "x509" ruby: "ruby-3.3" mongodb-version: "8.0" - topology: standalone + topology: server os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "jruby-auth" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: jruby-9.4 mongodb-version: "8.0" - topology: ["standalone", "replica-set", "sharded-cluster"] + topology: ["server", "replica_set", "sharded_cluster"] os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: zlib-"ruby-3.3" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-3.3" mongodb-version: "8.0" - topology: "replica-set" + topology: "replica_set" compressor: 'zlib' os: ubuntu2204 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: snappy-"ruby-3.3" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-3.3" mongodb-version: "8.0" - topology: "replica-set" + topology: "replica_set" compressor: 'snappy' os: ubuntu2204 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" # the zstd-ruby gem does not support JRuby (explicitly). However, there is # apparently a zstd-jni gem for JRuby that we could investigate here; if @@ -1422,57 +1438,57 @@ buildvariants: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-3.3" mongodb-version: "8.0" - topology: "replica-set" + topology: "replica_set" compressor: 'zstd' os: ubuntu2204 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: activesupport-"ruby-3.3" matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: replica-set + topology: replica_set as: as os: ubuntu2204 display_name: "AS ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: bson-"ruby-3.3" matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: replica-set + topology: replica_set bson: "*" os: ubuntu2204 display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: zlib-"ruby-2.7" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-2.7" mongodb-version: "6.0" - topology: "replica-set" + topology: "replica_set" compressor: 'zlib' os: ubuntu2004 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: snappy-"ruby-2.7" matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-2.7" mongodb-version: "6.0" - topology: "replica-set" + topology: "replica_set" compressor: 'snappy' os: ubuntu2004 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" # the zstd-ruby gem does not support JRuby (explicitly). However, there is # apparently a zstd-jni gem for JRuby that we could investigate here; if @@ -1483,40 +1499,40 @@ buildvariants: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: "ruby-2.7" mongodb-version: "6.0" - topology: "replica-set" + topology: "replica_set" compressor: 'zstd' os: ubuntu2004 display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: activesupport-"ruby-2.7" matrix_spec: ruby: "ruby-2.7" mongodb-version: "6.0" - topology: replica-set + topology: replica_set as: as os: ubuntu2004 display_name: "AS ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: bson-"ruby-2.7" matrix_spec: ruby: "ruby-2.7" mongodb-version: "6.0" - topology: replica-set + topology: replica_set bson: "*" os: ubuntu2004 display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "fle above 4.4" matrix_spec: auth-and-ssl: "noauth-and-nossl" ruby: ["ruby-3.3", "ruby-3.2", "ruby-3.1"] - topology: [replica-set, sharded-cluster] + topology: [replica_set, sharded_cluster] mongodb-version: [ '6.0', '7.0', '8.0' ] os: ubuntu2204 fle: helper @@ -1536,7 +1552,7 @@ buildvariants: matrix_spec: ruby: "ruby-3.3" mongodb-version: "8.0" - topology: standalone + topology: server os: ubuntu2204 auth-and-ssl: kerberos display_name: "Kerberos Tests" @@ -1547,7 +1563,7 @@ buildvariants: # matrix_spec: # auth-and-ssl: "noauth-and-nossl" # ruby: -# topology: [replica-set, sharded-cluster] +# topology: [replica_set, sharded_cluster] # mongodb-version: [ 'latest' ] # os: ubuntu2204 # fle: helper @@ -1563,7 +1579,7 @@ buildvariants: # https://jira.mongodb.org/browse/RUBY-3659 auth-and-ssl: [ aws-regular, aws-assume-role, aws-web-identity ] ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" @@ -1575,12 +1591,12 @@ buildvariants: ocsp-verifier: true # No JRuby due to https://github.com/jruby/jruby-openssl/issues/210 ruby: ["ruby-3.3", "ruby-3.2", "ruby-3.1"] - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP verifier: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-must-staple matrix_spec: @@ -1588,26 +1604,26 @@ buildvariants: ocsp-must-staple: on ocsp-delegate: on ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - must staple: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-unknown matrix_spec: ocsp-algorithm: rsa ocsp-status: unknown ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - unknown: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: @@ -1617,12 +1633,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "none" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1631,12 +1647,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "none" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1645,12 +1661,12 @@ buildvariants: ocsp-connectivity: fail extra-uri-options: "none" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1659,12 +1675,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsInsecure=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1673,12 +1689,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsInsecure=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1687,12 +1703,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsInsecure=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1701,12 +1717,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsAllowInvalidCertificates=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1715,12 +1731,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsAllowInvalidCertificates=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity matrix_spec: ocsp-algorithm: '*' @@ -1729,12 +1745,12 @@ buildvariants: ocsp-connectivity: pass extra-uri-options: "tlsAllowInvalidCertificates=true" ruby: "ruby-3.3" - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-connectivity-jruby matrix_spec: @@ -1749,19 +1765,19 @@ buildvariants: ocsp-delegate: '*' ocsp-connectivity: pass ruby: jruby-9.4 - topology: standalone + topology: server mongodb-version: "8.0" os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main # https://jira.mongodb.org/browse/RUBY-3540 #- matrix_name: testgcpkms-variant # matrix_spec: # ruby: "ruby-3.3" # fle: helper - # topology: standalone + # topology: server # os: ubuntu2204 # mongodb-version: "8.0" # display_name: "GCP KMS" @@ -1774,7 +1790,7 @@ buildvariants: # matrix_spec: # ruby: ruby-3.0 # fle: helper - # topology: standalone + # topology: server # os: debian11 # could eventually look at updating this to rhel80 # mongodb-version: 6.0 # display_name: "AZURE KMS" diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index 44e018f5de..cc914e4680 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -65,26 +65,18 @@ axes: - id: "topology" display_name: Topology values: - - id: "standalone" - display_name: Standalone + - id: "server" + display_name: Server variables: - TOPOLOGY: standalone - - id: "replica-set" + TOPOLOGY: server + - id: "replica_set" display_name: Replica Set variables: - TOPOLOGY: replica-set - - id: "replica-set-single-node" - display_name: Replica Set (Single Node) + TOPOLOGY: replica_set + - id: "sharded_cluster" + display_name: Sharded Cluster variables: - TOPOLOGY: replica-set-single-node - - id: "sharded-cluster" - display_name: Sharded - variables: - TOPOLOGY: sharded-cluster - - id: "load-balanced" - display_name: Load Balanced - variables: - TOPOLOGY: load-balanced + TOPOLOGY: sharded_cluster - id: "single-mongos" display_name: Single Mongos diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 85fddd3dd0..9d19977542 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -112,11 +112,6 @@ functions: export OCSP_CONNECTIVITY="${OCSP_CONNECTIVITY}" export OCSP_VERIFIER="${OCSP_VERIFIER}" - export ATLAS_REPLICA_SET_URI="${atlas_replica_set_uri}" - export ATLAS_SHARDED_URI="${atlas_sharded_uri}" - export ATLAS_FREE_TIER_URI="${atlas_free_tier_uri}" - export ATLAS_TLS11_URI="${atlas_tls11_uri}" - export ATLAS_TLS12_URI="${atlas_tls12_uri}" export RVM_RUBY="${RVM_RUBY}" EOT @@ -129,36 +124,111 @@ functions: params: file: src/expansion.yml - "export AWS auth credentials": + bootstrap-mongo-orchestration: - command: shell.exec - type: test params: - silent: true - working_dir: "src" + shell: "bash" script: | - cat < .env.private - IAM_AUTH_ASSUME_AWS_ACCOUNT="${iam_auth_assume_aws_account}" - IAM_AUTH_ASSUME_AWS_SECRET_ACCESS_KEY="${iam_auth_assume_aws_secret_access_key}" - IAM_AUTH_ASSUME_ROLE_NAME="${iam_auth_assume_role_name}" - IAM_AUTH_EC2_INSTANCE_ACCOUNT="${iam_auth_ec2_instance_account}" - IAM_AUTH_EC2_INSTANCE_PROFILE="${iam_auth_ec2_instance_profile}" - IAM_AUTH_EC2_INSTANCE_SECRET_ACCESS_KEY="${iam_auth_ec2_instance_secret_access_key}" - IAM_AUTH_ECS_ACCOUNT="${iam_auth_ecs_account}" - IAM_AUTH_ECS_ACCOUNT_ARN="${iam_auth_ecs_account_arn}" - IAM_AUTH_ECS_CLUSTER="${iam_auth_ecs_cluster}" - IAM_AUTH_ECS_SECRET_ACCESS_KEY="${iam_auth_ecs_secret_access_key}" - IAM_AUTH_ECS_SECURITY_GROUP="${iam_auth_ecs_security_group}" - IAM_AUTH_ECS_SUBNET_A="${iam_auth_ecs_subnet_a}" - IAM_AUTH_ECS_SUBNET_B="${iam_auth_ecs_subnet_b}" - IAM_AUTH_ECS_TASK_DEFINITION="${iam_auth_ecs_task_definition_ubuntu2004}" - - IAM_WEB_IDENTITY_ISSUER="${iam_web_identity_issuer}" - IAM_WEB_IDENTITY_JWKS_URI="${iam_web_identity_jwks_uri}" - IAM_WEB_IDENTITY_RSA_KEY="${iam_web_identity_rsa_key}" - IAM_WEB_IDENTITY_TOKEN_FILE="${iam_web_identity_token_file}" - IAM_AUTH_ASSUME_WEB_ROLE_NAME="${iam_auth_assume_web_role_name}" + set -x + ${PREPARE_SHELL} - EOT + MONGODB_VERSION=${VERSION} \ + TOPOLOGY=${TOPOLOGY} \ + AUTH=${AUTH} \ + SSL=${SSL} \ + ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ + LOAD_BALANCER=${LOAD_BALANCER} \ + sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + - command: expansions.update + params: + file: mo-expansion.yml + + run-valid-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ + -p 8100 -v + + run-revoked-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ + -p 8100 \ + -v \ + --fault revoked + + run-valid-delegate-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ + -p 8100 -v + + run-revoked-delegate-ocsp-server: + - command: shell.exec + params: + shell: "bash" + background: true + script: | + cd ${DRIVERS_TOOLS}/.evergreen/ocsp + . ./activate-ocspvenv.sh + + python ocsp_mock.py \ + --ca_file ${OCSP_ALGORITHM}/ca.pem \ + --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ + --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ + -p 8100 \ + -v \ + --fault revoked + + run-load-balancer: + - command: shell.exec + params: + shell: "bash" + script: | + DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start + - command: expansions.update + params: + file: lb-expansion.yml + + "export AWS auth credentials": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} + - command: subprocess.exec + type: test + params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run CSOT tests": - command: shell.exec @@ -168,46 +238,10 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi export CSOT_SPEC_TESTS=1 TEST_CMD="bundle exec rspec spec/spec_tests/client_side_operations_timeout_spec.rb" \ .evergreen/run-tests.sh - "export FLE credentials": - - command: shell.exec - type: test - params: - silent: true - working_dir: "src" - script: | - cat < .env.private - MONGO_RUBY_DRIVER_AWS_KEY="${fle_aws_key}" - MONGO_RUBY_DRIVER_AWS_SECRET="${fle_aws_secret}" - MONGO_RUBY_DRIVER_AWS_REGION="${fle_aws_region}" - MONGO_RUBY_DRIVER_AWS_ARN="${fle_aws_arn}" - - MONGO_RUBY_DRIVER_AZURE_TENANT_ID="${fle_azure_tenant_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_ID="${fle_azure_client_id}" - MONGO_RUBY_DRIVER_AZURE_CLIENT_SECRET="${fle_azure_client_secret}" - MONGO_RUBY_DRIVER_AZURE_IDENTITY_PLATFORM_ENDPOINT="${fle_azure_identity_platform_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_VAULT_ENDPOINT="${fle_azure_key_vault_endpoint}" - MONGO_RUBY_DRIVER_AZURE_KEY_NAME="${fle_azure_key_name}" - - MONGO_RUBY_DRIVER_GCP_EMAIL="${fle_gcp_email}" - MONGO_RUBY_DRIVER_GCP_PRIVATE_KEY="${fle_gcp_private_key}" - MONGO_RUBY_DRIVER_GCP_PROJECT_ID="${fle_gcp_project_id}" - MONGO_RUBY_DRIVER_GCP_LOCATION="${fle_gcp_location}" - MONGO_RUBY_DRIVER_GCP_KEY_RING="${fle_gcp_key_ring}" - MONGO_RUBY_DRIVER_GCP_KEY_NAME="${fle_gcp_key_name}" - MONGO_RUBY_DRIVER_MONGOCRYPTD_PORT="${fle_mongocryptd_port}" - EOT - "export Kerberos credentials": - command: shell.exec type: test @@ -338,7 +372,7 @@ functions: "upload test results": - command: attach.xunit_results params: - file: ./src/rspec.xml + file: ./src/tmp/*.xml "delete private environment": - command: shell.exec @@ -370,13 +404,6 @@ functions: working_dir: "src" script: | ${PREPARE_SHELL} - # Needed for generating temporary aws credentials. - if [ -n "${FLE}" ]; - then - export AWS_ACCESS_KEY_ID="${fle_aws_key}" - export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}" - export AWS_DEFAULT_REGION="${fle_aws_region}" - fi .evergreen/run-tests.sh "run AWS auth tests": @@ -408,14 +435,6 @@ functions: script: | ${PREPARE_SHELL} AUTH=${AUTH} SSL=${SSL} TOPOLOGY=${TOPOLOGY} RVM_RUBY="${RVM_RUBY}" \ - ATLAS_REPLICA_SET_URI=${atlas_replica_set_uri} ATLAS_SHARDED_URI=${atlas_sharded_uri} \ - ATLAS_FREE_TIER_URI=${atlas_free_tier_uri} ATLAS_TLS11_URI=${atlas_tls11_uri} \ - ATLAS_TLS12_URI=${atlas_tls12_uri} ATLAS_SERVERLESS_URI=${atlas_serverless_uri} \ - ATLAS_SERVERLESS_LB_URI=${atlas_serverless_lb_uri} \ - ATLAS_X509_CERT_BASE64="${atlas_x509_cert_base64}" \ - ATLAS_X509_URI="${atlas_x509}" \ - ATLAS_X509_DEV_CERT_BASE64="${atlas_x509_dev_cert_base64}" \ - ATLAS_X509_DEV_URI="${atlas_x509_dev}" \ .evergreen/run-tests-atlas.sh pre: @@ -423,11 +442,11 @@ pre: - func: "create expansions" post: + - func: "upload test results" - func: "delete private environment" # Removed, causing timeouts # - func: "upload working dir" - func: "upload mo artifacts" - # - func: "upload test results" - func: "upload test results to s3" task_groups: @@ -616,24 +635,29 @@ tasks: - name: "test-atlas" commands: - func: "run Atlas tests" - - name: "test-mlaunch" + - name: "test-main" commands: + - func: bootstrap-mongo-orchestration - func: "run tests" - name: "driver-bench" commands: + - func: bootstrap-mongo-orchestration - func: "run benchmarks" - name: "test-kerberos" commands: + - func: bootstrap-mongo-orchestration - func: "run Kerberos unit tests" - name: "test-csot" commands: + - func: bootstrap-mongo-orchestration - func: "run CSOT tests" - name: "test-fle" commands: - - func: "export FLE credentials" + - func: bootstrap-mongo-orchestration - func: "run tests" - name: "test-aws-auth" commands: + - func: bootstrap-mongo-orchestration - func: "export AWS auth credentials" - func: "run AWS auth tests" - name: "test-full-atlas-task" diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index d3eedb889d..afc6a80c48 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -1,5 +1,5 @@ <% - topologies = %w( standalone replica-set sharded-cluster ) + topologies = %w( server replica_set sharded_cluster ) # latest_ruby = the most recently released, stable version of Ruby # (make sure this version is being built by 10gen/mongo-ruby-toolchain) @@ -46,7 +46,7 @@ buildvariants: matrix_spec: ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: standalone + topology: server os: ubuntu2204 display_name: DriverBench tasks: @@ -61,7 +61,7 @@ buildvariants: os: ubuntu2204 display_name: ${auth-and-ssl} ${ruby} db-${mongodb-version} ${topology} tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-recent" matrix_spec: @@ -71,7 +71,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${os} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-8-arm" matrix_spec: @@ -81,7 +81,7 @@ buildvariants: os: ubuntu2404-arm display_name: "${mongodb-version} ${os} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-5.x" matrix_spec: @@ -91,7 +91,7 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-4.x" matrix_spec: @@ -101,7 +101,7 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-3.6" matrix_spec: @@ -111,46 +111,46 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "single-lb" matrix_spec: ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: load-balanced + topology: sharded_cluster single-mongos: single-mongos os: ubuntu2204 display_name: "${mongodb-version} ${topology} single-lb ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "mongo-api-version" matrix_spec: ruby: <%= latest_ruby %> mongodb-version: '7.0' - topology: standalone + topology: server api-version-required: yes os: ubuntu2204 display_name: "${mongodb-version} api-version-required ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "single-mongos" matrix_spec: ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: "sharded-cluster" + topology: "sharded_cluster" single-mongos: single-mongos os: ubuntu2204 display_name: "${mongodb-version} ${topology} single-mongos ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: CSOT matrix_spec: ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: replica-set-single-node + topology: replica_set os: ubuntu2204 display_name: "CSOT - ${mongodb-version}" tasks: @@ -165,18 +165,18 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${retry-reads} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "no-retry-writes" matrix_spec: retry-writes: no-retry-writes ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: [replica-set, sharded-cluster] + topology: [replica_set, sharded_cluster] os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${retry-writes} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: mmapv1 matrix_spec: @@ -187,7 +187,7 @@ buildvariants: os: ubuntu1804 display_name: "${mongodb-version} ${topology} mmapv1 ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "lint" matrix_spec: @@ -198,7 +198,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${lint} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "fork" matrix_spec: @@ -209,7 +209,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} fork ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "solo" matrix_spec: @@ -220,40 +220,40 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} solo ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "stress older" matrix_spec: stress: on ruby: <%= supported_mri_ruby_2 %> mongodb-version: ['4.2', '4.0', '3.6'] - topology: replica-set + topology: replica_set os: ubuntu1804 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "stress" matrix_spec: stress: on ruby: <%= latest_ruby %> mongodb-version: <%= recent_mdb %> - topology: replica-set + topology: replica_set os: ubuntu2204 display_name: "${mongodb-version} ${topology} stress ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "x509-tests" matrix_spec: auth-and-ssl: "x509" ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: standalone + topology: server os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: "jruby-auth" matrix_spec: @@ -264,7 +264,7 @@ buildvariants: os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" <% [ [latest_ruby, latest_stable_mdb, 'ubuntu2204'], @@ -276,24 +276,24 @@ buildvariants: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: <%= rubies %> mongodb-version: <%= mdb %> - topology: "replica-set" + topology: "replica_set" compressor: 'zlib' os: <%= distro %> display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: <%= "snappy-#{rubies}" %> matrix_spec: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: <%= rubies %> mongodb-version: <%= mdb %> - topology: "replica-set" + topology: "replica_set" compressor: 'snappy' os: <%= distro %> display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" # the zstd-ruby gem does not support JRuby (explicitly). However, there is # apparently a zstd-jni gem for JRuby that we could investigate here; if @@ -304,41 +304,41 @@ buildvariants: auth-and-ssl: [ "auth-and-ssl", "noauth-and-nossl" ] ruby: <%= rubies %> mongodb-version: <%= mdb %> - topology: "replica-set" + topology: "replica_set" compressor: 'zstd' os: <%= distro %> display_name: "${compressor} ${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: <%= "activesupport-#{rubies}" %> matrix_spec: ruby: <%= rubies %> mongodb-version: <%= mdb %> - topology: replica-set + topology: replica_set as: as os: <%= distro %> display_name: "AS ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" - matrix_name: <%= "bson-#{rubies}" %> matrix_spec: ruby: <%= rubies %> mongodb-version: <%= mdb %> - topology: replica-set + topology: replica_set bson: "*" os: <%= distro %> display_name: "bson-${bson} ${mongodb-version} ${topology} ${ruby}" tasks: - - name: "test-mlaunch" + - name: "test-main" <% end %> - matrix_name: "fle above 4.4" matrix_spec: auth-and-ssl: "noauth-and-nossl" ruby: <%= supported_mri_rubies_3_ubuntu %> - topology: [replica-set, sharded-cluster] + topology: [replica_set, sharded_cluster] mongodb-version: [ '6.0', '7.0', '8.0' ] os: ubuntu2204 fle: helper @@ -358,7 +358,7 @@ buildvariants: matrix_spec: ruby: <%= latest_ruby %> mongodb-version: <%= latest_stable_mdb %> - topology: standalone + topology: server os: ubuntu2204 auth-and-ssl: kerberos display_name: "Kerberos Tests" @@ -369,7 +369,7 @@ buildvariants: # matrix_spec: # auth-and-ssl: "noauth-and-nossl" # ruby: <%#= latest_ruby %> -# topology: [replica-set, sharded-cluster] +# topology: [replica_set, sharded_cluster] # mongodb-version: [ 'latest' ] # os: ubuntu2204 # fle: helper @@ -385,7 +385,7 @@ buildvariants: # https://jira.mongodb.org/browse/RUBY-3659 auth-and-ssl: [ aws-regular, aws-assume-role, aws-web-identity ] ruby: <%= latest_ruby %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" @@ -397,12 +397,12 @@ buildvariants: ocsp-verifier: true # No JRuby due to https://github.com/jruby/jruby-openssl/issues/210 ruby: <%= supported_mri_rubies_3_ubuntu %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 display_name: "OCSP verifier: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-must-staple matrix_spec: @@ -410,26 +410,26 @@ buildvariants: ocsp-must-staple: on ocsp-delegate: on ruby: <%= latest_ruby %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - must staple: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main - matrix_name: ocsp-unknown matrix_spec: ocsp-algorithm: rsa ocsp-status: unknown ruby: <%= latest_ruby %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 auth-and-ssl: noauth-and-ssl display_name: "OCSP integration - unknown: ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main <% [ %w(valid none pass), @@ -453,12 +453,12 @@ buildvariants: ocsp-connectivity: <%= outcome %> extra-uri-options: "<%= extra_uri_options %>" ruby: <%= latest_ruby %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${extra-uri-options} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main <% end %> - matrix_name: ocsp-connectivity-jruby @@ -474,19 +474,19 @@ buildvariants: ocsp-delegate: '*' ocsp-connectivity: pass ruby: <%= jrubies.first %> - topology: standalone + topology: server mongodb-version: <%= latest_stable_mdb %> os: ubuntu2204 display_name: "OCSP connectivity: ${ocsp-algorithm} ${ocsp-status} ${ocsp-delegate} ${mongodb-version} ${ruby}" tasks: - - name: test-mlaunch + - name: test-main # https://jira.mongodb.org/browse/RUBY-3540 #- matrix_name: testgcpkms-variant # matrix_spec: # ruby: <%= latest_ruby %> # fle: helper - # topology: standalone + # topology: server # os: ubuntu2204 # mongodb-version: <%= latest_stable_mdb %> # display_name: "GCP KMS" @@ -499,7 +499,7 @@ buildvariants: # matrix_spec: # ruby: ruby-3.0 # fle: helper - # topology: standalone + # topology: server # os: debian11 # could eventually look at updating this to rhel80 # mongodb-version: 6.0 # display_name: "AZURE KMS" diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6dfa54c343..07d1a5b4e0 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -42,144 +42,16 @@ show_local_instructions set_home set_env_vars -set_env_python set_env_ruby -prepare_server - -if test "$DOCKER_PRELOAD" != 1; then - install_mlaunch_venv -fi - # Make sure cmake is installed (in case we need to install the libmongocrypt # helper) if [ "$FLE" = "helper" ]; then install_cmake fi -if test "$TOPOLOGY" = load-balanced; then - install_haproxy -fi - -# Launching mongod under $MONGO_ORCHESTRATION_HOME -# makes its log available through log collecting machinery - -export dbdir="$MONGO_ORCHESTRATION_HOME"/db -mkdir -p "$dbdir" - -if test -z "$TOPOLOGY"; then - export TOPOLOGY=standalone -fi - -calculate_server_args -launch_ocsp_mock - -launch_server "$dbdir" - -uri_options="$URI_OPTIONS" - bundle_install -if test "$TOPOLOGY" = sharded-cluster; then - if test -n "$SINGLE_MONGOS"; then - # Some tests may run into https://jira.mongodb.org/browse/SERVER-16836 - # when executing against a multi-sharded mongos. - # At the same time, due to pinning in sharded transactions, - # it is beneficial to test a single shard to ensure that server - # monitoring and selection are working correctly and recover the driver's - # ability to operate in reasonable time after errors and fail points trigger - # on a single shard - echo Restricting to a single mongos - hosts=localhost:27017 - else - hosts=localhost:27017,localhost:27018 - fi -elif test "$TOPOLOGY" = replica-set; then - # To set FCV we use mongo shell, it needs to be placed in replica set topology - # or it can try to send the commands to secondaries. - hosts=localhost:27017,localhost:27018 - uri_options="$uri_options&replicaSet=test-rs" -elif test "$TOPOLOGY" = replica-set-single-node; then - hosts=localhost:27017 - uri_options="$uri_options&replicaSet=test-rs" -else - hosts=localhost:27017 -fi - -if test "$AUTH" = auth; then - hosts="bob:pwd123@$hosts" -elif test "$AUTH" = x509; then - create_user_cmd="`cat <<'EOT' - db.getSiblingDB("$external").runCommand( - { - createUser: "C=US,ST=New York,L=New York City,O=MongoDB,OU=x509,CN=localhost", - roles: [ - { role: "root", db: "admin" }, - ], - writeConcern: { w: "majority" , wtimeout: 5000 }, - } - ) -EOT - `" - - "$BINDIR"/mongosh --tls \ - --tlsCAFile spec/support/certificates/ca.crt \ - --tlsCertificateKeyFile spec/support/certificates/client-x509.pem \ - -u bootstrap -p bootstrap \ - --eval "$create_user_cmd" -elif test "$AUTH" = aws-regular; then - clear_instance_profile - - ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth - - hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts" -elif test "$AUTH" = aws-assume-role; then - clear_instance_profile - - ./.evergreen/aws -a "$MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID" \ - -s "$MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY" \ - -r us-east-1 \ - assume-role "$MONGO_RUBY_DRIVER_AWS_AUTH_ASSUME_ROLE_ARN" >.env.private.gen - eval `cat .env.private.gen` - export MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - export MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - export MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN=$AWS_SESSION_TOKEN - ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth - - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - export AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN - - aws sts get-caller-identity - - hosts="`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_ACCESS_KEY_ID`:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SECRET_ACCESS_KEY`@$hosts" - - uri_options="$uri_options&"\ -"authMechanismProperties=AWS_SESSION_TOKEN:`uri_escape $MONGO_RUBY_DRIVER_AWS_AUTH_SESSION_TOKEN`" -elif test "$AUTH" = aws-ec2; then - ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth - - # We need to assign an instance profile to the current instance, otherwise - # since we don't place credentials into the environment the test suite - # cannot connect to the MongoDB server while bootstrapping. - # The EC2 credential retrieval tests clears the instance profile as part - # of one of the tests. - ruby -Ispec -Ilib -I.evergreen/lib -rec2_setup -e Ec2Setup.new.assign_instance_profile -elif test "$AUTH" = aws-ecs; then - if test -z "$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; then - # drivers-evergreen-tools performs this operation in its ECS E2E tester. - eval export `strings /proc/1/environ |grep ^AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` - fi - - ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth -elif test "$AUTH" = aws-web-identity; then - clear_instance_profile - - ruby -Ilib -I.evergreen/lib -rserver_setup -e ServerSetup.new.setup_aws_auth -elif test "$AUTH" = kerberos; then - export MONGO_RUBY_DRIVER_KERBEROS=1 -fi - if test -n "$FLE"; then # Downloading crypt shared lib if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then @@ -200,30 +72,6 @@ if test -n "$FLE"; then cd - fi - # Start the KMS servers first so that they are launching while we are - # fetching libmongocrypt. - if test "$DOCKER_PRELOAD" != 1; then - # We already have a virtualenv activated for mlaunch, - # install kms dependencies into it. - #. .evergreen/csfle/activate_venv.sh - - # Adjusted package versions: - # cryptography 3.4 requires rust, see - # https://github.com/pyca/cryptography/issues/5771. - #pip install boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0 - pip3 install boto3~=1.19 'cryptography<3.4' pykmip~=0.10.0 'sqlalchemy<2.0.0' - fi - python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 & - python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/expired.pem --port 8000 & - python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/wrong-host.pem --port 8001 & - python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert & - python3 -u .evergreen/csfle/kms_kmip_server.py & - python3 -u .evergreen/csfle/fake_azure.py & - python3 -u .evergreen/csfle/kms_failpoint_server.py --port 9003 & - - # Obtain temporary AWS credentials - PYTHON=python3 . .evergreen/csfle/set-temp-creds.sh - if test "$FLE" = helper; then echo "Using helper gem" elif test "$FLE" = path; then @@ -258,50 +106,9 @@ if test -n "$FLE"; then echo "Unknown FLE value: $FLE" 1>&2 exit 1 fi - - echo "Waiting for mock KMS servers to start..." - wait_for_kms_server() { - for i in $(seq 60); do - if curl -s "localhost:$1"; test $? -ne 7; then - return 0 - else - sleep 1 - fi - done - echo "Could not detect mock KMS server on port $1" - return 1 - } - wait_for_kms_server 8000 - wait_for_kms_server 8001 - wait_for_kms_server 8002 - wait_for_kms_server 5698 - wait_for_kms_server 8080 - echo "Waiting for mock KMS servers to start... done." -fi - -if test -n "$OCSP_CONNECTIVITY"; then - # TODO Maybe OCSP_CONNECTIVITY=* should set SSL=ssl instead. - uri_options="$uri_options&tls=true" -fi - -if test -n "$EXTRA_URI_OPTIONS"; then - uri_options="$uri_options&$EXTRA_URI_OPTIONS" -fi - -export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options" - -if echo "$AUTH" |grep -q ^aws-assume-role; then - $BINDIR/mongosh "$MONGODB_URI" --eval 'db.runCommand({serverStatus: 1})' | wc -fi - -set_fcv - -if test "$TOPOLOGY" = replica-set || test "$TOPOLOGY" = replica-set-single-node; then - ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.setup_tags fi if test "$API_VERSION_REQUIRED" = 1; then - ruby -Ilib -I.evergreen/lib -rbundler/setup -rserver_setup -e ServerSetup.new.require_api_version export SERVER_API='version: "1"' fi @@ -310,14 +117,6 @@ if ! test "$OCSP_VERIFIER" = 1 && ! test -n "$OCSP_CONNECTIVITY"; then bundle exec rake spec:prepare fi -if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then - # On 3.6 server the sessions collection is not immediately available, - # wait for it to spring into existence - bundle exec rake spec:wait_for_sessions -fi - -export MONGODB_URI="mongodb://$hosts/?appName=test-suite$uri_options" - # Compression is handled via an environment variable, convert to URI option if test "$COMPRESSOR" = zlib && ! echo $MONGODB_URI |grep -q compressors=; then add_uri_option compressors=zlib @@ -331,7 +130,6 @@ if test "$COMPRESSOR" = zstd; then add_uri_option compressors=zstd fi - echo "Running tests" set +e if test -n "$TEST_CMD"; then @@ -376,12 +174,4 @@ if test -n "$OCSP_MOCK_PID"; then kill "$OCSP_MOCK_PID" fi -python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir" || true - -if test -n "$FLE" && test "$DOCKER_PRELOAD" != 1; then - # Terminate all kmip servers... and whatever else happens to be running - # that is a python script. - pkill python3 || true -fi - exit ${test_status} diff --git a/.gitmodules b/.gitmodules index e1bfe10123..4359ab5ade 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule ".mod/drivers-evergreen-tools"] path = .mod/drivers-evergreen-tools url = https://github.com/mongodb-labs/drivers-evergreen-tools -[submodule "spec/shared"] - path = spec/shared - url = https://github.com/mongodb-labs/mongo-ruby-spec-shared diff --git a/.rspec b/.rspec index 575f1e8769..10123e05fb 100644 --- a/.rspec +++ b/.rspec @@ -1,3 +1,9 @@ --tty --colour ---format <%= %w(1 true yes).include?(ENV['CI']&.downcase) ? 'Rfc::Riff' : 'Fuubar'%> +<% if %w(1 true yes).include?(ENV['CI']&.downcase) %> +--format 'Rfc::Riff' +--format RspecJunitFormatter +--out tmp/rspec.xml +<% else %> +--format Fuubar +<% end %> diff --git a/gemfiles/standard.rb b/gemfiles/standard.rb index c8065b3a1b..ded50fc70e 100644 --- a/gemfiles/standard.rb +++ b/gemfiles/standard.rb @@ -54,6 +54,7 @@ def standard_dependencies gem 'concurrent-ruby', platforms: :jruby gem 'dotenv' gem 'childprocess' + gem "rspec_junit_formatter", require: false end group :development do diff --git a/spec/shared b/spec/shared deleted file mode 160000 index 1017c94e4b..0000000000 --- a/spec/shared +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1017c94e4b0962d3b68eced52566e700ae4e70b4 diff --git a/spec/shared/LICENSE b/spec/shared/LICENSE new file mode 100644 index 0000000000..08c1768ef3 --- /dev/null +++ b/spec/shared/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2020 MongoDB, Inc. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/spec/shared/bin/get-mongodb-download-url b/spec/shared/bin/get-mongodb-download-url new file mode 100755 index 0000000000..6c860280ad --- /dev/null +++ b/spec/shared/bin/get-mongodb-download-url @@ -0,0 +1,17 @@ +#!/usr/bin/env ruby + +desired_version, arch = ARGV +if arch.nil? + STDERR.puts "Usage: get-mongodb-download-url desired-version arch" + exit 1 +end + +$: << File.join(File.dirname(__FILE__), '../lib') +require 'mrss/server_version_registry' + +begin + puts Mrss::ServerVersionRegistry.new(desired_version, arch).download_url +rescue Mrss::ServerVersionRegistry::Error => exc + STDERR.puts "Error: #{exc}" + exit 2 +end diff --git a/spec/shared/bin/s3-copy b/spec/shared/bin/s3-copy new file mode 100755 index 0000000000..78023d306f --- /dev/null +++ b/spec/shared/bin/s3-copy @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby + +require 'optparse' +require 'aws-sdk-s3' + +options = {} +OptionParser.new do |opts| + opts.banner = "Usage: s3-copy options" + + opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v| + options[:region] = v + end + + opts.on("-p", "--param=KEY=VALUE", "Specify parameter for new files") do |v| + options[:params] ||= {} + k, v = v.split('=', 2) + options[:params][k.to_sym] = v + end + + opts.on("-f", "--from=BUCKET:PATH", "Bucket name and key (or path) to copy from") do |v| + options[:from] = v + end + + opts.on("-t", "--to=BUCKET:PATH", "Bucket name and key (or path) to write to (may be specified more than once)") do |v| + options[:to] ||= [] + options[:to] << v + end +end.parse! + +ENV['AWS_REGION'] ||= options[:region] || 'us-east-1' + +bucket, key = options.fetch(:from).split(':', 2) + +s3 = Aws::S3::Client.new + +options.fetch(:to).each do |dest| + STDERR.puts "Copying to #{dest}" + dbucket, dkey = dest.split(':', 2) + s3.copy_object( + bucket: dbucket, + key: dkey, + copy_source: "/#{bucket}/#{key}", + **options[:params] || {}, + ) +end diff --git a/spec/shared/bin/s3-upload b/spec/shared/bin/s3-upload new file mode 100755 index 0000000000..95846f8cea --- /dev/null +++ b/spec/shared/bin/s3-upload @@ -0,0 +1,69 @@ +#!/usr/bin/env ruby + +require 'optparse' +require 'aws-sdk-s3' + +options = {} +OptionParser.new do |opts| + opts.banner = "Usage: s3-upload options" + + opts.on("-r", "--region=REGION", "AWS region to use (default us-east-1)") do |v| + options[:region] = v + end + + opts.on("-p", "--param=KEY=VALUE", "Specify parameter for S3 upload") do |v| + options[:params] ||= {} + k, v = v.split('=', 2) + options[:params][k.to_sym] = v + end + + opts.on("-f", "--file=PATH", "Path to the file to upload, - to upload standard input") do |v| + options[:file] = v + end + + opts.on("-w", "--write=BUCKET:PATH", "Bucket name and key (or path) to upload to") do |v| + options[:write] = v + end + + opts.on("-c", "--copy=BUCKET:PATH", "Bucket name and key (or path) to copy to (may be specified more than once)") do |v| + options[:copy] ||= [] + options[:copy] << v + end +end.parse! + +ENV['AWS_REGION'] ||= options[:region] || 'us-east-1' + +def upload(f, options) + s3 = Aws::S3::Client.new + write = options.fetch(:write) + STDERR.puts "Writing #{write}" + bucket, key = write.split(':', 2) + s3.put_object( + body: f.read, + bucket: bucket, + key: key, + **options[:params] || {}, + ) + if copy = options[:copy] + copy.each do |dest| + STDERR.puts "Copying to #{dest}" + dbucket, dkey = dest.split(':', 2) + s3.copy_object( + bucket: dbucket, + key: dkey, + copy_source: "/#{bucket}/#{key}", + **options[:params] || {}, + ) + end + end +end + +if options[:file] == '-' + upload(STDIN, options) +elsif options[:file] + File.open(options[:file]) do |f| + upload(f, options) + end +else + upload(STDIN, options) +end diff --git a/spec/shared/lib/mrss/child_process_helper.rb b/spec/shared/lib/mrss/child_process_helper.rb new file mode 100644 index 0000000000..3e75170382 --- /dev/null +++ b/spec/shared/lib/mrss/child_process_helper.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true +# encoding: utf-8 + +autoload :ChildProcess, 'childprocess' +autoload :Tempfile, 'tempfile' + +module Mrss + module ChildProcessHelper + class SpawnError < StandardError; end + + module_function def call(cmd, env: nil, cwd: nil) + process = ChildProcess.new(*cmd) + process.io.inherit! + if cwd + process.cwd = cwd + end + if env + env.each do |k, v| + process.environment[k.to_s] = v + end + end + process.start + process.wait + process + end + + module_function def check_call(cmd, env: nil, cwd: nil) + process = call(cmd, env: env, cwd: cwd) + unless process.exit_code == 0 + raise SpawnError, "Failed to execute: #{cmd}" + end + end + + module_function def get_output(cmd, env: nil, cwd: nil) + process = ChildProcess.new(*cmd) + process.io.inherit! + if cwd + process.cwd = cwd + end + if env + env.each do |k, v| + process.environment[k.to_s] = v + end + end + + output = '' + r, w = IO.pipe + + begin + process.io.stdout = w + process.start + w.close + + thread = Thread.new do + begin + loop do + output << r.readpartial(16384) + end + rescue EOFError + end + end + + process.wait + thread.join + ensure + r.close + end + + [process, output] + end + + module_function def check_output(*args) + process, output = get_output(*args) + unless process.exit_code == 0 + raise SpawnError,"Failed to execute: #{args}" + end + output + end + end +end diff --git a/spec/shared/lib/mrss/cluster_config.rb b/spec/shared/lib/mrss/cluster_config.rb new file mode 100644 index 0000000000..99e7d99385 --- /dev/null +++ b/spec/shared/lib/mrss/cluster_config.rb @@ -0,0 +1,231 @@ +# frozen_string_literal: true +# encoding: utf-8 + +# ClusterConfig requires ClientRegistry class provided by the host project. + +require 'singleton' + +module Mrss + class ClusterConfig + include Singleton + include RSpec::Core::Pending + + def single_server? + determine_cluster_config + @single_server + end + + def sharded_ish? + determine_cluster_config + @topology == :sharded || @topology == :load_balanced + end + + def replica_set_name + determine_cluster_config + @replica_set_name + end + + def server_version + determine_cluster_config + @server_version + end + + def enterprise? + determine_cluster_config + @enterprise + end + + def short_server_version + server_version.split('.')[0..1].join('.') + end + + def fcv + determine_cluster_config + @fcv + end + + # Per https://jira.mongodb.org/browse/SERVER-39052, working with FCV + # in sharded topologies is annoying. Also, FCV doesn't exist in servers + # less than 3.4. This method returns FCV on 3.4+ servers when in single + # or RS topologies, and otherwise returns the major.minor server version. + def fcv_ish + if server_version.nil? + raise "Deployment server version not known - check that connection to deployment succeeded" + end + + if server_version >= '3.4' && !sharded_ish? + fcv + else + if short_server_version == '4.1' + '4.2' + else + short_server_version + end + end + end + + # @return [ Mongo::Address ] The address of the primary in the deployment. + def primary_address + determine_cluster_config + @primary_address + end + + def primary_address_str + determine_cluster_config + @primary_address.seed + end + + def primary_address_host + both = primary_address_str + both.split(':').first + end + + def primary_address_port + both = primary_address_str + both.split(':')[1] || 27017 + end + + def primary_description + determine_cluster_config + @primary_description + end + + def server_parameters + determine_cluster_config + @server_parameters + end + + # Try running a command on the admin database to see if the mongod was + # started with auth. + def auth_enabled? + if @auth_enabled.nil? + @auth_enabled = begin + basic_client.use(:admin).command(getCmdLineOpts: 1).first["argv"].include?("--auth") + rescue => e + e.message =~ /(not authorized)|(unauthorized)|(no users authenticated)|(requires authentication)/ + end + end + @auth_enabled + end + + def topology + determine_cluster_config + @topology + end + + def storage_engine + @storage_engine ||= begin + # 2.6 does not have wired tiger + if short_server_version == '2.6' + :mmapv1 + else + client = ClientRegistry.instance.global_client('root_authorized') + if sharded_ish? + shards = client.use(:admin).command(listShards: 1).first + if shards['shards'].empty? + raise 'Shards are empty' + end + shard = shards['shards'].first + address_str = shard['host'].sub(/^.*\//, '').sub(/,.*/, '') + client = ClusterTools.instance.direct_client(address_str, + SpecConfig.instance.test_options.merge(SpecConfig.instance.auth_options).merge(connect: :direct)) + end + rv = client.use(:admin).command(serverStatus: 1).first + rv = rv['storageEngine']['name'] + rv_map = { + 'wiredTiger' => :wired_tiger, + 'mmapv1' => :mmapv1, + } + rv_map[rv] || rv + end + end + end + + # This method returns an alternate address for connecting to the configured + # deployment. For example, if the replica set is configured with nodes at + # of localhost:27017 and so on, this method will return 127.0.0.:27017. + # + # Note that the "alternate" refers to replica set configuration, not the + # addresses specified in test suite configuration. If the deployment topology + # is not a replica set, "alternate" refers to test suite configuration as + # this is the only configuration available. + def alternate_address + @alternate_address ||= begin + address = primary_address_host + str = case address + when '127.0.0.1' + 'localhost' + when /^(\d+\.){3}\d+$/ + skip 'This test requires a hostname or 127.0.0.1 as address' + else + # We don't know if mongod is listening on ipv4 or ipv6, in principle. + # Our tests use ipv4, so hardcode that for now. + # To support both we need to try both addresses which will make this + # test more complicated. + # + # JRuby chokes on primary_address_port as the port (e.g. 27017). + # Since the port does not actually matter, use a common port like 80. + resolved_address = Addrinfo.getaddrinfo(address, 80, Socket::PF_INET).first.ip_address + if resolved_address.include?(':') + "[#{resolved_address}]" + else + resolved_address + end + end + ":#{primary_address_port}" + Mongo::Address.new(str) + end + end + + private + + def determine_cluster_config + return if @primary_address + + # Run all commands to figure out the cluster configuration from the same + # client. This is somewhat wasteful when running a single test, but reduces + # test runtime for the suite overall because all commands are sent on the + # same connection rather than each command connecting to the cluster by + # itself. + client = ClientRegistry.instance.global_client('root_authorized') + + primary = client.cluster.next_primary + @primary_address = primary.address + @primary_description = primary.description + @replica_set_name = client.cluster.topology.replica_set_name + + @topology ||= begin + topology = client.cluster.topology.class.name.sub(/.*::/, '') + topology = topology.gsub(/([A-Z])/) { |match| '_' + match.downcase }.sub(/^_/, '') + if topology =~ /^replica_set/ + topology = 'replica_set' + end + topology.to_sym + end + + @single_server = client.cluster.servers_list.length == 1 + + build_info = client.database.command(buildInfo: 1).first + + @server_version = build_info['version'] + @enterprise = build_info['modules'] && build_info['modules'].include?('enterprise') + + @server_parameters = begin + client.use(:admin).command(getParameter: '*').first + rescue => e + STDERR.puts("WARNING: Failed to obtain server parameters: #{e.class}: #{e.message}") + {} + end + + if !sharded_ish? && short_server_version >= '3.4' + rv = @server_parameters['featureCompatibilityVersion'] + @fcv = rv['version'] || rv + end + end + + def basic_client + # Do not cache the result here so that if the client gets closed, + # client registry reconnects it in subsequent tests + ClientRegistry.instance.global_client('basic') + end + end +end diff --git a/spec/shared/lib/mrss/constraints.rb b/spec/shared/lib/mrss/constraints.rb new file mode 100644 index 0000000000..ed6d854ba0 --- /dev/null +++ b/spec/shared/lib/mrss/constraints.rb @@ -0,0 +1,378 @@ +# frozen_string_literal: true +# encoding: utf-8 + +module Mrss + module Constraints + def min_server_version(version) + parsed_version = Gem::Version.new(version) + + before(:all) do + if parsed_version > Gem::Version.new(ClusterConfig.instance.server_version) + skip "Server version #{version} or higher required, we have #{ClusterConfig.instance.server_version}" + end + end + end + + def max_server_version(version) + parsed_version = Gem::Version.new(version) + + before(:all) do + if parsed_version < Gem::Version.new(ClusterConfig.instance.server_version) + skip "Server version #{version} or lower required, we have #{ClusterConfig.instance.server_version}" + end + end + end + + def min_server_fcv(version) + parsed_version = Gem::Version.new(version) + + before(:all) do + unless Gem::Version.new(ClusterConfig.instance.fcv_ish) >= parsed_version + skip "FCV #{version} or higher required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})" + end + end + end + + def max_server_fcv(version) + parsed_version = Gem::Version.new(version) + + before(:all) do + if parsed_version < Gem::Version.new(ClusterConfig.instance.fcv_ish) + skip "FCV #{version} or lower required, we have #{ClusterConfig.instance.fcv_ish} (server #{ClusterConfig.instance.server_version})" + end + end + end + + def require_topology(*topologies) + invalid_topologies = topologies - [:single, :replica_set, :sharded, :load_balanced] + + unless invalid_topologies.empty? + raise ArgumentError, "Invalid topologies requested: #{invalid_topologies.join(', ')}" + end + + before(:all) do + unless topologies.include?(topology = ClusterConfig.instance.topology) + skip "Topology #{topologies.join(' or ')} required, we have #{topology}" + end + end + end + + def max_example_run_time(timeout) + around do |example| + TimeoutInterrupt.timeout(timeout, TimeoutInterrupt::Error.new("Test execution terminated after #{timeout} seconds")) do + example.run + end + end + end + + def require_transaction_support + before(:all) do + case ClusterConfig.instance.topology + when :single + skip 'Transactions tests require a replica set (4.0+) or a sharded cluster (4.2+)' + when :replica_set + unless ClusterConfig.instance.server_version >= '4.0' + skip 'Transactions tests in a replica set topology require server 4.0+' + end + when :sharded, :load_balanced + unless ClusterConfig.instance.server_version >= '4.2' + skip 'Transactions tests in a sharded cluster topology require server 4.2+' + end + else + raise NotImplementedError + end + end + end + + # Fail command fail point was added to mongod in 4.0 and to mongos in 4.2. + def require_fail_command + require_transaction_support + end + + def require_tls + before(:all) do + unless SpecConfig.instance.ssl? + skip "SSL not enabled" + end + end + end + + def require_no_tls + before(:all) do + if SpecConfig.instance.ssl? + skip "SSL enabled" + end + end + end + + def require_retry_writes + before(:all) do + unless SpecConfig.instance.retry_writes? + skip "Retry writes is disabled" + end + end + end + + def require_no_retry_writes + before(:all) do + if SpecConfig.instance.retry_writes? + skip "Retry writes is enabled" + end + end + end + + def require_compression + before(:all) do + if SpecConfig.instance.compressors.nil? + skip "Compression is not enabled" + end + end + end + + def require_zlib_compression + before(:all) do + compressors = SpecConfig.instance.compressors + unless compressors && compressors.include?('zlib') + skip "Zlib compression is not enabled" + end + end + end + + def require_snappy_compression + before(:all) do + compressors = SpecConfig.instance.compressors + unless compressors && compressors.include?('snappy') + skip "Snappy compression is not enabled" + end + end + end + + def require_no_snappy_compression + before(:all) do + compressors = SpecConfig.instance.compressors + if compressors && compressors.include?('snappy') + skip "Snappy compression is enabled" + end + end + end + + def require_zstd_compression + before(:all) do + compressors = SpecConfig.instance.compressors + unless compressors && compressors.include?('zstd') + skip "Zstd compression is not enabled" + end + end + end + + def require_no_zstd_compression + before(:all) do + compressors = SpecConfig.instance.compressors + if compressors && compressors.include?('zstd') + skip "Zstd compression is enabled" + end + end + end + + def require_no_compression + before(:all) do + if SpecConfig.instance.compressors + skip "Compression is enabled" + end + end + end + + def ruby_version_gte(version) + before(:all) do + if RUBY_VERSION < version + skip "Ruby version #{version} or higher required" + end + end + end + + def ruby_version_lt(version) + before(:all) do + if RUBY_VERSION >= version + skip "Ruby version less than #{version} required" + end + end + end + + def require_auth(*values) + before(:all) do + if values.any? + unless values.include?(ENV['AUTH']) + msg = values.map { |v| "AUTH=#{v}" }.join(' or ') + skip "This test requires #{msg}" + end + else + unless ENV['AUTH'] == 'auth' || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled? + skip "Auth required" + end + end + end + end + + def require_no_auth + before(:all) do + auth = ENV.fetch('AUTH', '') + if (!auth.empty? && auth != 'noauth') || SpecConfig.instance.user || ClusterConfig.instance.auth_enabled? + skip "Auth not allowed" + end + end + end + + def require_x509_auth + before(:all) do + unless SpecConfig.instance.x509_auth? + skip "X.509 auth required" + end + end + end + + def require_no_external_user + before(:all) do + if SpecConfig.instance.external_user? + skip "External user configurations are not compatible with this test" + end + end + end + + # Can the driver specify a write concern that won't be overridden? + # (mongos 4.0+ overrides the write concern) + def require_set_write_concern + before(:all) do + if %i(sharded load_balanced).include?(ClusterConfig.instance.topology) && + ClusterConfig.instance.short_server_version >= '4.0' + then + skip "mongos 4.0+ overrides write concern" + end + end + end + + def require_multi_mongos + before(:all) do + if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length == 1 + skip 'Test requires a minimum of two mongoses if run in sharded topology' + end + + if ClusterConfig.instance.topology == :load_balanced && SpecConfig.instance.single_mongos? + skip 'Test requires a minimum of two mongoses if run in load-balanced topology' + end + end + end + + # In sharded topology operations are distributed to the mongoses. + # When we set fail points, the fail point may be set on one mongos and + # operation may be executed on another mongos, causing failures. + # Tests that are not setting targeted fail points should utilize this + # method to restrict themselves to single mongos. + # + # In load-balanced topology, the same problem can happen when there is + # more than one mongos behind the load balancer. + def require_no_multi_mongos + before(:all) do + if ClusterConfig.instance.topology == :sharded && SpecConfig.instance.addresses.length > 1 + skip 'Test requires a single mongos if run in sharded topology' + end + if ClusterConfig.instance.topology == :load_balanced && !SpecConfig.instance.single_mongos? + skip 'Test requires a single mongos, as indicated by SINGLE_MONGOS=1 environment variable, if run in load-balanced topology' + end + end + end + + alias :require_no_multi_shard :require_no_multi_mongos + + def require_wired_tiger + before(:all) do + # Storage detection fails for serverless instances. However, it is safe to + # assume that a serverless instance uses WiredTiger Storage Engine. + if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger + skip 'Test requires WiredTiger storage engine' + end + end + end + + def require_wired_tiger_on_36 + before(:all) do + if ClusterConfig.instance.short_server_version >= '3.6' + # Storage detection fails for serverless instances. However, it is safe to + # assume that a serverless instance uses WiredTiger Storage Engine. + if !SpecConfig.instance.serverless? && ClusterConfig.instance.storage_engine != :wired_tiger + skip 'Test requires WiredTiger storage engine on 3.6+ servers' + end + end + end + end + + def require_mmapv1 + before(:all) do + if SpecConfig.instance.serverless? || ClusterConfig.instance.storage_engine != :mmapv1 + skip 'Test requires MMAPv1 storage engine' + end + end + end + + def require_enterprise + before(:all) do + unless ClusterConfig.instance.enterprise? + skip 'Test requires enterprise build of MongoDB' + end + end + end + + # Integration tests for SRV polling require internet connectivity to + # look up SRV records and a sharded cluster configured on default port on + # localhost (localhost:27017, localhost:27018). + def require_default_port_deployment + # Because the DNS records at test1.test.build.10gen.cc point to + # localhost:27017 & localhost:27018, the test suite must have been + # configured to use these addresses + before(:all) do + have_default_port = SpecConfig.instance.addresses.any? do |address| + %w(127.0.0.1 127.0.0.1:27017 localhost localhost:27017).include?(address) + end + unless have_default_port + skip 'This test requires the test suite to be configured for localhost:27017' + end + end + end + + # Some tests perform assertions on what the driver is logging. + # Some test configurations, for example OCSP with unknown response, + # produce warnings due to optional checks failing. + # This constraint skips tests that issue logging assertions on configurations + # that may produce non-test-originated log entries. + def require_warning_clean + before(:all) do + if ENV['OCSP_STATUS'] == 'unknown' + skip 'Unknown OCSP status is not global warning-clean' + end + end + end + + def require_required_api_version + before(:all) do + unless ENV['API_VERSION_REQUIRED'] == '1' + skip 'Set API_VERSION_REQUIRED=1 to run this test' + end + end + end + + def require_no_required_api_version + before(:all) do + if ENV['API_VERSION_REQUIRED'] == '1' + skip 'Cannot have API_VERSION_REQUIRED=1 to run this test' + end + end + end + + def require_unix_socket + before(:all) do + if ENV['TOPOLOGY'] == 'load-balanced' + skip 'Load balancer does not listen on Unix sockets' + end + end + end + end +end diff --git a/spec/shared/lib/mrss/docker_runner.rb b/spec/shared/lib/mrss/docker_runner.rb new file mode 100644 index 0000000000..4177066d29 --- /dev/null +++ b/spec/shared/lib/mrss/docker_runner.rb @@ -0,0 +1,298 @@ +# frozen_string_literal: true +# encoding: utf-8 + +require 'optparse' +require 'erb' +autoload :Dotenv, 'dotenv' + +module Mrss + autoload :ServerVersionRegistry, 'mrss/server_version_registry' + + class DockerRunner + def initialize(**opts) + # These options are required: + opts.fetch(:image_tag) + opts.fetch(:dockerfile_path) + opts.fetch(:default_script) + opts.fetch(:project_lib_subdir) + + @options = opts.merge(preload: true) + end + + attr_reader :options + + def run + process_arguments + unless @options[:exec_only] + create_dockerfile + create_image + end + if @options[:mongo_only] + run_deployment + else + run_tests + end + end + + private + + def process_arguments + #@options = {} + OptionParser.new do |opts| + opts.banner = "Usage: test-on-docker [-d distro] [evergreen_key=value ...]" + + opts.on("-a", "--add-env=PATH", "Load environment variables from PATH in .env format") do |path| + @options[:extra_env] ||= {} + unless File.exist?(path) + raise "-a option references nonexistent file #{path}" + end + Dotenv.parse(path).each do |k, v| + @options[:extra_env][k] = v + end + end + + opts.on("-d", "--distro=DISTRO", "Distro to use") do |v| + @options[:distro] = v + end + + opts.on('-e', '--exec-only', 'Execute tests using existing Dockerfile (for offline user)') do |v| + @options[:exec_only] = v + end + + opts.on('-m', '--mongo-only=PORT', 'Start the MongoDB deployment and expose it to host on ports starting with PORT') do |v| + @options[:mongo_only] = v.to_i + end + + opts.on('-p', '--preload', 'Preload Ruby toolchain and server binaries in docker (default)') do |v| + @options[:preload] = v + end + + opts.on('-P', '--no-preload', 'Do not preload Ruby toolchain and server binaries in docker') do + @options[:preload] = false + end + + opts.on('-s', '--script=SCRIPT', 'Test script to invoke') do |v| + @options[:script] = v + end + + opts.on('-i', '--interactive', 'Interactive mode - disable per-test timeouts') do |v| + @options[:interactive] = v + end + end.parse! + + @env = Hash[ARGV.map do |arg| + arg.split('=', 2) + end] + + @env['RVM_RUBY'] ||= 'ruby-2.7' + unless ruby =~ /^j?ruby-/ + raise "RVM_RUBY option is not in expected format: #{ruby}" + end + + @env['MONGODB_VERSION'] ||= '4.4' + end + + def create_dockerfile + template_path = File.join(File.dirname(__FILE__), '../../share/Dockerfile.erb') + result = ERB.new(File.read(template_path)).result(binding) + File.open(dockerfile_path, 'w') do |f| + f << result + end + end + + def image_tag + options.fetch(:image_tag) + end + + def dockerfile_path + options.fetch(:dockerfile_path) + end + + def create_image + run_command(['docker', 'build', + '-t', image_tag, + '-f', dockerfile_path, + '.']) + end + + BASE_TEST_COMMAND = %w(docker run --rm -i --tmpfs /tmpfs:exec).freeze + + def run_tests + run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [image_tag] + + script.split(/\s+/)) + end + + def run_deployment + run_command(BASE_TEST_COMMAND + tty_arg + extra_env + [ + '-e', %q`TEST_CMD=watch -x bash -c "ps awwxu |egrep 'mongo|ocsp'"`, + '-e', 'BIND_ALL=true', + ] + port_forwards + [image_tag] + script.split(/\s+/)) + end + + def tty_arg + tty = File.open('/dev/stdin') do |f| + f.isatty + end + if tty + %w(-t --init) + else + [] + end + end + + def extra_env + if @options[:extra_env] + @options[:extra_env].map do |k, v| + # Here the value must not be escaped + ['-e', "#{k}=#{v}"] + end.flatten + else + [] + end + end + + def port_forwards + args = (0...num_exposed_ports).map do |i| + host_port = @options[:mongo_only] + i + container_port = 27017 + i + ['-p', "#{host_port}:#{container_port}"] + end.flatten + + if @env['OCSP_ALGORITHM'] && !@env['OCSP_VERIFIER'] + args += %w(-p 8100:8100) + end + + args + end + + def run_command(cmd) + if pid = fork + Process.wait(pid) + unless $?.exitstatus == 0 + raise "Process exited with code #{$?.exitstatus}" + end + else + exec(*cmd) + end + end + + def distro + @options[:distro] || if app_tests? + 'ubuntu2004' + else + case server_version + when '3.6' + 'debian9' + when '4.0', '4.2' + 'ubuntu1804' + else + 'ubuntu2004' + end + end + end + + BASE_IMAGES = { + 'debian81' => 'debian:jessie', + 'debian92' => 'debian:stretch', + 'debian10' => 'debian:buster', + 'debian11' => 'debian:bullseye', + 'ubuntu1404' => 'ubuntu:trusty', + 'ubuntu1604' => 'ubuntu:xenial', + 'ubuntu1804' => 'ubuntu:bionic', + 'ubuntu2004' => 'ubuntu:focal', + 'ubuntu2204' => 'ubuntu:jammy', + 'rhel62' => 'centos:6', + 'rhel70' => 'centos:7', + 'rhel80' => 'rockylinux:8', + }.freeze + + def base_image + BASE_IMAGES[distro] or raise "Unknown distro: #{distro}" + end + + def ruby + @env['RVM_RUBY'] + end + + def ruby_head? + ruby == 'ruby-head' + end + + def system_ruby? + %w(1 true yes).include?(@env['SYSTEM_RUBY']&.downcase) + end + + def server_version + @env['MONGODB_VERSION'] + end + + def script + @options[:script] || options.fetch(:default_script) + end + + def debian? + distro =~ /debian|ubuntu/ + end + + def ubuntu? + distro=~ /ubuntu/ + end + + def preload? + !!@options[:preload] + end + + def interactive? + !!@options[:interactive] + end + + def project_lib_subdir + options.fetch(:project_lib_subdir) + end + + def server_download_url + @server_download_url ||= ServerVersionRegistry.new(server_version, distro).download_url + end + + def libmongocrypt_path + case distro + when /ubuntu1604/ + "./ubuntu1604/nocrypto/lib64/libmongocrypt.so" + when /ubuntu1804/ + "./ubuntu1804-64/nocrypto/lib64/libmongocrypt.so" + when /debian92/ + "./debian92/nocrypto/lib64/libmongocrypt.so" + else + raise "This script does not support running FLE tests on #{distro}. Use ubuntu1604, ubuntu1804 or debian92 instead" + end + end + + def expose? + !!@options[:mongo_only] + end + + def fle? + %w(1 true yes).include?(@env['FLE']&.downcase) + end + + # Mongoid + def app_tests? + %w(1 true yes).include?(@env['APP_TESTS']&.downcase) + end + + def num_exposed_ports + case @env['TOPOLOGY'] || 'standalone' + when 'standalone', 'replica-set-single-node' + 1 + when 'replica-set' + 3 + when 'sharded-cluster' + if @env['SINGLE_MONGOS'] + 1 + else + 2 + end + end + end + end +end diff --git a/spec/shared/lib/mrss/eg_config_utils.rb b/spec/shared/lib/mrss/eg_config_utils.rb new file mode 100644 index 0000000000..2e6269a323 --- /dev/null +++ b/spec/shared/lib/mrss/eg_config_utils.rb @@ -0,0 +1,51 @@ +autoload :YAML, 'yaml' +require 'erubi' +require 'erubi/capture_end' +require 'tilt' + +module Mrss + module EgConfigUtils + + DEBIAN_FOR_RUBY = { + 'ruby-2.3' => 'debian92', + 'ruby-2.4' => 'debian92', + 'ruby-2.5' => 'debian10', + 'ruby-2.6' => 'debian10', + 'ruby-2.7' => 'debian10', + 'ruby-3.0' => 'debian10', + } + + def standard_debian_rubies(rubies, key: nil, &block) + rubies.flatten! + text = block.call + contents = YAML.load(text) + out = rubies.map do |ruby| + contents.merge( + 'matrix_name' => "#{contents['matrix_name']} - #{ruby}", + 'matrix_spec' => contents['matrix_spec'].merge( + 'ruby' => ruby, + key || 'os' => DEBIAN_FOR_RUBY.fetch(ruby), + ), + ) + end.to_yaml + text =~ /\A\n?(\s+)/ + unless text + raise "Couldn't figure out indentation level" + end + indent = ' ' * ($1.length - 2) + "\n" + out.sub(/\A---.*\n/, indent).gsub("\n", "\n#{indent}") + end + + def transform_config(template_path, context) + Tilt.new(template_path, engine_class: Erubi::CaptureEndEngine).render(context) + end + + def generated_file_warning + <<-EOT +# GENERATED FILE - DO NOT EDIT. +# Run ./.evergreen/update-evergreen-configs to regenerate this file. + +EOT + end + end +end diff --git a/spec/shared/lib/mrss/event_subscriber.rb b/spec/shared/lib/mrss/event_subscriber.rb new file mode 100644 index 0000000000..80371992c4 --- /dev/null +++ b/spec/shared/lib/mrss/event_subscriber.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +module Mrss + # Test event subscriber. + class EventSubscriber + + # The mappings of event names to types. + MAPPINGS = { + 'topology_opening_event' => Mongo::Monitoring::Event::TopologyOpening, + 'topology_description_changed_event' => Mongo::Monitoring::Event::TopologyChanged, + 'topology_closed_event' => Mongo::Monitoring::Event::TopologyClosed, + 'server_opening_event' => Mongo::Monitoring::Event::ServerOpening, + 'server_description_changed_event' => Mongo::Monitoring::Event::ServerDescriptionChanged, + 'server_closed_event' => Mongo::Monitoring::Event::ServerClosed + }.freeze + + attr_reader :all_events + + attr_reader :started_events + + attr_reader :succeeded_events + + attr_reader :failed_events + + attr_reader :published_events + + # @param [ String ] name Optional name for the event subscriber. + def initialize(name: nil) + @mutex = Mutex.new + clear_events! + @name = name + end + + def to_s + %Q`#` + end + + alias :inspect :to_s + + # Event retrieval + + def select_started_events(cls) + started_events.select do |event| + event.is_a?(cls) + end + end + + def select_succeeded_events(cls) + succeeded_events.select do |event| + event.is_a?(cls) + end + end + + def select_completed_events(*classes) + (succeeded_events + failed_events).select do |event| + classes.any? { |c| c === event } + end + end + + def select_published_events(cls) + published_events.select do |event| + event.is_a?(cls) + end + end + + # Filters command started events for the specified command name. + def command_started_events(command_name) + started_events.select do |event| + event.command[command_name] + end + end + + def non_auth_command_started_events + started_events.reject do |event| + %w(authenticate getnonce saslSstart saslContinue).any? do |cmd| + event.command[cmd] + end + end + end + + # Locates command stated events for the specified command name, + # asserts that there is exactly one such event, and returns it. + def single_command_started_event(command_name, include_auth: false, database_name: nil) + events = if include_auth + started_events + else + non_auth_command_started_events + end + get_one_event(events, command_name, 'started', database_name: database_name) + end + + # Locates command succeeded events for the specified command name, + # asserts that there is exactly one such event, and returns it. + def single_command_succeeded_event(command_name, database_name: nil) + get_one_event(succeeded_events, command_name, 'succeeded', database_name: database_name) + end + + def get_one_event(events, command_name, kind, database_name: nil) + events = events.select do |event| + event.command_name == command_name and + database_name.nil? || database_name == event.database_name + end + if events.length != 1 + raise "Expected a single '#{command_name}' #{kind} event#{database_name ? " for '#{database_name}'" : ''} but we have #{events.length}" + end + events.first + end + + # Get the first succeeded event published for the name, and then delete it. + # + # @param [ String ] name The event name. + # + # @return [ Event ] The matching event. + def first_event(name) + cls = MAPPINGS[name] + if cls.nil? + raise ArgumentError, "Bogus event name #{name}" + end + matching = succeeded_events.find do |event| + cls === event + end + succeeded_events.delete(matching) + matching + end + + # Event recording + + # Cache the started event. + # + # @param [ Event ] event The event. + def started(event) + @mutex.synchronize do + started_events << event + all_events << event + end + end + + # Cache the succeeded event. + # + # @param [ Event ] event The event. + def succeeded(event) + @mutex.synchronize do + succeeded_events << event + all_events << event + end + end + + # Cache the failed event. + # + # @param [ Event ] event The event. + def failed(event) + @mutex.synchronize do + failed_events << event + all_events << event + end + end + + def published(event) + @mutex.synchronize do + published_events << event + all_events << event + end + end + + # Clear all cached events. + def clear_events! + @all_events = [] + @started_events = [] + @succeeded_events = [] + @failed_events = [] + @published_events = [] + self + end + end + # Only handles succeeded events correctly. + class PhasedEventSubscriber < EventSubscriber + def initialize + super + @phase_events = {} + end + + def phase_finished(phase_index) + @phase_events[phase_index] = succeeded_events + @succeeded_events = [] + end + + def phase_events(phase_index) + @phase_events[phase_index] + end + + def event_count + @phase_events.inject(0) do |sum, event| + sum + event.length + end + end + end + + class VerboseEventSubscriber < EventSubscriber + %w(started succeeded failed published).each do |meth| + define_method(meth) do |event| + puts event.summary + super(event) + end + end + end +end diff --git a/spec/shared/lib/mrss/lite_constraints.rb b/spec/shared/lib/mrss/lite_constraints.rb new file mode 100644 index 0000000000..763b7e710b --- /dev/null +++ b/spec/shared/lib/mrss/lite_constraints.rb @@ -0,0 +1,238 @@ +# frozen_string_literal: true +# encoding: utf-8 + +module Mrss + module LiteConstraints + + # Constrain tests that use TimeoutInterrupt to MRI (and Unix). + def require_mri + before(:all) do + unless SpecConfig.instance.mri? + skip "MRI required, we have #{SpecConfig.instance.platform}" + end + end + end + + def require_jruby + before(:all) do + unless BSON::Environment.jruby? + skip "JRuby required, we have #{SpecConfig.instance.platform}" + end + end + end + + # This is for marking tests that fail on JRuby that should + # in principle work (as opposed to being fundamentally incompatible + # with JRuby). + # Often times these failures happen only in Evergreen. + def fails_on_jruby(version=nil) + before(:all) do + if BSON::Environment.jruby? + if version + min_parts = version.split('.').map(&:to_i) + actual_parts = JRUBY_VERSION.split('.').map(&:to_i)[0...min_parts.length] + actual = actual_parts.join('.') + if actual <= version + skip "Fails on jruby through #{version}" + end + else + skip "Fails on jruby" + end + end + end + end + + # Indicates that the respective test uses the internet in some capacity, + # for example the test resolves SRV DNS records. + def require_external_connectivity + before(:all) do + if ENV['EXTERNAL_DISABLED'] + skip "Test requires external connectivity" + end + end + end + + def require_mongo_kerberos + before(:all) do + # TODO Use a more generic environment variable name if/when + # Mongoid tests get Kerberos configurations. + unless %w(1 yes true).include?(ENV['MONGO_RUBY_DRIVER_KERBEROS']&.downcase) + skip 'Set MONGO_RUBY_DRIVER_KERBEROS=1 in environment to run Kerberos unit tests' + end + require 'mongo_kerberos' + end + end + + def require_linting + before(:all) do + unless Mongo::Lint.enabled? + skip "Linting is not enabled" + end + end + end + + # Some tests will fail if linting is enabled: + # 1. Tests that pass invalid options to client, etc. which the linter + # rejects. + # 2. Tests that set expectations on topologies, server descriptions, etc. + # (since setting expectations requires mutating said objects, and when + # linting is on those objects are frozen). + def require_no_linting + before(:all) do + if Mongo::Lint.enabled? + skip "Linting is enabled" + end + end + end + + def require_libmongocrypt + before(:all) do + # If FLE is set in environment, the entire test run is supposed to + # include FLE therefore run the FLE tests. + if (ENV['LIBMONGOCRYPT_PATH'] || '').empty? && (ENV['FLE'] || '').empty? + skip 'Test requires path to libmongocrypt to be specified in LIBMONGOCRYPT_PATH env variable' + end + end + end + + def min_libmongocrypt_version(version) + require_libmongocrypt + before(:all) do + actual_version = Utils.parse_version(Mongo::Crypt::Binding.mongocrypt_version(nil)) + min_version = Utils.parse_version(version) + unless actual_version >= min_version + skip "libmongocrypt version #{min_version} required, but version #{actual_version} is available" + end + end + end + + def require_no_libmongocrypt + before(:all) do + if ENV['LIBMONGOCRYPT_PATH'] + skip 'Test requires libmongocrypt to not be configured' + end + end + end + + def require_aws_auth + before(:all) do + unless (ENV['AUTH'] || '') =~ /^aws/ + skip 'This test requires AUTH=aws* and an appropriately configured runtime environment' + end + end + end + + def require_ec2_host + before(:all) do + if $have_aws.nil? + $have_aws = begin + require 'open-uri' + begin + Timeout.timeout(3.81) do + URI.parse('http://169.254.169.254/latest/meta-data/profile').open.read + end + true + # When trying to use the EC2 metadata endpoint on ECS: + # Errno::EINVAL: Failed to open TCP connection to 169.254.169.254:80 (Invalid argument - connect(2) for "169.254.169.254" port 80) + rescue Timeout::Error, Errno::ETIMEDOUT, Errno::EINVAL, OpenURI::HTTPError => $aws_error + false + end + end + end + unless $have_aws + skip "EC2 instance metadata is not available - assuming not running on an EC2 instance: #{$aws_error.class}: #{$aws_error}" + end + end + end + + def require_stress + before(:all) do + if !SpecConfig.instance.stress? + skip 'Set STRESS=1 in environment to run stress tests' + end + end + end + + def require_fork + before(:all) do + if !SpecConfig.instance.fork? + skip 'Set FORK=1 in environment to run fork tests' + end + end + end + + def require_ocsp + before(:all) do + if !SpecConfig.instance.ocsp? + skip 'Set OCSP=1 in environment to run OCSP tests' + end + end + end + + def require_ocsp_verifier + before(:all) do + if !SpecConfig.instance.ocsp_verifier? + skip 'Set OCSP_VERIFIER=1 in environment to run OCSP verifier tests' + end + end + end + + def require_ocsp_connectivity + before(:all) do + if !SpecConfig.instance.ocsp_connectivity? + skip 'Set OCSP_CONNECTIVITY=pass or OCSP_CONNECTIVITY=fail in environment to run OCSP connectivity tests' + end + end + end + + def require_active_support + before(:all) do + if !SpecConfig.instance.active_support? + skip 'This test requires ActiveSupport; set WITH_ACTIVE_SUPPORT=1 in environment' + end + end + end + + def no_active_support + before(:all) do + if SpecConfig.instance.active_support? + skip 'This test requires no ActiveSupport; unset WITH_ACTIVE_SUPPORT in environment' + end + end + end + + def require_fallbacks + before(:all) do + unless %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase) + skip 'Set TEST_I18N_FALLBACKS=1 environment variable to run these tests' + end + end + end + + def require_no_fallbacks + before(:all) do + if %w(yes true 1).include?((ENV['TEST_I18N_FALLBACKS'] || '').downcase) + skip 'Set TEST_I18N_FALLBACKS=0 environment variable to run these tests' + end + end + end + + # This is a macro for retrying flaky tests on CI that occasionally fail. + # Note that the tests will only be retried on CI. + # + # @param [ Integer ] :tries The number of times to retry. + # @param [ Integer ] :sleep The number of seconds to sleep in between retries. + # If nothing, or nil, is passed, we won't wait in between retries. + def retry_test(tries: 3, sleep: nil) + if %w(1 yes true).include?(ENV['CI']) + around do |example| + if sleep + example.run_with_retry retry: tries, retry_wait: sleep + else + example.run_with_retry retry: tries + end + end + end + end + end +end diff --git a/spec/shared/lib/mrss/release/candidate.rb b/spec/shared/lib/mrss/release/candidate.rb new file mode 100644 index 0000000000..46d910feec --- /dev/null +++ b/spec/shared/lib/mrss/release/candidate.rb @@ -0,0 +1,281 @@ +# frozen_string_literal: true + +require 'json' + +require_relative 'product_data' + +module Mrss + module Release + class Candidate + # Release note section titles, by pr type + SECTION_TITLE = { + bcbreak: "Breaking Changes", + feature: "New Features", + bug: "Bug Fixes", + }.freeze + + # GitHub labels + BCBREAK = 'bcbreak' + FEATURE = 'feature' + BUG = 'bug' + PATCH = 'patch' + + def self.instance + @instance ||= new + + yield @instance if block_given? + + @instance + end + + def product + @product ||= ProductData.new + end + + def bump_version + product.bump_version(release_type) + end + + def bump_version! + product.bump_version!(release_type) + end + + def branch_name + @branch_name ||= "rc-#{product.version}" + end + + # return a string of commit names since the last release + def pending_changes + @changes ||= begin + range = product.tag_exists? ? "#{product.tag_name}.." : "" + `git log --pretty=format:"%s" #{range}` + end + end + + # return a list of PR numbers since the last release + def pending_pr_numbers + @pending_pr_numbers ||= pending_changes. + lines. + map { |line| line.match(/\(#(\d+)\)$/).then { |m| m && m[1] } }. + compact. + sort.reverse + end + + # return a JSON string of PR data + def pending_pr_dump + @pending_pr_dump ||= `gh pr list --state all --limit 256 --json number,title,labels,url,body --jq 'map(select([.number] | inside([#{pending_pr_numbers.join(',')}]))) | sort_by(.number)'` + end + + # return a list of PR data since the last release + def pending_prs + @pending_prs ||= JSON.parse(pending_pr_dump) + end + + # return a list of pending prs with additional attributes (summary, + # short title, jira issue number). + def decorated_prs + @decorated_prs ||= pending_prs.map do |pr| + jira_issue, pr_title = split_pr_title(pr) + summary = extract_summary(pr) + type = pr_type(pr) + type_code = pr_type_code(type) + patch_flag = pr_patch_flag?(pr) + + pr.merge('jira' => jira_issue, + 'short-title' => pr_title, + 'summary' => summary, + 'type' => type, + 'type-code' => type_code, + 'patch' => patch_flag) + end + end + + # return a hash of decorated prs grouped by :bcbreak, :feature, or :bug + def prs_by_type + @prs_by_type ||= decorated_prs.group_by { |pr| pr['type'] } + end + + # returns 'major', 'minor', or 'patch', depending on the presence of + # (respectively) :bcbreak, :feature, or :bug labels. + # + # If the RELEASE environment variable is set, its value will be used + # directly, ignoring whatever PR labels might exist. + def release_type + @release_type ||= if ENV['RELEASE'] + ENV['RELEASE'] + elsif prs_by_type[:bcbreak] + 'major' + elsif prs_by_type[:feature] && prs_by_type[:feature].any? { |pr| !pr['patch'] } + 'minor' + else + 'patch' + end + end + + # returns the generated release notes as a string + def release_notes + @release_notes ||= release_notes_intro + + %i[ bcbreak feature bug ]. + flat_map { |type| release_notes_for_type(type) }.join("\n") + end + + private + + # returns an array of strings, each string representing a single line + # in the release notes for the PR's of the given type. + def release_notes_for_type(type) + return [] unless prs_by_type[type] + + [].tap do |lines| + lines << "\# #{SECTION_TITLE[type]}" + lines << '' + + prs = prs_by_type[type] + summarized, unsummarized = prs.partition { |pr| pr['summary'] } + + summarized.each do |pr| + header = [ '### ' ] + header << "[#{pr['jira']}](#{jira_url(pr['jira'])}) " if pr['jira'] + header << "#{pr['short-title']} ([PR](#{pr['url']}))" + lines << header.join + lines << '' + lines << pr['summary'] + lines << '' + end + + if summarized.any? && unsummarized.any? + lines << '' + lines << [ '### Other ', SECTION_TITLE[type] ].join + lines << '' + end + + unsummarized.each do |pr| + line = [ '* ' ] + line << "[#{pr['jira']}](#{jira_url(pr['jira'])}) " if pr['jira'] + line << "#{pr['short-title']} ([PR](#{pr['url']}))" + + lines << line.join + end + + lines << '' + end + end + + # returns the URL of for the given jira issue + def jira_url(issue) + "https://jira.mongodb.org/browse/#{issue}" + end + + # assumes a pr title in the format of "JIRA-1234 PR Title (#1234)", + # returns a tuple of [ jira-issue, title ], where jira-issue may be + # blank (if no jira issue is in the title). + def split_pr_title(pr) + title = pr['title'].gsub(/\(#\d+\)/, '').strip + + if title =~ /^(\w+-\d+) (.*)$/ + [ $1, $2 ] + else + [ nil, title ] + end + end + + # extracts the summary section from the pr and returns it (or returns nil + # if no summary section is detected) + def extract_summary(pr) + summary = [] + accumulating = false + level = nil + + pr['body'].lines.each do |line| + # a header of any level titled "summary" will begin the summary + if !accumulating && line =~ /^(\#+)\s+summary\s+$/i + accumulating = true + level = $1.length + + # a header of any level less than or equal to the summary header's + # level will end the summary + elsif accumulating && line =~ /^\#{1,#{level}}\s+/ + break + + # otherwise, the line is part of the summary + elsif accumulating + summary << line + end + end + + summary.any? ? summary.join.strip : nil + end + + # Returns a symbol (:bcbreak, :feature, or :bug) that identifies the + # type of this PR that would most strongly influence what type of release + # it requires. + def pr_type(pr) + if pr['labels'].any? { |l| l['name'] == BCBREAK } + :bcbreak + elsif pr['labels'].any? { |l| l['name'] == FEATURE } + :feature + elsif pr['labels'].any? { |l| l['name'] == BUG } + :bug + else + nil + end + end + + # `true` if the `patch` label is applied to the PR. This is used to + # indicate that a "feature" PR should be treated as a patch, for + # determining the release type only. + def pr_patch_flag?(pr) + pr['labels'].any? { |l| l['name'] == PATCH } + end + + def pr_type_code(type) + case type + when :bcbreak then 'x' + when :feature then 'f' + when :bug then 'b' + else '?' + end + end + + def series + major, minor, = product.version_parts + + case release_type + when 'minor' then + "#{major}.x" + when 'patch' then + "#{major}.#{minor}.x" + end + end + + # Return a string containing the markdown-formatted intro block for + # the release notes of this candidate. + def release_notes_intro + release_description = case release_type + when 'major' then 'major release' + when 'minor' then "minor release in the #{series} series" + when 'patch' then "patch release in the #{series} series" + end + + <<~INTRO + The MongoDB Ruby team is pleased to announce version #{product.version} of the `#{product.package}` gem - #{product.description}. This is a new #{release_description} of #{product.name}. + + Install this release using [RubyGems](https://rubygems.org/) via the command line as follows: + + ~~~ + gem install -v #{product.version} #{product.package} + ~~~ + + Or simply add it to your `Gemfile`: + + ~~~ + gem '#{product.package}', '#{product.version}' + ~~~ + + Have any feedback? Click on through to MongoDB's JIRA and [open a new ticket](#{product.jira_project_url}) to let us know what's on your mind 🧠. + + INTRO + end + end + end +end diff --git a/spec/shared/lib/mrss/release/product_data.rb b/spec/shared/lib/mrss/release/product_data.rb new file mode 100644 index 0000000000..c6c94fb8cd --- /dev/null +++ b/spec/shared/lib/mrss/release/product_data.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require 'yaml' + +module Mrss + module Release + class ProductData + FILE_PATH = 'product.yml' + + def self.init! + if File.exist?(FILE_PATH) + raise "#{FILE_PATH} already exists; refusing to overwrite it" + end + + initial_data = { + 'name' => 'Product Name', + 'description' => 'a very short description of the product', + 'package' => 'product_package', + 'jira' => 'https://url.to.jira/project', + 'version' => { 'number' => '1.0.0', + 'file' => 'path/to/version.rb' } + } + + File.write(FILE_PATH, initial_data.to_yaml) + end + + def initialize + @hash = YAML.load_file(FILE_PATH) + end + + def save_product_file! + File.write(FILE_PATH, @hash.to_yaml) + end + + def rewrite_version_file! + version_module = File.read(version_file) + new_module = version_module. + sub(/^(\s*)(VERSION\s*=\s*).*$/) { "#{$1}#{$2}#{quoted_version}" } + File.write(version_file, new_module) + end + + def version + @hash['version']['number'] + end + + def quoted_version + if version.include?("'") + version.inspect + else + "'#{version}'" + end + end + + def version=(number) + @hash['version']['number'] = number + end + + # returns an array of [ major, minor, patch, suffix ]. + # + # each element will be returned as a String. + def version_parts + version.split(/\./, 4) + end + + # bump the version according to the given release type: + # + # 'major' -> increment major component, zero the others + # 'minor' -> increment minor component, zero the patch + # 'patch' -> increment the patch component + def bump_version(release) + major, minor, patch, suffix = version_parts + + case release + when 'major' then + major = major.to_i + 1 + minor = patch = 0 + when 'minor' + minor = minor.to_i + 1 + patch = 0 + when 'patch' + patch = patch.to_i + 1 + else + raise ArgumentError, "invalid release type: #{release.inspect}" + end + + self.version = [ major, minor, patch ].join('.') + end + + # Invokes `#bump_version`, and then saves the new version to the + # product.yml file and to the version.rb file. + def bump_version!(release) + bump_version(release) + save_product_file! + rewrite_version_file! + end + + def version_file + @hash['version']['file'] + end + + def name + @hash['name'] + end + + # The description is intended to be used in places where it can be + # appended to the end of a sentence, e.g. + # + # "We just released #{product.name} - #{product.description}!" + # + # Markdown formatting is allowed (even expected). + def description + @hash['description'] + end + + def package + @hash['package'] + end + + def jira_project_url + @hash['jira'] + end + + def tag_name + "v#{version}" + end + + def tag_exists?(tag = tag_name) + `git tag -l #{tag}`.strip == tag + end + + def branch_exists?(branch) + `git branch -l #{branch}`.strip == branch + end + + def base_branch + @base_branch ||= begin + major, minor, = version_parts + branch = "#{major}.#{minor}-stable" + branch_exists?(branch) ? branch : 'master' + end + end + end + end +end diff --git a/spec/shared/lib/mrss/server_version_registry.rb b/spec/shared/lib/mrss/server_version_registry.rb new file mode 100644 index 0000000000..2646df24c1 --- /dev/null +++ b/spec/shared/lib/mrss/server_version_registry.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true +# encoding: utf-8 + +autoload :JSON, 'json' +require 'open-uri' + +module Mrss + class ServerVersionRegistry + class Error < StandardError + end + + class UnknownVersion < Error + end + + class MissingDownloadUrl < Error + end + + class BrokenDownloadUrl < Error + end + + def initialize(desired_version, arch) + @desired_version, @arch = desired_version, arch.sub(/-arm$/, '') + end + + attr_reader :desired_version, :arch + + def target_arch + # can't use RbConfig::CONFIG["arch"] because JRuby doesn't + # return anything meaningful there. + # + # also, need to use `uname -a` instead of (e.g.) `uname -p` + # because debian (at least) does not return anything meaningful + # for `uname -p`. + uname = `uname -a`.strip + @target_arch ||= case uname + when /aarch/ then "aarch64" + when /x86/ then "x86_64" + else raise "unsupported architecture #{uname.inspect}" + end + end + + def download_url + @download_url ||= begin + version, version_ok = detect_version(current_catalog) + if version.nil? + version, full_version_ok = detect_version(full_catalog) + version_ok ||= full_version_ok + end + if version.nil? + if version_ok + raise MissingDownloadUrl, "No downloads for version #{desired_version}" + else + raise UnknownVersion, "No version #{desired_version}" + end + end + dl = version['downloads'].detect do |dl| + dl['archive']['url'].index("enterprise-#{arch}") && + dl['arch'] == target_arch + end + unless dl + raise MissingDownloadUrl, "No download for #{arch} for #{version['version']}" + end + url = dl['archive']['url'] + end + end + + private + + def uri_open(*args) + if RUBY_VERSION < '2.5' + open(*args) + else + URI.open(*args) + end + end + + def detect_version(catalog) + candidate_versions = catalog['versions'].select do |version| + version['version'].start_with?(desired_version) && + !version['version'].include?('-') + end + version_ok = !candidate_versions.empty? + # Sometimes the download situation is borked and there is a release + # with no downloads... skip those. + version = candidate_versions.detect do |version| + !version['downloads'].empty? + end + # Allow RC releases if there isn't a GA release. + if version.nil? + candidate_versions = catalog['versions'].select do |version| + version['version'].start_with?(desired_version) + end + version_ok ||= !candidate_versions.empty? + version = candidate_versions.detect do |version| + !version['downloads'].empty? + end + end + [version, version_ok] + end + + def current_catalog + @current_catalog ||= begin + JSON.load(uri_open('http://downloads.mongodb.org/current.json').read) + end + end + + def full_catalog + @full_catalog ||= begin + JSON.load(uri_open('http://downloads.mongodb.org/full.json').read) + end + end + end +end diff --git a/spec/shared/lib/mrss/session_registry.rb b/spec/shared/lib/mrss/session_registry.rb new file mode 100644 index 0000000000..46b3b6df9f --- /dev/null +++ b/spec/shared/lib/mrss/session_registry.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true +# encoding: utf-8 + +require 'singleton' + +module Mrss + + def self.patch_mongo_for_session_registry + + Mongo::Client.class_eval do + alias :get_session_without_tracking :get_session + + def get_session(options = {}) + get_session_without_tracking(options).tap do |session| + SessionRegistry.instance.register(session) if session&.materialized? + end + end + end + + Mongo::Session.class_eval do + alias :end_session_without_tracking :end_session + + def end_session + SessionRegistry.instance.unregister(self) + end_session_without_tracking + end + + alias :materialize_if_needed_without_tracking :materialize_if_needed + + def materialize_if_needed + materialize_if_needed_without_tracking.tap do + SessionRegistry.instance.register(self) + end + end + end + end +end + +module Mrss + class SessionRegistry + include Singleton + + def initialize + @registry = {} + end + + def register(session) + @registry[session.session_id] = session if session + end + + def unregister(session) + return if session.ended? || !session.materialized? + @registry.delete(session.session_id) + end + + def verify_sessions_ended! + @registry.delete_if { |_, session| session.ended? } + + unless @registry.empty? + sessions = @registry.map { |_, session| session } + raise "Session registry contains live sessions: #{sessions.join(', ')}" + end + end + + def clear_registry + @registry = {} + end + end +end diff --git a/spec/shared/lib/mrss/session_registry_legacy.rb b/spec/shared/lib/mrss/session_registry_legacy.rb new file mode 100644 index 0000000000..7abc980634 --- /dev/null +++ b/spec/shared/lib/mrss/session_registry_legacy.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +# encoding: utf-8 + +require 'singleton' + +module Mrss + + def self.patch_mongo_for_session_registry + + Mongo::Client.class_eval do + alias :get_session_without_tracking :get_session + + def get_session(options = {}) + get_session_without_tracking(options).tap do |session| + SessionRegistry.instance.register(session) + end + end + end + + Mongo::Session.class_eval do + alias :end_session_without_tracking :end_session + + def end_session + SessionRegistry.instance.unregister(self) + end_session_without_tracking + end + end + end +end + +module Mrss + class SessionRegistry + include Singleton + + def initialize + @registry = {} + end + + def register(session) + @registry[session.session_id] = session if session + end + + def unregister(session) + @registry.delete(session.session_id) unless session.ended? + end + + def verify_sessions_ended! + @registry.delete_if { |_, session| session.ended? } + + unless @registry.empty? + sessions = @registry.map { |_, session| session } + raise "Session registry contains live sessions: #{sessions.join(', ')}" + end + end + + def clear_registry + @registry = {} + end + end +end diff --git a/spec/shared/lib/mrss/spec_organizer.rb b/spec/shared/lib/mrss/spec_organizer.rb new file mode 100644 index 0000000000..2198c35536 --- /dev/null +++ b/spec/shared/lib/mrss/spec_organizer.rb @@ -0,0 +1,208 @@ +# frozen_string_literal: true +# encoding: utf-8 + +autoload :JSON, 'json' +autoload :FileUtils, 'fileutils' +autoload :Find, 'find' + +module Mrss + + autoload :ChildProcessHelper, 'mrss/child_process_helper' + + # Organizes and runs all of the tests in the test suite in batches. + # + # Organizing the tests in batches serves two purposes: + # + # 1. This allows running unit tests before integration tests, therefore + # in theory revealing failures quicker on average. + # 2. This allows running some tests that have high intermittent failure rate + # in their own test process. + # + # This class aggregates RSpec results after the test runs. + class SpecOrganizer + + class BucketsNotPrioritized < StandardError + end + + def initialize(root: nil, classifiers:, priority_order:, + spec_root: nil, rspec_json_path: nil, rspec_all_json_path: nil, rspec_xml_path: nil, randomize: false + ) + @spec_root = spec_root || File.join(root, 'spec') + @classifiers = classifiers + @priority_order = priority_order + @rspec_json_path = rspec_json_path || File.join(root, 'tmp/rspec.json') + @rspec_all_json_path = rspec_all_json_path || File.join(root, 'tmp/rspec-all.json') + @rspec_xml_path = rspec_xml_path || File.join(root, 'tmp/rspec.xml') + @randomize = !!randomize + end + + attr_reader :spec_root, :classifiers, :priority_order + attr_reader :rspec_json_path, :rspec_all_json_path, :rspec_xml_path + + def randomize? + @randomize + end + + def seed + @seed ||= (rand * 100_000).to_i + end + + # Remove all XML files from tmp directory before running tests + def cleanup_xml_files + xml_pattern = File.join(File.dirname(rspec_xml_path), '*.xml') + Dir.glob(xml_pattern).each do |xml_file| + FileUtils.rm_f(xml_file) + end + end + + # Move the XML file to a timestamped version for evergreen upload + def archive_xml_file(category) + return unless File.exist?(rspec_xml_path) + + timestamp = Time.now.strftime('%Y%m%d_%H%M%S_%3N') + archived_path = rspec_xml_path.sub(/\.xml$/, "-#{category}-#{timestamp}.xml") + + FileUtils.mv(rspec_xml_path, archived_path) + puts "Archived XML results to #{archived_path}" + end + + def buckets + @buckets ||= {}.tap do |buckets| + Find.find(spec_root) do |path| + next unless File.file?(path) + next unless path =~ /_spec\.rb\z/ + rel_path = path[(spec_root.length + 1)..path.length] + + found = false + classifiers.each do |(regexp, category)| + if regexp =~ rel_path + buckets[category] ||= [] + buckets[category] << File.join('spec', rel_path) + found = true + break + end + end + + unless found + buckets[nil] ||= [] + buckets[nil] << File.join('spec', rel_path) + end + end + end.freeze + end + + def ordered_buckets + @ordered_buckets ||= {}.tap do |ordered_buckets| + buckets = self.buckets.dup + priority_order.each do |category| + files = buckets.delete(category) + ordered_buckets[category] = files + end + + if files = buckets.delete(nil) + ordered_buckets[nil] = files + end + + unless buckets.empty? + raise BucketsNotPrioritized, "Some buckets were not prioritized: #{buckets.keys.map(&:to_s).join(', ')}" + end + end.freeze + end + + def run + run_buckets(*buckets.keys) + end + + def run_buckets(*buckets) + FileUtils.rm_f(rspec_all_json_path) + # Clean up all XML files before starting test runs + cleanup_xml_files + + buckets.each do |bucket| + if bucket && !self.buckets[bucket] + raise "Unknown bucket #{bucket}" + end + end + buckets = Hash[self.buckets.select { |k, v| buckets.include?(k) }] + + failed = [] + + priority_order.each do |category| + if files = buckets.delete(category) + unless run_files(category, files) + failed << category + end + end + end + if files = buckets.delete(nil) + unless run_files('remaining', files) + failed << 'remaining' + end + end + + unless buckets.empty? + raise "Some buckets were not executed: #{buckets.keys.map(&:to_s).join(', ')}" + end + + if failed.any? + raise "The following buckets failed: #{failed.map(&:to_s).join(', ')}" + end + end + + def run_files(category, paths) + puts "Running #{category.to_s.gsub('_', ' ')} tests" + FileUtils.rm_f(rspec_json_path) + FileUtils.rm_f(rspec_xml_path) # Clean up XML file before running this bucket + + cmd = %w(rspec) + paths + # Add junit formatter for XML output + cmd += ['--format', 'RspecJunitFormatter', '--out', rspec_xml_path] + + if randomize? + cmd += %W(--order rand:#{seed}) + end + + begin + puts "Running #{cmd.join(' ')}" + ChildProcessHelper.check_call(cmd) + ensure + if File.exist?(rspec_json_path) + if File.exist?(rspec_all_json_path) + merge_rspec_results + else + FileUtils.cp(rspec_json_path, rspec_all_json_path) + end + end + + # Archive XML file after running this bucket + archive_xml_file(category) + end + + true + rescue ChildProcessHelper::SpawnError + false + end + + def merge_rspec_results + all = JSON.parse(File.read(rspec_all_json_path)) + new = JSON.parse(File.read(rspec_json_path)) + all['examples'] += new.delete('examples') + new.delete('summary').each do |k, v| + all['summary'][k] += v + end + new.delete('version') + new.delete('summary_line') + # The spec organizer runs all buckets with the same seed, hence + # we can drop the seed from new results. + new.delete('seed') + unless new.empty? + raise "Unhandled rspec results keys: #{new.keys.join(', ')}" + end + # We do not merge summary lines, delete them from aggregated results + all.delete('summary_line') + File.open(rspec_all_json_path, 'w') do |f| + f << JSON.dump(all) + end + end + end +end diff --git a/spec/shared/lib/mrss/utils.rb b/spec/shared/lib/mrss/utils.rb new file mode 100644 index 0000000000..54c6e49d35 --- /dev/null +++ b/spec/shared/lib/mrss/utils.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true +# encoding: utf-8 + +module Mrss + module Utils + extend self + + def print_backtrace(dest=STDERR) + raise + rescue => e + dest.puts e.backtrace.join("\n") + end + + # Parses the given version string, accounting for suffix information that + # Gem::Version cannot successfully parse. + # + # @param [ String ] version the version to parse + # + # @return [ Gem::Version ] the parsed version + # + # @raise [ ArgumentError ] if the string cannot be parsed. + def parse_version(version) + Gem::Version.new(version) + rescue ArgumentError + match = version.match(/\A(?\d+)\.(?\d+)\.(?\d+)?(-[A-Za-z\+\d]+)?\z/) + raise ArgumentError.new("Malformed version number string #{version}") if match.nil? + + Gem::Version.new( + [ + match[:major], + match[:minor], + match[:patch] + ].join('.') + ) + end + end +end diff --git a/spec/shared/lib/tasks/candidate.rake b/spec/shared/lib/tasks/candidate.rake new file mode 100644 index 0000000000..d4347695c4 --- /dev/null +++ b/spec/shared/lib/tasks/candidate.rake @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require_relative '../mrss/release/candidate' + +namespace :candidate do + desc 'Initialize a new product.yml file' + task :init do + Mrss::Release::ProductData.init! + puts "product.yml file created" + end + + desc 'Print the release notes for the next candidate release' + task :preview do + Mrss::Release::Candidate.instance do |candidate| + # load the pending changes before bumping the version, since it + # depends on the value of the current version. + candidate.pending_changes + candidate.bump_version + puts candidate.release_notes + end + end + + desc 'List the pull requests to be included in the next release' + task :prs do + Mrss::Release::Candidate.instance.decorated_prs.each do |pr| + print "\##{pr['number']}[#{pr['type-code']}] " + print "#{pr['jira']} " if pr['jira'] + puts pr['short-title'] + end + end + + desc 'Create a new branch and pull request for the candidate' + task create: :check_branch_status do + Mrss::Release::Candidate.instance do |candidate| + origin = `git config get remote.origin.url` + match = origin.match(/:(.*?)\//) or raise "origin url is not in expected format: #{origin.inspect}" + user = match[1] + + puts 'gathering candidate info and bumping version...' + candidate.bump_version! + + puts 'writing release notes to /tmp/pr-body.md...' + File.write('/tmp/pr-body.md', candidate.release_notes) + + sh 'git', 'checkout', '-b', candidate.branch_name + sh 'git', 'commit', '-am', "Bump version to #{candidate.product.version}" + sh 'git', 'push', 'origin', candidate.branch_name + + sh 'gh', 'pr', 'create', + '--head', "#{user}:#{candidate.branch_name}", + '--base', candidate.product.base_branch, + '--title', "Release candidate for #{candidate.product.version}", + '--label', 'release-candidate', + '--body-file', '/tmp/pr-body.md' + end + end + + # Ensures the current branch is up-to-date with no uncommitted changes + task :check_branch_status do + sh 'git pull >/dev/null', verbose: false + changes = `git status --short --untracked-files=no`.strip + abort "There are uncommitted changes. Commit (or revert) the changes and try again." if changes.length > 0 + end +end diff --git a/spec/shared/share/Dockerfile.erb b/spec/shared/share/Dockerfile.erb new file mode 100644 index 0000000000..ad3874d739 --- /dev/null +++ b/spec/shared/share/Dockerfile.erb @@ -0,0 +1,251 @@ +<% + +def ruby_toolchain_url(ruby) + "http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-toolchain/library/#{distro}/#{ruby}.tar.xz" +end + +%> + +FROM --platform=linux/arm64 <%= base_image %> + +ENV DOCKER=1 + +<% if debian? %> + + ENV DEBIAN_FRONTEND=noninteractive + +<% else %> + + RUN echo assumeyes=1 |tee -a /etc/yum.conf + +<% end %> + +<% if debian? %> + + # zsh is not required for any scripts but it is a better interactive shell + # than bash. + # Ruby runtime dependencies: libyaml-0-2 + # Compiling ruby libraries: gcc make + # Compiling python packages: python3-dev + # JRuby: openjdk-17-jdk-headless + # Server dependencies: libsnmp30 libcurl3/libcurl4 + # Determining OS we are running on: lsb-release + # Load balancer testing: haproxy + # Kerberos testing: krb5-user + # Local Kerberos server: krb5-kdc krb5-admin-server + # Installing mlaunch from git: git + # ruby-head archive: bzip2 + # nio4r on JRuby: libgmp-dev + # Snappy compression: libsnappy-dev + # nokogiri: zlib1g-dev + # Mongoid testing: tzdata shared-mime-info + # Mongoid application testing: nodejs (8.x or newer) + # Test suite: procps for ps (to kill JRubies) + # + # We currently use Python 2-compatible version of mtools, which + # is installable via pip (which uses Python 2). All of the MongoDB + # distros have pip installed (but none as of this writing have pip3) + # therefore install python-pip in all configurations here. + + <% packages = %w( + procps lsb-release bzip2 curl wget gpg zsh + git make gcc g++ libyaml-dev libgmp-dev zlib1g-dev libsnappy-dev + krb5-user krb5-kdc krb5-admin-server libsasl2-dev libsasl2-modules-gssapi-mit + haproxy libcurl4 + tzdata shared-mime-info software-properties-common xz-utils nodejs npm + openjdk-17-jdk-headless + ) %> + + <% if distro =~ /ubuntu2004/ %> + <% packages << 'libsnmp35' %> + <% elsif distro =~ /ubuntu2204|debian11/ %> + <% packages << 'libsnmp40' %> + <% else %> + <% packages << 'libsnmp30' %> + <% end %> + + <% if distro !~ /ubuntu2004|ubuntu2204|debian11/ %> + <% packages << 'python-pip' %> + <% end %> + + <% if distro =~ /ubuntu2204|debian11/ %> + <% packages << 'python3-venv' %> + <% end %> + + <% if distro =~ /ubuntu2004|ubuntu2204/ %> + <% packages += %w(ruby bundler) %> + <% end %> + + RUN apt-get update && apt-get install -y <%= packages.join(' ') %> + + <% if ubuntu? %> + RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null + <% end %> + RUN apt-add-repository ppa:deadsnakes/ppa -y + RUN apt-get update && apt-get install -y cmake python3.10 python3.10-dev python3.10-venv + RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 +<% else %> + + # Enterprise server: net-snmp + # lsb_release: redhat-lsb-core + # our runner scripts: which + # Ruby dependency: libyaml + # compiling python packages: gcc python-devel + # Kerberos tests: krb5-workstation + cyrus-sasl-devel to build the + # mongo_kerberos gem + cyrus-sasl-gssapi for authentication to work + # Local Kerberos server: krb5-server + # JRuby: java-17-openjdk + # + # Note: lacking cyrus-sasl-gssapi produces a cryptic message + # "SASL(-4): no mechanism available: No worthy mechs found" + # https://github.com/farorm/python-ad/issues/10 + + RUN yum --enablerepo=powertools install -y redhat-lsb-core which git gcc gcc-c++ libyaml-devel krb5-server \ + krb5-workstation cyrus-sasl-devel cyrus-sasl-gssapi java-17-openjdk \ + net-snmp python38 python38-devel cmake nodejs npm xz + +<% end %> + +<% if preload? %> + + <% if distro =~ /debian9|ubuntu1604|ubuntu1804/ %> + # Install python 3.7 for mlaunch. + RUN curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/<%= distro %>-python37.tar.xz | \ + tar xfJ - -C /opt + ENV PATH=/opt/python37/bin:$PATH + RUN python3 -V + <% end %> + + <% if true || distro =~ /rhel|ubuntu1604/ %> + + # Ubuntu 12.04 ships pip 1.0 which is ancient and does not work. + # + # Ubuntu 16.04 apparently also ships a pip that does not work: + # https://stackoverflow.com/questions/37495375/python-pip-install-throws-typeerror-unsupported-operand-types-for-retry + # Potentially this only affects environments with less than ideal + # connectivity (or, perhaps, when python package registry is experiencing + # availability issues) when pip must retry to install packages. + # + # rhel apparently does not package pip at all in core repoitories, + # therefore install it the manual way. + # + # https://pip.pypa.io/en/stable/installing/ + RUN curl --retry 3 -fL https://bootstrap.pypa.io/pip/get-pip.py | python3 + RUN python3 -m pip install --upgrade pip setuptools wheel + + <% end %> + + # Current virtualenv fails with + # https://github.com/pypa/virtualenv/issues/1630 + <% mtools = 'legacy' %> + <% case mtools + when 'legacy' %> + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + RUN python3 -m pip install 'virtualenv<20' 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil + <% when 'git' %> + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil + + # Install mtools from git because released versions do not work with pymongo 4.0 + RUN git clone https://github.com/p-mongodb/mtools && \ + cd mtools && \ + python3 setup.py install + <% else %> + # mtools[mlaunch] does not work: https://github.com/rueckstiess/mtools/issues/856 + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + RUN python3 -m pip install virtualenv 'pymongo>=4' python-dateutil psutil mtools + <% end %> + + <% if @env.fetch('MONGODB_VERSION') >= '4.4' %> + # ubuntu1604 installs MarkupSafe 0.0.0 here instead of 2.0.0+ + # as specified by dependencies, causing OCSP mock to not work. + RUN python3 -mpip install asn1crypto oscrypto flask --upgrade --ignore-installed + <% end %> + + # FLE is tested against 4.0+ servers. + <% if @env.fetch('MONGODB_VERSION') >= '4.0' %> + # Requirements in drivers-evergreen-tools: + # boto3~=1.19 cryptography~=3.4.8 pykmip~=0.10.0 + # cryptography does not install due to lacking setuptools_rust + # (either that version or anything that isn't part of system packages) + RUN python3 -mpip install boto3~=1.19 cryptography pykmip~=0.10.0 'sqlalchemy<2.0.0' + <% end %> + + <% unless ruby_head? || system_ruby? %> + + RUN curl --retry 3 -fL <%= ruby_toolchain_url(ruby) %> |tar -xC /opt -Jf - + ENV PATH=/opt/rubies/<%= ruby %>/bin:$PATH \ + USE_OPT_TOOLCHAIN=1 + + <% end %> + +<% end %> + +<% if distro =~ /debian|ubuntu/ %> + # mkdir was moved from /usr/bin to /bin and MongoDB's distros + # apparently keep using the old location. + # This definitely affects debian10. + # https://stackoverflow.com/questions/64653051/make-usr-bin-mkdir-command-not-found-during-gem-install-nokogiri-in-ubuntu + RUN test -f /usr/bin/mkdir || ln -s /bin/mkdir /usr/bin/mkdir +<% end %> + +WORKDIR /app + +<% if preload? && !ruby_head? %> + + COPY Gemfile . + COPY gemfiles gemfiles + COPY *.gemspec . + COPY lib/<%= project_lib_subdir %>/version.rb lib/<%= project_lib_subdir %>/version.rb + RUN bundle install + COPY .evergreen/patch-debuggers .evergreen/patch-debuggers + <% if system_ruby? %> + # Running under docker with root access + RUN .evergreen/patch-debuggers /var/lib/gems + <% else %> + RUN .evergreen/patch-debuggers /opt/rubies + <% end %> + +<% end %> + +<% if fle? %> + RUN curl --retry 3 -fL "https://s3.amazonaws.com/mciuploads/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz" |tar zxf - + + <%= "ENV LIBMONGOCRYPT_PATH #{libmongocrypt_path}" %> +<% end %> + +<% if preload? %> + ENV DOCKER_PRELOAD=1 +<% end %> + +RUN npm install --global yarn + +ENV MONGO_ORCHESTRATION_HOME=/tmpfs \ + PROJECT_DIRECTORY=/app \ + <%= @env.map { |k, v| %Q`#{k}="#{v.gsub('$', "\\$").gsub('"', "\\\"")}"` }.join(" \\\n ") %> + +<% if interactive? %> + ENV INTERACTIVE=1 +<% end %> + +COPY . . + +RUN bash -c '. .evergreen/download-mongodb.sh && get_distro && get_mongodb_download_url_for "$DISTRO" "<%= server_version %>" && curl --retry 3 -fL $MONGODB_DOWNLOAD_URL |tar xzf - && mv mongo*/ /opt/mongodb' +ENV USE_OPT_MONGODB=1 USE_SYSTEM_PYTHON_PACKAGES=1 + +<% if expose? %> + + <% ports = [] %> + + <% 0.upto(num_exposed_ports-1) do |i| %> + <% ports << 27017 + i %> + <% end %> + + <% if @env['OCSP_ALGORITHM'] %> + <% ports << 8100 %> + <% end %> + + EXPOSE <%= ports.map(&:to_s).join(' ') %> + +<% end %> diff --git a/spec/shared/share/haproxy-1.conf b/spec/shared/share/haproxy-1.conf new file mode 100644 index 0000000000..b3f476f67c --- /dev/null +++ b/spec/shared/share/haproxy-1.conf @@ -0,0 +1,16 @@ +# Modeled after +# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh + +defaults + mode tcp + timeout connect 7s + timeout client 55s + timeout server 55s + +frontend mongos_frontend + bind *:27017 + use_backend mongos_backend + +backend mongos_backend + mode tcp + server mongos_one 127.0.0.1:27117 check diff --git a/spec/shared/share/haproxy-2.conf b/spec/shared/share/haproxy-2.conf new file mode 100644 index 0000000000..359b8f8ea6 --- /dev/null +++ b/spec/shared/share/haproxy-2.conf @@ -0,0 +1,17 @@ +# Modeled after +# https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-load-balancer.sh + +defaults + mode tcp + timeout connect 7s + timeout client 55s + timeout server 55s + +frontend mongos_frontend + bind *:27017 + use_backend mongos_backend + +backend mongos_backend + mode tcp + server mongos_one 127.0.0.1:27117 check + server mongos_two 127.0.0.1:27118 check diff --git a/spec/shared/shlib/config.sh b/spec/shared/shlib/config.sh new file mode 100644 index 0000000000..99b3cdd764 --- /dev/null +++ b/spec/shared/shlib/config.sh @@ -0,0 +1,27 @@ +show_local_instructions_impl() { + local arch="$1" + shift + + echo To test this configuration locally: + local params= + while test -n "$1"; do + key="$1" + shift + # ${!foo} syntax is bash specific: + # https://stackoverflow.com/questions/14049057/bash-expand-variable-in-a-variable + value="${!key}" + if test -n "$value"; then + params="$params $key=$value" + fi + done + + # $0 has the current script being executed which is also the script that + # was initially invoked EXCEPT for the AWS configurations which use the + # wrapper script. + if echo "$AUTH" |grep -q ^aws; then + script=.evergreen/run-tests-aws-auth.sh + else + script="$0" + fi + echo ./.evergreen/test-on-docker -d $arch $params -s "$script" +} diff --git a/spec/shared/shlib/distro.sh b/spec/shared/shlib/distro.sh new file mode 100644 index 0000000000..0d295f03a0 --- /dev/null +++ b/spec/shared/shlib/distro.sh @@ -0,0 +1,84 @@ +detected_distro= + +host_distro() { + if test -z "$detected_distro"; then + detected_distro=`_detect_distro` + fi + echo "$detected_distro" +} + +_detect_distro() { + local distro + distro= + if test -f /etc/debian_version; then + # Debian or Ubuntu + if test "`uname -m`" = aarch64; then + release=`lsb_release -rs |tr -d .` + distro="ubuntu$release"-arm + elif lsb_release -is |grep -q Debian; then + release=`lsb_release -rs |tr -d .` + # In docker, release is something like 9.11. + # In evergreen, release is 9.2. + release=`echo $release |sed -e 's/^9.*/92/'` + distro="debian$release" + elif lsb_release -is |grep -q Ubuntu; then + if test "`uname -m`" = ppc64le; then + release=`lsb_release -rs |tr -d .` + distro="ubuntu$release-ppc" + else + release=`lsb_release -rs |tr -d .` + distro="ubuntu$release" + fi + else + echo 'Unknown Debian flavor' 1>&2 + exit 1 + fi + elif lsb_release -is |grep -qi suse; then + if test "`uname -m`" = s390x; then + release=`lsb_release -rs |sed -e 's/\..*//'` + distro="suse$release-s390x" + else + echo 'Unknown Suse arch' 1>&2 + exit 1 + fi + elif test -f /etc/redhat-release; then + # RHEL or CentOS + if test "`uname -m`" = s390x; then + distro=rhel72-s390x + elif test "`uname -m`" = ppc64le; then + distro=rhel71-ppc + elif lsb_release >/dev/null 2>&1; then + if lsb_release -is |grep -q RedHat; then + release=`lsb_release -rs |tr -d .` + distro="rhel$release" + elif lsb_release -is |grep -q CentOS; then + release=`lsb_release -rs |cut -c 1 |sed -e s/7/70/ -e s/6/62/ -e s/8/80/` + distro="rhel$release" + else + echo 'Unknown RHEL flavor' 1>&2 + exit 1 + fi + else + echo lsb_release missing, using /etc/redhat-release 1>&2 + release=`grep -o 'release [0-9]' /etc/redhat-release |awk '{print $2}'` + release=`echo $release |sed -e s/7/70/ -e s/6/62/ -e s/8/80/` + distro=rhel$release + fi + elif test -f /etc/os-release; then + name=`grep -o '^NAME=.*' /etc/os-release | awk -F '"' '{ print $2 }'` + version=`grep -o '^VERSION=.*' /etc/os-release | awk -F '"' '{ print $2 }'` + if test "$name" = "Amazon Linux"; then + distro=amazon$version + else + cat /etc/os-release + echo 'Unknown distro' 1>&2 + exit 1 + fi + else + lsb_release -a + echo 'Unknown distro' 1>&2 + exit 1 + fi + echo "Detected distro: $distro" 1>&2 + echo $distro +} diff --git a/spec/shared/shlib/server.sh b/spec/shared/shlib/server.sh new file mode 100644 index 0000000000..166c8c677d --- /dev/null +++ b/spec/shared/shlib/server.sh @@ -0,0 +1,423 @@ +# This file contains functions pertaining to downloading, starting and +# configuring a MongoDB server. + +# Note that mlaunch is executed with (and therefore installed with) Python 2. +# The reason for this is that in the past, some of the distros we tested on +# had an ancient version of Python 3 that was unusable (e.g. it couldn't +# install anything from PyPI due to outdated TLS/SSL implementation). +# It is likely that all of the current distros we use have a recent enough +# and working Python 3 implementation, such that we could use Python 3 for +# everything. +# +# Note that some distros (e.g. ubuntu2004) do not contain a `python' binary +# at all, thus python2 or python3 must be explicitly specified depending on +# the desired version. + +set_fcv() { + if test -n "$FCV"; then + mongo --eval 'assert.commandWorked(db.adminCommand( { setFeatureCompatibilityVersion: "'"$FCV"'" } ));' "$MONGODB_URI" + mongo --quiet --eval 'db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } )' |grep "version.*$FCV" + fi +} + +add_uri_option() { + opt=$1 + + if ! echo $MONGODB_URI |sed -e s,//,, |grep -q /; then + MONGODB_URI="$MONGODB_URI/" + fi + + if ! echo $MONGODB_URI |grep -q '?'; then + MONGODB_URI="$MONGODB_URI?" + fi + + MONGODB_URI="$MONGODB_URI&$opt" +} + +prepare_server() { + if test -n "$USE_OPT_MONGODB"; then + export BINDIR=/opt/mongodb/bin + export PATH=$BINDIR:$PATH + return + fi + + . $PROJECT_DIRECTORY/.mod/drivers-evergreen-tools/.evergreen/download-mongodb.sh + + get_distro + arch="${1:-$DISTRO}" + + get_mongodb_download_url_for "$arch" "$MONGODB_VERSION" + prepare_server_from_url "$MONGODB_DOWNLOAD_URL" "$MONGOSH_DOWNLOAD_URL" +} + +prepare_server_from_url() { + server_url=$1 + mongosh_url=$2 + + dirname=`basename $server_url |sed -e s/.tgz//` + mongodb_dir="$MONGO_ORCHESTRATION_HOME"/mdb/"$dirname" + mkdir -p "$mongodb_dir" + curl --retry 3 $server_url | tar xz -C "$mongodb_dir" --strip-components 1 -f - + + if test -n "$mongosh_url"; then + curl --retry 3 $mongosh_url | tar xz -C "$mongodb_dir" --strip-components 1 -f - + fi + + BINDIR="$mongodb_dir"/bin + export PATH="$BINDIR":$PATH +} + +install_mlaunch_venv() { + python3 -V || true + if ! python3 -m venv -h >/dev/null; then + # Current virtualenv fails with + # https://github.com/pypa/virtualenv/issues/1630 + python3 -m pip install venv --user + fi + if ! python3 -m ensurepip -h > /dev/null; then + # Debian11/Ubuntu2204 have venv installed, but it is nonfunctional unless + # the python3-venv package is also installed (it lacks the ensurepip + # module). + sudo apt-get update && sudo apt-get install --yes python3-venv + fi + if test "$USE_SYSTEM_PYTHON_PACKAGES" = 1 && + python3 -m pip list |grep mtools + then + # Use the existing mtools-legacy + : + else + # Spawn a virtual environment, but only if one is not already + # active... + if test -z "$VIRTUAL_ENV"; then + venvpath="$MONGO_ORCHESTRATION_HOME"/venv + python3 -m venv $venvpath + . $venvpath/bin/activate + fi + + # [mlaunch] does not work: + # https://github.com/rueckstiess/mtools/issues/856 + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + #pip install 'mtools==1.7' 'pymongo==4.1' python-dateutil psutil + + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + pip install --upgrade setuptools + pip install 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil + fi +} + +install_mlaunch_pip() { + if test -n "$USE_OPT_MONGODB" && which mlaunch >/dev/null 2>&1; then + # mlaunch is preinstalled in the docker image, do not install it here + return + fi + + python -V || true + python3 -V || true + pythonpath="$MONGO_ORCHESTRATION_HOME"/python + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + pip install -t "$pythonpath" 'mtools-legacy[mlaunch]' 'pymongo<4' python-dateutil + export PATH="$pythonpath/bin":$PATH + export PYTHONPATH="$pythonpath" +} + +install_mlaunch_git() { + repo=$1 + branch=$2 + python -V || true + python3 -V || true + which pip || true + which pip3 || true + + if false; then + if ! virtualenv --version; then + python3 `which pip3` install --user virtualenv + export PATH=$HOME/.local/bin:$PATH + virtualenv --version + fi + + venvpath="$MONGO_ORCHESTRATION_HOME"/venv + virtualenv -p python3 $venvpath + . $venvpath/bin/activate + + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + pip3 install psutil pymongo python-dateutil + + git clone $repo mlaunch + cd mlaunch + git checkout origin/$branch + python3 setup.py install + cd .. + else + pip install --user 'virtualenv==13' + export PATH=$HOME/.local/bin:$PATH + + venvpath="$MONGO_ORCHESTRATION_HOME"/venv + virtualenv $venvpath + . $venvpath/bin/activate + + # dateutil dependency is missing in mtools: https://github.com/rueckstiess/mtools/issues/864 + pip install psutil pymongo python-dateutil + + git clone $repo mlaunch + (cd mlaunch && + git checkout origin/$branch && + python2 setup.py install + ) + fi +} + +install_haproxy() { + if ! command -v haproxy &> /dev/null; then + if ! command -v apt-get &> /dev/null; then + # no apt-get; assume RHEL + sudo yum -y install haproxy + else + sudo apt-get update && sudo apt-get install --yes haproxy + fi + else + echo 'haproxy is present' + fi +} + +install_cmake() { + if ! command -v cmake &> /dev/null; then + if ! command -v apt-get &> /dev/null; then + # no apt-get; assume RHEL + sudo yum -y install cmake libarchive + else + sudo apt-get update && sudo apt-get install --yes cmake + fi + else + echo 'cmake is present' + fi +} + +# This function sets followong global variables: +# server_cert_path +# server_ca_path +# server_client_cert_path +# +# These variables are used later to connect to processes via mongo client. +calculate_server_args() { + local mongo_version=`echo $MONGODB_VERSION |tr -d .` + + if test -z "$mongo_version"; then + echo "$MONGODB_VERSION must be set and not contain only dots" 1>&2 + exit 3 + fi + + if test $mongo_version = latest; then + mongo_version=70 + fi + + local args="--setParameter enableTestCommands=1" + + if test $mongo_version -ge 50; then + args="$args --setParameter acceptApiVersion2=1" + elif test $mongo_version -ge 47; then + args="$args --setParameter acceptAPIVersion2=1" + fi + + args="$args --setParameter diagnosticDataCollectionEnabled=false" + + local uri_options= + if test "$TOPOLOGY" = replica-set; then + args="$args --replicaset --name test-rs --nodes 2 --arbiter" + export HAVE_ARBITER=1 + elif test "$TOPOLOGY" = replica-set-single-node; then + args="$args --replicaset --name test-rs --nodes 1" + elif test "$TOPOLOGY" = sharded-cluster; then + args="$args --replicaset --nodes 2 --sharded 1 --name test-rs" + if test -z "$SINGLE_MONGOS"; then + args="$args --mongos 2" + fi + elif test "$TOPOLOGY" = standalone; then + args="$args --single" + elif test "$TOPOLOGY" = load-balanced; then + args="$args --replicaset --nodes 2 --sharded 1 --name test-rs --port 27117" + if test -z "$MRSS_ROOT"; then + echo "Please set MRSS_ROOT" 1>&2 + exit 2 + fi + if test -n "$SINGLE_MONGOS"; then + haproxy_config=$MRSS_ROOT/share/haproxy-1.conf + else + args="$args --mongos 2" + haproxy_config=$MRSS_ROOT/share/haproxy-2.conf + fi + uri_options="$uri_options&loadBalanced=true" + else + echo "Unknown topology: $TOPOLOGY" 1>&2 + exit 1 + fi + if test -n "$MMAPV1"; then + args="$args --storageEngine mmapv1 --smallfiles --noprealloc" + uri_options="$uri_options&retryReads=false&retryWrites=false" + fi + if test "$AUTH" = auth; then + args="$args --auth --username bob --password pwd123" + elif test "$AUTH" = x509; then + args="$args --auth --username bootstrap --password bootstrap" + elif echo "$AUTH" |grep -q ^aws; then + args="$args --auth --username bootstrap --password bootstrap" + args="$args --setParameter authenticationMechanisms=MONGODB-AWS,SCRAM-SHA-1,SCRAM-SHA-256" + uri_options="$uri_options&authMechanism=MONGODB-AWS&authSource=\$external" + fi + + if test -n "$OCSP"; then + if test -z "$OCSP_ALGORITHM"; then + echo "OCSP_ALGORITHM must be set if OCSP is set" 1>&2 + exit 1 + fi + fi + + if test "$SSL" = ssl || test -n "$OCSP_ALGORITHM"; then + if test -n "$OCSP_ALGORITHM"; then + if test "$OCSP_MUST_STAPLE" = 1; then + server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server-mustStaple.pem + else + server_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem + fi + server_ca_path=spec/support/ocsp/$OCSP_ALGORITHM/ca.crt + server_client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem + else + server_cert_path=spec/support/certificates/server-second-level-bundle.pem + server_ca_path=spec/support/certificates/ca.crt + server_client_cert_path=spec/support/certificates/client.pem + fi + + if test -n "$OCSP_ALGORITHM"; then + client_cert_path=spec/support/ocsp/$OCSP_ALGORITHM/server.pem + elif test "$AUTH" = x509; then + client_cert_path=spec/support/certificates/client-x509.pem + + uri_options="$uri_options&authMechanism=MONGODB-X509" + elif echo $RVM_RUBY |grep -q jruby; then + # JRuby does not grok chained certificate bundles - + # https://github.com/jruby/jruby-openssl/issues/181 + client_cert_path=spec/support/certificates/client.pem + else + client_cert_path=spec/support/certificates/client-second-level-bundle.pem + fi + + uri_options="$uri_options&tls=true&"\ +"tlsCAFile=$server_ca_path&"\ +"tlsCertificateKeyFile=$client_cert_path" + + args="$args --sslMode requireSSL"\ +" --sslPEMKeyFile $server_cert_path"\ +" --sslCAFile $server_ca_path"\ +" --sslClientCertificate $server_client_cert_path" + fi + + # Docker forwards ports to the external interface, not to the loopback. + # Hence we must bind to all interfaces here. + if test -n "$BIND_ALL"; then + args="$args --bind_ip_all" + fi + + # MongoDB servers pre-4.2 do not enable zlib compression by default + if test "$COMPRESSOR" = snappy; then + args="$args --networkMessageCompressors snappy" + elif test "$COMPRESSOR" = zlib; then + args="$args --networkMessageCompressors zlib" + fi + + if test -n "$OCSP_ALGORITHM" || test -n "$OCSP_VERIFIER"; then + python3 -m pip install asn1crypto oscrypto flask + fi + + local ocsp_args= + if test -n "$OCSP_ALGORITHM"; then + if test -z "$server_ca_path"; then + echo "server_ca_path must have been set" 1>&2 + exit 1 + fi + ocsp_args="--ca_file $server_ca_path" + if test "$OCSP_DELEGATE" = 1; then + ocsp_args="$ocsp_args \ + --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.crt \ + --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ocsp-responder.key \ + " + else + ocsp_args="$ocsp_args \ + --ocsp_responder_cert spec/support/ocsp/$OCSP_ALGORITHM/ca.crt \ + --ocsp_responder_key spec/support/ocsp/$OCSP_ALGORITHM/ca.key \ + " + fi + if test -n "$OCSP_STATUS"; then + ocsp_args="$ocsp_args --fault $OCSP_STATUS" + fi + fi + + OCSP_ARGS="$ocsp_args" + SERVER_ARGS="$args" + URI_OPTIONS="$uri_options" +} + +launch_ocsp_mock() { + if test -n "$OCSP_ARGS"; then + # Bind to 0.0.0.0 for Docker + python3 spec/support/ocsp/ocsp_mock.py $OCSP_ARGS -b 0.0.0.0 -p 8100 & + OCSP_MOCK_PID=$! + fi +} + +launch_server() { + local dbdir="$1" + python3 -m mtools.mlaunch.mlaunch --dir "$dbdir" --binarypath "$BINDIR" $SERVER_ARGS + + if test "$TOPOLOGY" = sharded-cluster && test $MONGODB_VERSION = 3.6; then + # On 3.6 server the sessions collection is not immediately available, + # so we run the refreshLogicalSessionCacheNow command on the config server + # and again on each mongos in order for the mongoses + # to correctly report logicalSessionTimeoutMinutes. + mongos_regex="\s*mongos\s+([0-9]+)\s+running\s+[0-9]+" + config_server_regex="\s*config\sserver\s+([0-9]+)\s+running\s+[0-9]+" + config_server="" + mongoses=() + if test "$AUTH" = auth + then + base_url="mongodb://bob:pwd123@localhost" + else + base_url="mongodb://localhost" + fi + if test "$SSL" = "ssl" + then + mongo_command="${BINDIR}/mongo --ssl --sslPEMKeyFile $server_cert_path --sslCAFile $server_ca_path" + else + mongo_command="${BINDIR}/mongo" + fi + + while read -r line + do + if [[ $line =~ $config_server_regex ]] + then + port="${BASH_REMATCH[1]}" + config_server="${base_url}:${port}" + fi + if [[ $line =~ $mongos_regex ]] + then + port="${BASH_REMATCH[1]}" + mongoses+=("${base_url}:${port}") + fi + done < <(python2 -m mtools.mlaunch.mlaunch list --dir "$dbdir" --binarypath "$BINDIR") + + if [ -n "$config_server" ]; then + ${mongo_command} "$config_server" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")' + for mongos in ${mongoses[*]} + do + ${mongo_command} "$mongos" --eval 'db.adminCommand("refreshLogicalSessionCacheNow")' + done + fi + fi + + if test "$TOPOLOGY" = load-balanced; then + if test -z "$haproxy_config"; then + echo haproxy_config should have been set 1>&2 + exit 3 + fi + + haproxy -D -f $haproxy_config -p $mongodb_dir/haproxy.pid + fi +} diff --git a/spec/shared/shlib/set_env.sh b/spec/shared/shlib/set_env.sh new file mode 100644 index 0000000000..a983c501c7 --- /dev/null +++ b/spec/shared/shlib/set_env.sh @@ -0,0 +1,110 @@ +# When changing, also update the hash in share/Dockerfile. +JDK_VERSION=jdk21 + +set_env_java() { + ls -l /opt || true + ls -l /usr/lib/jvm || true + + # Use toolchain java if it exists + if [ -f /opt/java/$JDK_VERSION/bin/java ]; then + export JAVACMD=/opt/java/$JDK_VERSION/bin/java + else + echo Could not find $JDK_VERSION in /opt/java + fi + + if test -n "$JAVACMD"; then + eval $JAVACMD -version + elif which java 2>/dev/null; then + java -version + else + echo No java runtime found + fi +} + +set_env_python() { + if test "$DOCKER_PRELOAD" != 1; then + if test -n "$DOCKER"; then + # If we are running in Docker and not preloading, we need to fetch the + # Python binary. + curl -fL --retry 3 https://github.com/p-mongodb/deps/raw/main/"$arch"-python37.tar.xz | \ + tar xfJ - -C /opt + fi + + if test -d /opt/python/3.7/bin; then + # Most Evergreen configurations. + export PATH=/opt/python/3.7/bin:$PATH + elif test -d /opt/python37/bin; then + # Configurations that use Docker in Evergreen - these don't preload. + export PATH=/opt/python37/bin:$PATH + fi + + python3 -V + fi +} + +set_env_node() { + if test "$DOCKER_PRELOAD" != 1; then + dir=`ls -d /opt/nodejs/node-v12* |head -1` + if test -z "$dir"; then + echo "Node 12 missing" 1>&2 + exit 2 + fi + export PATH="$dir/bin:$PATH" + elif test -d /opt/node/bin; then + # Node from toolchain in Evergreen + export PATH=/opt/node/bin:$PATH + fi + + node -v +} + +set_env_ruby() { + if test -z "$RVM_RUBY"; then + echo "Empty RVM_RUBY, aborting" + exit 2 + fi + + #ls -l /opt + + # Necessary for jruby + set_env_java + + if [ "$RVM_RUBY" == "ruby-head" ]; then + # When we use ruby-head, we do not install the Ruby toolchain. + # But we still need Python 3.6+ to run mlaunch. + # Since the ruby-head tests are run on ubuntu1604, we can use the + # globally installed Python toolchain. + #export PATH=/opt/python/3.7/bin:$PATH + + # 12.04, 14.04 and 16.04 are good + curl --retry 3 -fL http://rubies.travis-ci.org/ubuntu/`lsb_release -rs`/x86_64/ruby-head.tar.bz2 |tar xfj - + # TODO adjust gem path? + export PATH=`pwd`/ruby-head/bin:`pwd`/ruby-head/lib/ruby/gems/2.6.0/bin:$PATH + ruby --version + ruby --version |grep dev + elif test "$SYSTEM_RUBY" = 1; then + # Nothing + : + else + if test "$USE_OPT_TOOLCHAIN" = 1; then + # Nothing, also PATH is already set + : + else + # For testing unpublished builds: + #build_url=https://s3.amazonaws.com/mciuploads/mongo-ruby-toolchain/library/`host_distro`/$RVM_RUBY.tar.xz + + build_url=http://boxes.10gen.com/build/toolchain-drivers/mongo-ruby-toolchain/library/`host_distro`/$RVM_RUBY.tar.xz + curl --retry 3 -fL $build_url |tar Jxf - + export PATH=`pwd`/rubies/$RVM_RUBY/bin:$PATH + fi + + ruby --version + + # Ensure we're using the right ruby + ruby_name=`echo $RVM_RUBY |awk -F- '{print $1}'` + ruby_version=`echo $RVM_RUBY |awk -F- '{print $2}' |cut -c 1-3` + + ruby -v |fgrep $ruby_name + ruby -v |fgrep $ruby_version + fi +}