From f8e5583e083c2ec7f1c99df84cfd3e114fe76b8d Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:04:02 +0100 Subject: [PATCH 01/14] RUBY-3571 Add TransientTransactionError label to PoolClearedError (#2906) --- .github/workflows/test.yml | 16 ++++++++-------- lib/mongo/error/pool_cleared_error.rb | 1 + spec/mongo/error/pool_cleared_error_spec.rb | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 spec/mongo/error/pool_cleared_error_spec.rb diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3f468fe8b..7144427790 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,33 +16,33 @@ jobs: fail-fast: false matrix: os: [ ubuntu-20.04 ] - ruby: ["2.7", "3.0", "3.1", "3.2"] - mongodb: ["3.6", "4.4", "5.0", "6.0", "7.0"] + ruby: ["2.7", "3.0", "3.1", "3.2", "3.3"] + mongodb: ["4.4", "5.0", "6.0", "7.0", "8.0"] topology: [replica_set, sharded_cluster] include: - os: macos ruby: "2.7" - mongodb: "5.0" + mongodb: "7.0" topology: server - os: macos ruby: "3.0" - mongodb: "5.0" + mongodb: "7.0" topology: server - os: ubuntu-latest ruby: "2.7" - mongodb: "5.0" + mongodb: "7.0" topology: server - os: ubuntu-latest ruby: "3.1" - mongodb: "5.0" + mongodb: "7.0" topology: server - os: ubuntu-latest ruby: "3.2" - mongodb: "5.0" + mongodb: "7.0" topology: server - os: ubuntu-latest ruby: "3.2" - mongodb: "6.0" + mongodb: "8.0" topology: replica_set steps: - name: repo checkout diff --git a/lib/mongo/error/pool_cleared_error.rb b/lib/mongo/error/pool_cleared_error.rb index 6f8b493e1a..d52b1ae198 100644 --- a/lib/mongo/error/pool_cleared_error.rb +++ b/lib/mongo/error/pool_cleared_error.rb @@ -31,6 +31,7 @@ class PoolClearedError < PoolError # # @api private def initialize(address, pool) + add_label('TransientTransactionError') super(address, pool, "Connection to #{address} interrupted due to server monitor timeout " + "(for pool 0x#{pool.object_id})") diff --git a/spec/mongo/error/pool_cleared_error_spec.rb b/spec/mongo/error/pool_cleared_error_spec.rb new file mode 100644 index 0000000000..38a3d6a9e3 --- /dev/null +++ b/spec/mongo/error/pool_cleared_error_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'lite_spec_helper' + +describe Mongo::Error::PoolClearedError do + describe '#initialize' do + let(:error) do + described_class.new( + instance_double(Mongo::Address), instance_double(Mongo::Server::ConnectionPool) + ) + end + + it 'appends TransientTransactionError' do + expect(error.labels).to include('TransientTransactionError') + end + end +end From 45bfc81de360d8da1e17a1b58f56d196d012a6ba Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 4 Nov 2024 08:12:11 -0700 Subject: [PATCH 02/14] RUBY-3164 Use mongosh instead of mongo (#2905) * RUBY-3164 use mongosh instead of mongo * config tweaks * use correct ubuntu version * submodule shenanigans trying to reference the experimental spec/shared changes for installing mongosh * fetch updates to spec/shared * use the distro detected by drivers-evergreen-tools * there is no 5.3 in the mongodb download script, so let's ignore it * bump spec/shared * make sure PROJECT_DIRECTORY is set so the AWS tests work * swich spec/shared back to canonical repo --- .evergreen/config.yml | 27 +++++--------------- .evergreen/config/axes.yml.erb | 5 ---- .evergreen/config/standard.yml.erb | 25 +++++------------- .evergreen/functions-kerberos.sh | 4 +-- .evergreen/run-tests-kerberos-integration.sh | 2 +- .evergreen/run-tests.sh | 10 +++++--- .gitmodules | 2 +- spec/README.aws-auth.md | 4 +-- spec/README.md | 6 ++--- spec/shared | 2 +- spec/support/certificates/README.md | 10 ++++---- 11 files changed, 35 insertions(+), 62 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0dfd9fab60..10fb23b8cf 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -927,11 +927,6 @@ axes: display_name: "6.0" variables: MONGODB_VERSION: "6.0" - - id: "5.3" - display_name: "5.3" - variables: - MONGODB_VERSION: "5.3" - CRYPT_SHARED_VERSION: "6.0.5" - id: "5.0" display_name: "5.0" variables: @@ -1363,7 +1358,7 @@ buildvariants: - matrix_name: "mongo-5.x" matrix_spec: ruby: ["ruby-3.3", "ruby-3.2", "jruby-9.4"] - mongodb-version: ['5.3'] + mongodb-version: ['5.0'] topology: ["standalone", "replica-set", "sharded-cluster"] os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" @@ -1524,15 +1519,10 @@ buildvariants: - matrix_name: "x509-tests" matrix_spec: auth-and-ssl: "x509" - ruby: 'ruby-3.1' - # needs the latest_5x_mdb because run-tests.sh uses `mongo` to configure - # the server for certain auth mechanisms. Once run-tests.sh is made smart - # enough to install mongosh, and then use either mongo or mongosh - # (depending on server version and what's available), we can bump this to - # the latest stable db version. - mongodb-version: "5.3" + ruby: "ruby-3.3" + mongodb-version: "7.0" topology: standalone - os: ubuntu1804 + os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - name: "test-mlaunch" @@ -1721,13 +1711,8 @@ buildvariants: auth-and-ssl: [ aws-regular, aws-assume-role, aws-ecs, aws-web-identity ] ruby: "ruby-3.3" topology: standalone - # needs the latest_5x_mdb because run-tests.sh uses `mongo` to configure - # the server for certain auth mechanisms. Once run-tests.sh is made smart - # enough to install mongosh, and then use either mongo or mongosh - # (depending on server version and what's available), we can bump this to - # the latest stable db version. - mongodb-version: "5.3" - os: ubuntu1804 + mongodb-version: "7.0" + os: ubuntu2204 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" tasks: - name: "test-aws-auth" diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index dbdcfbfdb6..44e018f5de 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -30,11 +30,6 @@ axes: display_name: "6.0" variables: MONGODB_VERSION: "6.0" - - id: "5.3" - display_name: "5.3" - variables: - MONGODB_VERSION: "5.3" - CRYPT_SHARED_VERSION: "6.0.5" - id: "5.0" display_name: "5.0" variables: diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 83cf0157db..4c4ea57630 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -37,9 +37,8 @@ actual_and_upcoming_mdb = %w( latest 8.0 7.0 ) recent_mdb = %w( 8.0 7.0 ) - latest_5x_mdb = "5.3".inspect # so it gets quoted as a string - all_dbs = %w(latest 8.0 7.0 6.0 5.3 5.0 4.4 4.2 4.0 3.6) + all_dbs = %w(latest 8.0 7.0 6.0 5.0 4.4 4.2 4.0 3.6) %> buildvariants: @@ -87,7 +86,7 @@ buildvariants: - matrix_name: "mongo-5.x" matrix_spec: ruby: <%= recent_rubies %> - mongodb-version: ['5.3'] + mongodb-version: ['5.0'] topology: <%= topologies %> os: ubuntu1804 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" @@ -248,15 +247,10 @@ buildvariants: - matrix_name: "x509-tests" matrix_spec: auth-and-ssl: "x509" - ruby: 'ruby-3.1' - # needs the latest_5x_mdb because run-tests.sh uses `mongo` to configure - # the server for certain auth mechanisms. Once run-tests.sh is made smart - # enough to install mongosh, and then use either mongo or mongosh - # (depending on server version and what's available), we can bump this to - # the latest stable db version. - mongodb-version: <%= latest_5x_mdb %> + ruby: <%= latest_ruby %> + mongodb-version: <%= latest_stable_mdb %> topology: standalone - os: ubuntu1804 + os: ubuntu2204 display_name: "${mongodb-version} ${topology} ${auth-and-ssl} ${ruby}" tasks: - name: "test-mlaunch" @@ -390,13 +384,8 @@ buildvariants: auth-and-ssl: [ aws-regular, aws-assume-role, aws-ecs, aws-web-identity ] ruby: <%= latest_ruby %> topology: standalone - # needs the latest_5x_mdb because run-tests.sh uses `mongo` to configure - # the server for certain auth mechanisms. Once run-tests.sh is made smart - # enough to install mongosh, and then use either mongo or mongosh - # (depending on server version and what's available), we can bump this to - # the latest stable db version. - mongodb-version: <%= latest_5x_mdb %> - os: ubuntu1804 + mongodb-version: <%= latest_stable_mdb %> + os: ubuntu2204 display_name: "AWS ${auth-and-ssl} ${mongodb-version} ${ruby}" tasks: - name: "test-aws-auth" diff --git a/.evergreen/functions-kerberos.sh b/.evergreen/functions-kerberos.sh index 310e58218c..2ce893f175 100644 --- a/.evergreen/functions-kerberos.sh +++ b/.evergreen/functions-kerberos.sh @@ -76,8 +76,8 @@ configure_local_kerberos() { EOT `" - "$BINDIR"/mongo --eval "$create_user_cmd" - "$BINDIR"/mongo --eval 'db.getSiblingDB("kerberos").test.insert({kerberos: true, authenticated: "yeah"})' + "$BINDIR"/mongosh --eval "$create_user_cmd" + "$BINDIR"/mongosh --eval 'db.getSiblingDB("kerberos").test.insert({kerberos: true, authenticated: "yeah"})' pkill mongod sleep 1 diff --git a/.evergreen/run-tests-kerberos-integration.sh b/.evergreen/run-tests-kerberos-integration.sh index e7002d898e..b56e22ec1a 100755 --- a/.evergreen/run-tests-kerberos-integration.sh +++ b/.evergreen/run-tests-kerberos-integration.sh @@ -55,7 +55,7 @@ configure_kerberos_ip_addr # To test authentication using the mongo shell, note that the host name # must be uppercased when it is used in the username. # The following call works when using the docker image: -# /opt/mongodb/bin/mongo --host $SASL_HOST --authenticationMechanism=GSSAPI \ +# /opt/mongodb/bin/mongosh --host $SASL_HOST --authenticationMechanism=GSSAPI \ # --authenticationDatabase='$external' --username $SASL_USER@`echo $SASL_HOST |tr a-z A-Z` echo "Install dependencies" diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a308e78486..653ca3b480 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -22,6 +22,10 @@ else set -x fi +if test -z "$PROJECT_DIRECTORY"; then + PROJECT_DIRECTORY=`realpath $(dirname $0)/..` +fi + MRSS_ROOT=`dirname "$0"`/../spec/shared . $MRSS_ROOT/shlib/distro.sh @@ -41,7 +45,7 @@ set_env_vars set_env_python set_env_ruby -prepare_server $arch +prepare_server if test "$DOCKER_PRELOAD" != 1; then install_mlaunch_venv @@ -118,7 +122,7 @@ elif test "$AUTH" = x509; then EOT `" - "$BINDIR"/mongo --tls \ + "$BINDIR"/mongosh --tls \ --tlsCAFile spec/support/certificates/ca.crt \ --tlsCertificateKeyFile spec/support/certificates/client-x509.pem \ -u bootstrap -p bootstrap \ @@ -286,7 +290,7 @@ fi export MONGODB_URI="mongodb://$hosts/?serverSelectionTimeoutMS=30000$uri_options" if echo "$AUTH" |grep -q ^aws-assume-role; then - $BINDIR/mongo "$MONGODB_URI" --eval 'db.runCommand({serverStatus: 1})' |wc + $BINDIR/mongosh "$MONGODB_URI" --eval 'db.runCommand({serverStatus: 1})' | wc fi set_fcv diff --git a/.gitmodules b/.gitmodules index e1bab0957a..6d428f359d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/mongodb-labs/drivers-evergreen-tools [submodule "spec/shared"] path = spec/shared - url = git@github.com:mongodb-labs/mongo-ruby-spec-shared.git + url = git@github.com:mongodb-labs/mongo-ruby-spec-shared.git diff --git a/spec/README.aws-auth.md b/spec/README.aws-auth.md index 5f4bfef08d..82a757ca47 100644 --- a/spec/README.aws-auth.md +++ b/spec/README.aws-auth.md @@ -24,7 +24,7 @@ AWS authentication, and add a bootstrap user: Then connect as the bootstrap user and create AWS-mapped users: - mongo mongodb://root:toor@localhost:27017 + mongosh mongodb://root:toor@localhost:27017 # In the mongo shell: use $external @@ -41,7 +41,7 @@ With the server user created, it is possible to authenticate using AWS. The following example uses regular user credentials for an IAM user created as described in the next section; - mongo 'mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external' + mongosh 'mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external' To authenticate, provide the IAM user's access key id as the username and secret access key as the password. Note that the username and the password diff --git a/spec/README.md b/spec/README.md index cd100311c3..98d4b5a0d2 100644 --- a/spec/README.md +++ b/spec/README.md @@ -292,7 +292,7 @@ to a variable as follows: Use the MongoDB shell to execute this command: - mongo --tls \ + mongosh --tls \ --tlsCAFile `pwd`/spec/support/certificates/ca.crt \ --tlsCertificateKeyFile `pwd`/spec/support/certificates/client-x509.pem \ -u bootstrap -p bootstrap \ @@ -301,14 +301,14 @@ Use the MongoDB shell to execute this command: Verify that authentication is required by running the following command, which should fail: - mongo --tls \ + mongosh --tls \ --tlsCAFile `pwd`/spec/support/certificates/ca.crt \ --tlsCertificateKeyFile `pwd`/spec/support/certificates/client-x509.pem \ --eval 'db.serverStatus()' Verify that X.509 authentication works by running the following command: - mongo --tls \ + mongosh --tls \ --tlsCAFile `pwd`/spec/support/certificates/ca.crt \ --tlsCertificateKeyFile `pwd`/spec/support/certificates/client-x509.pem \ --authenticationDatabase '$external' \ diff --git a/spec/shared b/spec/shared index 2fe5724b0a..d73378a4cc 160000 --- a/spec/shared +++ b/spec/shared @@ -1 +1 @@ -Subproject commit 2fe5724b0a586fbeac602f15f7d43ccc7bbe531b +Subproject commit d73378a4cc9d07cda8375e1300c15add2c6b9a6c diff --git a/spec/support/certificates/README.md b/spec/support/certificates/README.md index c6251ea80b..d447a19c66 100644 --- a/spec/support/certificates/README.md +++ b/spec/support/certificates/README.md @@ -81,9 +81,9 @@ To sum up, openssl's command line tools appear to only handle certificate chains provided by the client when the server is verifying them, not the other way around and not when trying to standalone verify the chain. -## Manual Testing - mongo +## Manual Testing - mongosh -When it comes to `mongod` and `mongo`, certificate chains are supported in +When it comes to `mongod` and `mongosh`, certificate chains are supported in both directions: mongod --sslMode requireSSL \ @@ -91,16 +91,16 @@ both directions: --sslPEMKeyFile server-second-level-bundle.pem \ --sslClientCertificate client.pem - mongo --host localhost --ssl \ + mongosh --host localhost --ssl \ --sslCAFile ca.crt \ --sslPEMKeyFile client-second-level-bundle.pem -The `--host` option needs to be given to `mongo` because the certificates here +The `--host` option needs to be given to `mongosh` because the certificates here do not include 127.0.0.1 in subject alternate name. If the intermediate certificate is not provided, the connection should fail. # Expected to fail - mongo --host localhost --ssl \ + mongosh --host localhost --ssl \ --sslCAFile ca.crt \ --sslPEMKeyFile client-second-level.pem From 4b9b347deddb1b6c8ab0cfe598fadf477b7801f8 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 5 Nov 2024 08:49:58 -0700 Subject: [PATCH 03/14] RUBY-1934 clean up DNS server termination (#2903) * RUBY-1934 clean up DNS server termination The ticket originally called for pulling in another dependency to manage the DNS server, but I don't think it's necessary. The async-container code would be doing essentially what we already had, just with a cleaner technique for sending the termination signal. * need to require rubydns * even cleaner * RubyDNS is failing oddly with Ruby 2.7, so let's just skip it --- spec/integration/reconnect_spec.rb | 4 +- spec/integration/srv_monitoring_spec.rb | 4 +- spec/integration/srv_spec.rb | 4 -- spec/support/common_shortcuts.rb | 56 +++++++++---------------- spec/support/constraints.rb | 10 +++++ spec/support/dns.rb | 16 ------- 6 files changed, 32 insertions(+), 62 deletions(-) delete mode 100644 spec/support/dns.rb diff --git a/spec/integration/reconnect_spec.rb b/spec/integration/reconnect_spec.rb index f478e233dd..0fa47c29af 100644 --- a/spec/integration/reconnect_spec.rb +++ b/spec/integration/reconnect_spec.rb @@ -111,6 +111,8 @@ # thread.kill should've similarly failed, but it doesn't. fails_on_jruby + minimum_mri_version '3.0.0' + it 'recreates SRV monitor' do wait_for_discovery @@ -181,8 +183,6 @@ end around do |example| - require 'support/dns' - rules = [ ['_mongodb._tcp.test-fake.test.build.10gen.cc', :srv, [0, 0, 2799, 'localhost.test.build.10gen.cc'], diff --git a/spec/integration/srv_monitoring_spec.rb b/spec/integration/srv_monitoring_spec.rb index 6d7da1f730..ffa58b053f 100644 --- a/spec/integration/srv_monitoring_spec.rb +++ b/spec/integration/srv_monitoring_spec.rb @@ -76,9 +76,7 @@ # NotImplementedError: recvmsg_nonblock is not implemented fails_on_jruby - before(:all) do - require 'support/dns' - end + minimum_mri_version '3.0.0' around do |example| # Speed up the tests by listening on the fake ports we are using. diff --git a/spec/integration/srv_spec.rb b/spec/integration/srv_spec.rb index da3d529339..594ff9e69b 100644 --- a/spec/integration/srv_spec.rb +++ b/spec/integration/srv_spec.rb @@ -12,10 +12,6 @@ # NotImplementedError: recvmsg_nonblock is not implemented fails_on_jruby - before(:all) do - require 'support/dns' - end - let(:uri) do "mongodb+srv://test-fake.test.build.10gen.cc/?tls=#{SpecConfig.instance.ssl?}&tlsInsecure=true" end diff --git a/spec/support/common_shortcuts.rb b/spec/support/common_shortcuts.rb index 8eacdf2c7c..bdb4c8511f 100644 --- a/spec/support/common_shortcuts.rb +++ b/spec/support/common_shortcuts.rb @@ -337,51 +337,33 @@ def stop_monitoring(*clients) [:tcp, "0.0.0.0", 5300], ] - def mock_dns(config) - semaphore = Mongo::Semaphore.new - - thread = Thread.new do - RubyDNS::run_server(DNS_INTERFACES) do - config.each do |(query, type, *answers)| - - resource_cls = Resolv::DNS::Resource::IN.const_get(type.to_s.upcase) - resources = answers.map do |answer| - resource_cls.new(*answer) - end - match(query, resource_cls) do |req| - req.add(resources) - end + # Starts the DNS server and returns it; should be run from within an + # Async block. Prefer #mock_dns instead, which does the setup for you. + def start_dns_server(config) + RubyDNS::run_server(DNS_INTERFACES) do + config.each do |(query, type, *answers)| + resource_cls = Resolv::DNS::Resource::IN.const_get(type.to_s.upcase) + resources = answers.map do |answer| + resource_cls.new(*answer) end - semaphore.signal + match(query, resource_cls) do |req| + req.add(resources) + end end end + end - semaphore.wait + # Starts and runs a DNS server, then yields to the attached block. + def mock_dns(config) + # only require rubydns when we need it; it's MRI-only. + require 'rubydns' - begin + Async do |task| + server = start_dns_server(config) yield ensure - 10.times do - if $last_async_task - break - end - sleep 0.5 - end - - # Hack to stop the server - https://github.com/socketry/rubydns/issues/75 - if $last_async_task.nil? - STDERR.puts "No async task - server never started?" - else - begin - $last_async_task.stop - rescue NoMethodError => e - STDERR.puts "Error stopping async task: #{e}" - end - end - - thread.kill - thread.join + server.stop end end diff --git a/spec/support/constraints.rb b/spec/support/constraints.rb index 6d92409937..8c7f3f940d 100644 --- a/spec/support/constraints.rb +++ b/spec/support/constraints.rb @@ -17,6 +17,16 @@ def require_local_tls end end + def minimum_mri_version(version) + require_mri + + before(:all) do + if RUBY_VERSION < version + skip "Ruby #{version} or greater is required" + end + end + end + def forbid_x509_auth before(:all) do skip 'X.509 auth not allowed' if SpecConfig.instance.x509_auth? diff --git a/spec/support/dns.rb b/spec/support/dns.rb deleted file mode 100644 index d868f7bc3b..0000000000 --- a/spec/support/dns.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true -# rubocop:todo all - -require 'rubydns' - -# Hack to stop the server - https://github.com/socketry/rubydns/issues/75 -module Async - class Task - alias :run_without_record :run - def run(*args) - run_without_record.tap do - $last_async_task = self - end - end - end -end From 192418721c4f70444812fcc6f483653eef9111c6 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 5 Nov 2024 08:52:56 -0700 Subject: [PATCH 04/14] RUBY-1933 Add debug level logging around initial DNS seed list query (#2904) * RUBY-1933 Log when attempting to resolve a srv hostname * method comments --- lib/mongo/client.rb | 103 ++++++++++++++++++++-------- lib/mongo/uri/srv_protocol.rb | 2 + spec/mongo/uri/srv_protocol_spec.rb | 14 +++- 3 files changed, 90 insertions(+), 29 deletions(-) diff --git a/lib/mongo/client.rb b/lib/mongo/client.rb index a5d3e030b4..70f5768628 100644 --- a/lib/mongo/client.rb +++ b/lib/mongo/client.rb @@ -502,35 +502,15 @@ def hash def initialize(addresses_or_uri, options = nil) options = options ? options.dup : {} - srv_uri = nil - if addresses_or_uri.is_a?(::String) - uri = URI.get(addresses_or_uri, options) - if uri.is_a?(URI::SRVProtocol) - # If the URI is an SRV URI, note this so that we can start - # SRV polling if the topology is a sharded cluster. - srv_uri = uri - end - addresses = uri.servers - uri_options = uri.client_options.dup - # Special handing for :write and :write_concern: allow client Ruby - # options to override URI options, even when the Ruby option uses the - # deprecated :write key and the URI option uses the current - # :write_concern key - if options[:write] - uri_options.delete(:write_concern) - end - options = uri_options.merge(options) - @srv_records = uri.srv_records - else - addresses = addresses_or_uri - addresses.each do |addr| - if addr =~ /\Amongodb(\+srv)?:\/\//i - raise ArgumentError, "Host '#{addr}' should not contain protocol. Did you mean to not use an array?" - end - end + processed = process_addresses(addresses_or_uri, options) - @srv_records = nil - end + uri = processed[:uri] + addresses = processed[:addresses] + options = processed[:options] + + # If the URI is an SRV URI, note this so that we can start + # SRV polling if the topology is a sharded cluster. + srv_uri = uri if uri.is_a?(URI::SRVProtocol) options = self.class.canonicalize_ruby_options(options) @@ -1217,6 +1197,73 @@ def timeout_sec private + # Attempts to parse the given list of addresses, using the provided options. + # + # @param [ String | Array ] addresses the list of addresses + # @param [ Hash ] options the options that may drive how the list is + # processed. + # + # @return [ Hash<:uri, :addresses, :options> ] the results of processing the + # list of addresses. + def process_addresses(addresses, options) + if addresses.is_a?(String) + process_addresses_string(addresses, options) + else + process_addresses_array(addresses, options) + end + end + + # Attempts to parse the given list of addresses, using the provided options. + # + # @param [ String ] addresses the list of addresses + # @param [ Hash ] options the options that may drive how the list is + # processed. + # + # @return [ Hash<:uri, :addresses, :options> ] the results of processing the + # list of addresses. + def process_addresses_string(addresses, options) + {}.tap do |processed| + processed[:uri] = uri = URI.get(addresses, options) + processed[:addresses] = uri.servers + + uri_options = uri.client_options.dup + # Special handing for :write and :write_concern: allow client Ruby + # options to override URI options, even when the Ruby option uses the + # deprecated :write key and the URI option uses the current + # :write_concern key + if options[:write] + uri_options.delete(:write_concern) + end + + processed[:options] = uri_options.merge(options) + + @srv_records = uri.srv_records + end + end + + # Attempts to parse the given list of addresses, using the provided options. + # + # @param [ Array ] addresses the list of addresses + # @param [ Hash ] options the options that may drive how the list is + # processed. + # + # @return [ Hash<:uri, :addresses, :options> ] the results of processing the + # list of addresses. + def process_addresses_array(addresses, options) + {}.tap do |processed| + processed[:addresses] = addresses + processed[:options] = options + + addresses.each do |addr| + if addr =~ /\Amongodb(\+srv)?:\/\//i + raise ArgumentError, "Host '#{addr}' should not contain protocol. Did you mean to not use an array?" + end + end + + @srv_records = nil + end + end + # Create a new encrypter object using the client's auto encryption options def build_encrypter @encrypter = Crypt::AutoEncrypter.new( diff --git a/lib/mongo/uri/srv_protocol.rb b/lib/mongo/uri/srv_protocol.rb index 5324c9caf9..4336d1c814 100644 --- a/lib/mongo/uri/srv_protocol.rb +++ b/lib/mongo/uri/srv_protocol.rb @@ -147,6 +147,8 @@ def parse!(remaining) validate_srv_hostname(hostname) @query_hostname = hostname + log_debug "attempting to resolve #{hostname}" + @srv_result = resolver.get_records(hostname, uri_options[:srv_service_name], uri_options[:srv_max_hosts]) if srv_result.empty? raise Error::NoSRVRecords.new(NO_SRV_RECORDS % hostname) diff --git a/spec/mongo/uri/srv_protocol_spec.rb b/spec/mongo/uri/srv_protocol_spec.rb index 3efc41f910..a1b13a0195 100644 --- a/spec/mongo/uri/srv_protocol_spec.rb +++ b/spec/mongo/uri/srv_protocol_spec.rb @@ -2,6 +2,7 @@ # rubocop:todo all require 'lite_spec_helper' +require 'support/recording_logger' describe Mongo::URI::SRVProtocol do require_external_connectivity @@ -21,6 +22,18 @@ end end + describe 'logging' do + let(:logger) { RecordingLogger.new } + let(:uri) { described_class.new(string, logger: logger) } + let(:host) { 'test5.test.build.10gen.cc' } + let(:string) { "#{scheme}#{host}" } + + it 'logs when resolving the address' do + expect { uri }.not_to raise_error + expect(logger.contents).to include("attempting to resolve #{host}") + end + end + describe 'invalid uris' do context 'when there is more than one hostname' do @@ -228,7 +241,6 @@ end describe 'valid uris' do - require_external_connectivity describe 'invalid query results' do From f1d1f911a9811b7c8de3c25d161519ef83705b6c Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:19:05 +0100 Subject: [PATCH 05/14] RUBY-3429 Retry failed KMS requests (#2907) --- .evergreen/run-tests.sh | 1 + .rubocop.yml | 3 + gemfiles/standard.rb | 2 +- lib/mongo/crypt/binding.rb | 58 ++++++++- lib/mongo/crypt/context.rb | 82 ++++++++----- lib/mongo/crypt/encryption_io.rb | 6 +- lib/mongo/crypt/handle.rb | 2 +- lib/mongo/error/kms_error.rb | 9 ++ .../kms_retry_prose_spec.rb | 112 ++++++++++++++++++ .../kms_tls_options_spec.rb | 2 +- spec/integration/search_indexes_prose_spec.rb | 2 - 11 files changed, 244 insertions(+), 35 deletions(-) create mode 100644 spec/integration/client_side_encryption/kms_retry_prose_spec.rb diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 653ca3b480..6dfa54c343 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -219,6 +219,7 @@ if test -n "$FLE"; then python3 -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert & python3 -u .evergreen/csfle/kms_kmip_server.py & python3 -u .evergreen/csfle/fake_azure.py & + python3 -u .evergreen/csfle/kms_failpoint_server.py --port 9003 & # Obtain temporary AWS credentials PYTHON=python3 . .evergreen/csfle/set-temp-creds.sh diff --git a/.rubocop.yml b/.rubocop.yml index fa7d04f8c6..63d1066dc4 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -110,3 +110,6 @@ Style/TrailingCommaInArrayLiteral: Style/TrailingCommaInHashLiteral: Enabled: false + +RSpec/ExampleLength: + Max: 10 diff --git a/gemfiles/standard.rb b/gemfiles/standard.rb index 65c628b56b..6a11bc3404 100644 --- a/gemfiles/standard.rb +++ b/gemfiles/standard.rb @@ -68,6 +68,6 @@ def standard_dependencies gem 'ruby-lsp', platforms: :mri end - gem 'libmongocrypt-helper', '~> 1.11.0' if ENV['FLE'] == 'helper' + gem 'libmongocrypt-helper', '~> 1.12.0' if ENV['FLE'] == 'helper' end # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength diff --git a/lib/mongo/crypt/binding.rb b/lib/mongo/crypt/binding.rb index 5d4b1fc830..e729802a8b 100644 --- a/lib/mongo/crypt/binding.rb +++ b/lib/mongo/crypt/binding.rb @@ -83,7 +83,7 @@ class Binding # will cause a `LoadError`. # # @api private - MIN_LIBMONGOCRYPT_VERSION = Gem::Version.new("1.7.0") + MIN_LIBMONGOCRYPT_VERSION = Gem::Version.new("1.12.0") # @!method self.mongocrypt_version(len) # @api private @@ -1113,6 +1113,62 @@ def self.check_kms_ctx_status(kms_context) end end + # @!method self.mongocrypt_kms_ctx_usleep(ctx) + # @api private + # + # Indicates how long to sleep before sending KMS request. + # + # @param [ FFI::Pointer ] ctx A pointer to a mongocrypt_ctx_t object. + # @return [ int64 ] A 64-bit encoded number of microseconds of how long to sleep. + attach_function :mongocrypt_kms_ctx_usleep, [:pointer], :int64 + + # Returns number of milliseconds to sleep before sending KMS request + # for the given KMS context. + # + # @param [ Mongo::Crypt::KmsContext ] kms_context KMS Context we are going + # to send KMS request for. + # @return [ Integer ] A 64-bit encoded number of microseconds to sleep. + def self.kms_ctx_usleep(kms_context) + mongocrypt_kms_ctx_usleep(kms_context.kms_ctx_p) + end + + # @!method self.mongocrypt_kms_ctx_fail(ctx) + # @api private + # + # Indicate a network-level failure. + # + # @param [ FFI::Pointer ] ctx A pointer to a mongocrypt_ctx_t object. + # @return [ Boolean ] whether the failed request may be retried. + attach_function :mongocrypt_kms_ctx_fail, [:pointer], :bool + + # Check whether the last failed request for the KMS context may be retried. + # + # @param [ Mongo::Crypt::KmsContext ] kms_context KMS Context + # @return [ true, false ] whether the failed request may be retried. + def self.kms_ctx_fail(kms_context) + mongocrypt_kms_ctx_fail(kms_context.kms_ctx_p) + end + + # @!method self.mongocrypt_setopt_retry_kms(crypt, enable) + # @api private + # + # Enable or disable KMS retry behavior. + # + # @param [ FFI::Pointer ] crypt A pointer to a mongocrypt_t object + # @param [ Boolean ] enable A boolean indicating whether to retry operations. + # @return [ Boolean ] indicating success. + attach_function :mongocrypt_setopt_retry_kms, [:pointer, :bool], :bool + + # Enable or disable KMS retry behavior. + # + # @param [ Mongo::Crypt::Handle ] handle + # @param [ true, false ] value whether to retry operations. + # @return [ true, fale ] true is the option was set, otherwise false. + def self.kms_ctx_setopt_retry_kms(handle, value) + mongocrypt_setopt_retry_kms(handle.ref, value) + end + + # @!method self.mongocrypt_kms_ctx_done(ctx) # @api private # diff --git a/lib/mongo/crypt/context.rb b/lib/mongo/crypt/context.rb index d8c6772999..625c89dc16 100644 --- a/lib/mongo/crypt/context.rb +++ b/lib/mongo/crypt/context.rb @@ -49,7 +49,6 @@ def initialize(mongocrypt_handle, io) Binding.mongocrypt_ctx_new(@mongocrypt_handle.ref), Binding.method(:mongocrypt_ctx_destroy) ) - @encryption_io = io @cached_azure_token = nil end @@ -90,35 +89,13 @@ def run_state_machine(timeout_holder) when :done return nil when :need_mongo_keys - filter = Binding.ctx_mongo_op(self) - - @encryption_io.find_keys(filter, timeout_ms: timeout_ms).each do |key| - mongocrypt_feed(key) if key - end - - mongocrypt_done + provide_keys(timeout_ms) when :need_mongo_collinfo - filter = Binding.ctx_mongo_op(self) - - result = @encryption_io.collection_info(@db_name, filter, timeout_ms: timeout_ms) - mongocrypt_feed(result) if result - - mongocrypt_done + provide_collection_info(timeout_ms) when :need_mongo_markings - cmd = Binding.ctx_mongo_op(self) - - result = @encryption_io.mark_command(cmd, timeout_ms: timeout_ms) - mongocrypt_feed(result) - - mongocrypt_done + provide_markings(timeout_ms) when :need_kms - while kms_context = Binding.ctx_next_kms_ctx(self) do - provider = Binding.kms_ctx_get_kms_provider(kms_context) - tls_options = @mongocrypt_handle.kms_tls_options(provider) - @encryption_io.feed_kms(kms_context, tls_options) - end - - Binding.ctx_kms_done(self) + feed_kms when :need_kms_credentials Binding.ctx_provide_kms_providers( self, @@ -134,6 +111,57 @@ def run_state_machine(timeout_holder) private + def provide_markings(timeout_ms) + cmd = Binding.ctx_mongo_op(self) + + result = @encryption_io.mark_command(cmd, timeout_ms: timeout_ms) + mongocrypt_feed(result) + + mongocrypt_done + end + + def provide_collection_info(timeout_ms) + filter = Binding.ctx_mongo_op(self) + + result = @encryption_io.collection_info(@db_name, filter, timeout_ms: timeout_ms) + mongocrypt_feed(result) if result + + mongocrypt_done + end + + def provide_keys(timeout_ms) + filter = Binding.ctx_mongo_op(self) + + @encryption_io.find_keys(filter, timeout_ms: timeout_ms).each do |key| + mongocrypt_feed(key) if key + end + + mongocrypt_done + end + + def feed_kms + while (kms_context = Binding.ctx_next_kms_ctx(self)) do + begin + delay = Binding.kms_ctx_usleep(kms_context) + sleep(delay / 1_000_000.0) unless delay.nil? + provider = Binding.kms_ctx_get_kms_provider(kms_context) + tls_options = @mongocrypt_handle.kms_tls_options(provider) + @encryption_io.feed_kms(kms_context, tls_options) + rescue Error::KmsError => e + if e.network_error? + if Binding.kms_ctx_fail(kms_context) + next + else + raise + end + else + raise + end + end + end + Binding.ctx_kms_done(self) + end + # Indicate that state machine is done feeding I/O responses back to libmongocrypt def mongocrypt_done Binding.mongocrypt_ctx_mongo_done(ctx_p) diff --git a/lib/mongo/crypt/encryption_io.rb b/lib/mongo/crypt/encryption_io.rb index 6bd5bccf4e..2417652d46 100644 --- a/lib/mongo/crypt/encryption_io.rb +++ b/lib/mongo/crypt/encryption_io.rb @@ -363,8 +363,10 @@ def with_ssl_socket(endpoint, tls_options, timeout_ms: nil) tls_options.merge(socket_options) ) yield(mongo_socket.socket) - rescue => e - raise Error::KmsError, "Error when connecting to KMS provider: #{e.class}: #{e.message}" + rescue Error::KmsError + raise + rescue StandardError => e + raise Error::KmsError.new("Error when connecting to KMS provider: #{e.class}: #{e.message}", network_error: true) ensure mongo_socket&.close end diff --git a/lib/mongo/crypt/handle.rb b/lib/mongo/crypt/handle.rb index b6cd0f7f67..cfc417d2c6 100644 --- a/lib/mongo/crypt/handle.rb +++ b/lib/mongo/crypt/handle.rb @@ -71,7 +71,7 @@ def initialize(kms_providers, kms_tls_options, options={}) Binding.mongocrypt_new, Binding.method(:mongocrypt_destroy) ) - + Binding.kms_ctx_setopt_retry_kms(self, true) @kms_providers = kms_providers @kms_tls_options = kms_tls_options diff --git a/lib/mongo/error/kms_error.rb b/lib/mongo/error/kms_error.rb index 98978d5301..77c1dfdfc4 100644 --- a/lib/mongo/error/kms_error.rb +++ b/lib/mongo/error/kms_error.rb @@ -20,6 +20,15 @@ class Error # A KMS-related error during client-side encryption. class KmsError < CryptError + def initialize(message, code: nil, network_error: nil) + @network_error = network_error + super(message, code: code) + end + end + + # @return [ true, false ] whether this error was caused by a network error. + def network_error? + @network_error == true end end end diff --git a/spec/integration/client_side_encryption/kms_retry_prose_spec.rb b/spec/integration/client_side_encryption/kms_retry_prose_spec.rb new file mode 100644 index 0000000000..29c1bac8ee --- /dev/null +++ b/spec/integration/client_side_encryption/kms_retry_prose_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require 'spec_helper' + +def simulate_failure(type, times = 1) + url = URI.parse("https://localhost:9003/set_failpoint/#{type}") + data = { count: times }.to_json + http = Net::HTTP.new(url.host, url.port) + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.ca_file = '.evergreen/x509gen/ca.pem' + request = Net::HTTP::Post.new(url.path, { 'Content-Type' => 'application/json' }) + request.body = data + http.request(request) +end + +describe 'KMS Retry Prose Spec' do + require_libmongocrypt + require_enterprise + min_server_version '4.2' + + include_context 'define shared FLE helpers' + + let(:key_vault_client) do + ClientRegistry.instance.new_local_client(SpecConfig.instance.addresses) + end + + let(:client_encryption) do + Mongo::ClientEncryption.new( + key_vault_client, + kms_tls_options: { + aws: default_kms_tls_options_for_provider, + gcp: default_kms_tls_options_for_provider, + azure: default_kms_tls_options_for_provider, + }, + key_vault_namespace: key_vault_namespace, + # For some reason libmongocrypt ignores custom endpoints for Azure and CGP + # kms_providers: aws_kms_providers.merge(azure_kms_providers).merge(gcp_kms_providers) + kms_providers: aws_kms_providers + ) + end + + shared_examples 'kms_retry prose spec' do + it 'createDataKey and encrypt with TCP retry' do + simulate_failure('network') + data_key_id = client_encryption.create_data_key(kms_provider, master_key: master_key) + simulate_failure('network') + expect do + client_encryption.encrypt(123, key_id: data_key_id, algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic') + end.not_to raise_error + end + + it 'createDataKey and encrypt with HTTP retry' do + simulate_failure('http') + data_key_id = client_encryption.create_data_key(kms_provider, master_key: master_key) + simulate_failure('http') + expect do + client_encryption.encrypt(123, key_id: data_key_id, algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic') + end.not_to raise_error + end + + it 'createDataKey fails after too many retries' do + simulate_failure('network', 4) + expect do + client_encryption.create_data_key(kms_provider, master_key: master_key) + end.to raise_error(Mongo::Error::KmsError) + end + end + + context 'with AWS KMS provider' do + let(:kms_provider) { 'aws' } + + let(:master_key) do + { + region: 'foo', + key: 'bar', + endpoint: '127.0.0.1:9003', + } + end + + include_examples 'kms_retry prose spec' + end + + context 'with GCP KMS provider', skip: 'For some reason libmongocrypt ignores custom endpoints for Azure and CGP' do + let(:kms_provider) { 'gcp' } + + let(:master_key) do + { + project_id: 'foo', + location: 'bar', + key_ring: 'baz', + key_name: 'qux', + endpoint: '127.0.0.1:9003' + } + end + + include_examples 'kms_retry prose spec' + end + + context 'with Azure KMS provider', skip: 'For some reason libmongocrypt ignores custom endpoints for Azure and CGP' do + let(:kms_provider) { 'azure' } + + let(:master_key) do + { + key_vault_endpoint: '127.0.0.1:9003', + key_name: 'foo', + } + end + + include_examples 'kms_retry prose spec' + end +end diff --git a/spec/integration/client_side_encryption/kms_tls_options_spec.rb b/spec/integration/client_side_encryption/kms_tls_options_spec.rb index 0766662670..a36b1d745b 100644 --- a/spec/integration/client_side_encryption/kms_tls_options_spec.rb +++ b/spec/integration/client_side_encryption/kms_tls_options_spec.rb @@ -323,7 +323,7 @@ } ) rescue Mongo::Error::KmsError => exc - exc.message.should =~ /Error when connecting to KMS provider/ + exc.message.should =~ /Error when connecting to KMS provider|Empty KMS response/ exc.message.should =~ /libmongocrypt error code/ exc.message.should_not =~ /CryptError/ else diff --git a/spec/integration/search_indexes_prose_spec.rb b/spec/integration/search_indexes_prose_spec.rb index 6093aff9d0..1ee3dfd2d9 100644 --- a/spec/integration/search_indexes_prose_spec.rb +++ b/spec/integration/search_indexes_prose_spec.rb @@ -147,7 +147,6 @@ def filter_results(result, names) .first end - # rubocop:disable RSpec/ExampleLength it 'succeeds' do expect(create_index).to be == name helper.wait_for(name) @@ -158,7 +157,6 @@ def filter_results(result, names) expect(index['latestDefinition']).to be == new_definition end - # rubocop:enable RSpec/ExampleLength end # Case 5: dropSearchIndex suppresses namespace not found errors From 6709482eeea58551cc4bcb86633b5ec07e769640 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Tue, 19 Nov 2024 20:02:13 +0100 Subject: [PATCH 06/14] New certs for OCSP tests (#2908) --- spec/support/certificates/atlas-ocsp.crt | 196 +++++++++++------------ 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/spec/support/certificates/atlas-ocsp.crt b/spec/support/certificates/atlas-ocsp.crt index 9a486282bb..f975ace900 100644 --- a/spec/support/certificates/atlas-ocsp.crt +++ b/spec/support/certificates/atlas-ocsp.crt @@ -2,52 +2,52 @@ Certificate: Data: Version: 3 (0x2) Serial Number: - 03:92:42:45:e6:7a:a2:13:84:ea:7c:7e:ce:da:6e:d4:63:67 + 03:10:94:1a:54:cc:b3:27:3c:ad:5f:e9:e7:60:69:ed:b9:4e Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, O=Let's Encrypt, CN=R10 Validity - Not Before: Aug 20 13:05:04 2024 GMT - Not After : Nov 18 13:05:03 2024 GMT + Not Before: Nov 4 12:07:32 2024 GMT + Not After : Feb 2 12:07:31 2025 GMT Subject: CN=*.g6fyiaq.mongodb-dev.net Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (4096 bit) Modulus: - 00:c2:ff:e0:09:f7:d1:30:47:e6:a1:c1:92:c2:ba: - cf:de:2c:86:2e:34:3a:63:f4:5a:a9:45:25:89:47: - eb:44:e8:fb:4e:0d:ac:99:5d:f8:42:74:07:a5:95: - ba:81:8b:9f:f1:64:f0:37:4e:e7:f6:59:c3:3a:08: - 5b:82:55:cd:ea:81:94:b6:e7:ec:7a:d8:6b:09:41: - 40:40:2f:27:b4:0e:09:d8:61:82:dc:b3:43:5b:50: - 5d:03:78:08:ed:35:52:e6:3a:9b:ad:de:4f:73:b3: - 56:2f:25:0d:e7:0b:61:83:e8:35:fe:73:89:e3:0e: - cb:7d:c1:29:00:d4:e2:a8:c2:9c:5c:b0:fd:4d:be: - 1a:86:a7:7f:b3:d3:d9:3b:35:4c:82:b3:8d:55:0a: - 9b:77:40:b0:dd:b1:e6:91:f4:91:dc:50:ee:96:92: - cc:af:11:bb:43:0a:a4:2b:5a:00:ab:9c:17:a2:e1: - bf:7c:9e:92:04:01:8f:bf:16:5f:85:9c:e4:5c:37: - 97:ff:29:50:18:d7:01:66:c5:bc:51:11:ac:8a:a3: - de:58:5d:1c:44:e4:f4:fe:77:83:99:3f:57:71:2d: - 2e:95:07:cc:78:b8:3c:50:4b:ea:ca:a8:20:e8:9b: - 05:91:5c:40:ba:a9:c3:87:d5:95:d7:ab:67:33:03: - 1d:b9:c6:d7:ef:3a:4e:aa:7d:81:bc:1b:52:fb:0e: - b4:7c:ce:9e:ff:d8:08:2b:33:ee:b6:d3:5c:7c:1a: - 7e:a3:cf:cd:92:42:7f:1b:a4:36:0b:d9:51:39:c6: - cb:c1:65:c1:e2:84:53:30:ba:2e:f1:c3:07:71:09: - 69:dd:ff:92:16:2c:05:1d:60:28:1c:af:5e:76:88: - 9f:df:e7:97:fb:cd:19:48:7a:87:f6:24:e0:e1:e1: - ff:76:95:93:65:72:44:29:5e:69:5d:2d:26:2c:fb: - a7:06:63:ff:7f:02:29:82:61:42:d9:9a:0b:44:ea: - 89:c8:bc:4a:75:17:58:05:85:04:62:1f:70:bd:79: - 66:b6:bb:27:a6:88:c8:27:db:41:da:88:ec:4e:71: - 0a:20:e6:e3:79:2a:ee:b5:af:99:96:72:9d:ca:c3: - e7:4f:9d:cd:e4:6b:22:e4:3b:54:2f:e2:e8:0c:df: - 6f:14:f8:74:4c:21:15:28:2c:51:5e:c8:8c:86:8e: - e0:5e:0e:2d:e1:25:cc:47:8c:9e:b5:94:bb:34:e4: - 43:b1:cd:55:2a:6f:1f:14:fa:c2:2f:3c:a1:ba:65: - f5:09:8d:1c:20:12:0d:80:33:35:f7:2f:d1:8b:ca: - b8:77:f0:a3:7d:fa:bd:31:ba:3a:f4:c7:5e:8d:55: - a7:c9:69 + 00:be:52:41:09:c0:22:8a:8d:83:f3:c3:d3:a0:f8: + 87:31:7f:de:b5:25:b5:6c:f7:22:ea:d4:c2:6a:5b: + ed:c1:9d:ee:e5:3d:15:d5:85:83:18:5f:ce:71:a8: + c0:02:5f:81:65:73:7e:cc:ad:71:a8:63:c0:13:ef: + 0d:e3:d2:4b:7b:8f:f4:4c:64:5d:33:10:de:5e:94: + 01:2d:a0:94:46:c5:be:a1:85:b6:7a:bf:b9:f6:b6: + 62:53:72:2b:de:e0:ac:70:6d:c2:02:73:e3:e4:fa: + 57:92:71:b9:39:4f:ce:85:1e:fa:d7:89:49:18:89: + 01:ef:a3:50:d9:d8:64:25:df:42:74:06:d1:2c:b8: + 27:98:92:b1:af:77:b5:46:d5:3a:d8:1b:0b:33:7b: + 24:f8:7d:fc:85:80:2c:d5:2f:c4:35:f8:5e:5e:8a: + 45:6d:01:6d:ef:0e:a5:4e:6b:3b:e7:14:93:30:9c: + d3:66:4a:0c:af:17:7c:a1:75:65:ff:43:28:df:af: + e8:73:59:d7:84:79:ec:33:6f:38:af:9a:d5:f3:89: + 97:bb:eb:a0:61:d5:d6:0b:c7:d5:cd:f3:9e:f9:52: + 04:dd:58:c1:8d:ab:fa:6a:9c:1a:14:78:92:73:32: + d3:2b:82:f1:bf:a0:7f:9e:c6:f6:02:a6:0b:29:ff: + c0:0a:28:47:b2:cf:03:0f:4d:09:9a:de:57:e6:bf: + b3:3d:c4:e9:c2:40:75:51:c4:9e:d2:6a:6d:a3:64: + ee:a5:ac:22:73:2b:27:81:a1:77:f5:48:96:03:5e: + 5b:68:7f:27:f8:eb:ea:ff:67:56:c1:c3:10:2f:41: + 04:e6:ef:f2:de:cf:9e:57:04:02:c1:dc:af:02:b7: + 07:60:6e:29:98:7d:24:8b:80:e8:c1:3f:93:90:01: + 7e:21:00:74:78:e5:68:b5:57:37:5c:6d:2e:40:52: + fd:17:26:a5:d2:26:c4:2f:3e:80:35:39:0a:70:4a: + f4:dd:ed:8a:1f:41:a6:6c:b7:90:e0:ad:1e:f2:b6: + ec:92:12:ba:c3:78:75:ea:3e:5a:d0:92:52:11:9b: + d6:f8:f0:66:d1:d7:db:07:32:7b:bb:b1:f7:15:7c: + c1:0e:6a:89:bb:22:31:c7:73:6e:0e:4d:ff:fe:3c: + 33:91:d9:6d:7b:17:0b:88:e9:d9:5e:1e:dc:18:f8: + 72:81:b7:fb:2a:40:4b:da:47:32:e4:5f:d0:f9:d6: + 9e:2a:15:32:5e:07:b8:41:59:bb:37:e9:d7:75:fd: + c9:ba:c8:ee:b6:91:d0:ad:36:b3:8e:20:18:21:f7: + 88:0a:67:93:76:a3:ec:03:61:36:19:32:42:e2:f1: + c0:fe:91 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Key Usage: critical @@ -57,7 +57,7 @@ Certificate: X509v3 Basic Constraints: critical CA:FALSE X509v3 Subject Key Identifier: - B8:6F:A5:A0:D0:70:D7:5E:AE:1E:54:5A:F1:26:DA:DD:CB:2C:5E:6F + 16:A9:BA:C6:C3:EE:83:45:AF:53:62:D8:76:5A:BB:82:5C:93:C2:4A X509v3 Authority Key Identifier: BB:BC:C3:47:A5:E4:BC:A9:C6:C3:A4:72:0C:10:8D:A2:35:E1:C8:E8 Authority Information Access: @@ -70,78 +70,78 @@ Certificate: CT Precertificate SCTs: Signed Certificate Timestamp: Version : v1 (0x0) - Log ID : 76:FF:88:3F:0A:B6:FB:95:51:C2:61:CC:F5:87:BA:34: - B4:A4:CD:BB:29:DC:68:42:0A:9F:E6:67:4C:5A:3A:74 - Timestamp : Aug 20 14:03:34.519 2024 GMT + Log ID : E6:D2:31:63:40:77:8C:C1:10:41:06:D7:71:B9:CE:C1: + D2:40:F6:96:84:86:FB:BA:87:32:1D:FD:1E:37:8E:50 + Timestamp : Nov 4 13:06:02.672 2024 GMT Extensions: none Signature : ecdsa-with-SHA256 - 30:46:02:21:00:C6:92:47:D8:40:3A:B9:0C:BD:FC:45: - 3E:C6:28:06:80:62:0F:4A:EF:44:56:5F:8C:16:DF:C6: - A4:0E:4C:76:DD:02:21:00:CA:36:0F:54:6B:84:37:F3: - AD:50:5B:E8:DD:3D:D2:EA:34:82:72:50:D2:2C:2A:47: - FB:EC:16:76:5A:E8:9E:0C + 30:44:02:20:57:F3:C2:E9:C5:52:6F:34:F8:AB:7E:55: + F4:D2:70:17:D5:5D:94:FF:DA:7F:AD:C1:D1:6B:04:BD: + 3C:E3:B0:8C:02:20:22:67:7B:C7:D5:C3:F3:26:5F:C7: + CB:19:30:8D:52:7E:ED:6B:21:19:EC:5F:17:A9:1D:45: + 32:36:4A:F4:33:E5 Signed Certificate Timestamp: Version : v1 (0x0) - Log ID : DF:E1:56:EB:AA:05:AF:B5:9C:0F:86:71:8D:A8:C0:32: - 4E:AE:56:D9:6E:A7:F5:A5:6A:01:D1:C1:3B:BE:52:5C - Timestamp : Aug 20 14:03:34.658 2024 GMT + Log ID : 13:4A:DF:1A:B5:98:42:09:78:0C:6F:EF:4C:7A:91:A4: + 16:B7:23:49:CE:58:57:6A:DF:AE:DA:A7:C2:AB:E0:22 + Timestamp : Nov 4 13:06:02.699 2024 GMT Extensions: none Signature : ecdsa-with-SHA256 - 30:45:02:20:23:5B:E1:35:10:78:F9:99:9F:42:7F:FE: - 73:F4:74:1B:55:3D:B9:93:C9:A8:EE:E7:B2:61:52:12: - D1:C9:06:0C:02:21:00:87:EA:87:A3:3A:B2:C6:F0:EA: - 52:5A:B2:7F:02:2E:CF:68:C8:A5:CB:54:0F:CB:CE:6A: - CC:E0:3A:D1:09:D3:9C + 30:44:02:20:1A:B2:DE:AE:0B:1F:E8:D9:37:07:F2:02: + 8E:A7:AB:E1:BD:A4:0F:33:E0:95:3D:25:6E:10:38:6E: + F9:48:FA:80:02:20:49:55:9A:DA:DE:EE:94:FB:EB:BE: + FA:DA:95:D2:39:F0:BE:40:5B:BF:F9:5F:35:57:3B:09: + BC:6E:58:4E:C9:58 Signature Algorithm: sha256WithRSAEncryption Signature Value: - 1d:11:b5:c6:7e:71:6c:63:34:8e:d4:eb:e6:42:ca:ce:fd:0d: - 9a:7d:ea:49:43:8d:de:46:ad:27:09:a6:a7:5c:58:89:2f:47: - 03:68:e2:19:0f:f6:76:be:47:0a:b6:d1:ed:5d:71:13:2a:12: - de:5e:41:cf:e1:a3:2c:46:07:81:da:b1:86:66:61:b0:0b:70: - 19:33:4c:a3:29:e4:e6:79:f6:3f:1b:4a:51:6a:2a:0c:c1:07: - 2c:db:cc:9a:3f:17:a2:ff:ac:19:76:9e:a2:d4:9b:c9:c2:75: - 48:5c:fd:d4:5e:ff:cc:6b:f0:ea:73:da:0b:f8:fd:c5:92:42: - ca:ca:43:51:98:e5:4b:77:b6:0f:da:d2:83:33:77:bc:60:5c: - b7:60:12:42:10:78:5b:ed:cd:83:42:63:ba:96:de:0e:d8:9e: - a5:97:6a:6f:70:82:7c:82:2c:ca:e3:a3:34:61:7a:70:d1:03: - fc:89:06:1e:b4:f3:ed:b4:64:5f:54:b8:d5:6e:31:e0:fa:0b: - f6:be:b7:6c:38:78:f8:bb:22:f2:7c:6b:44:54:3e:91:3a:8c: - bd:4d:1b:b5:8a:a6:df:17:9b:cf:3a:bd:dc:c3:1e:c5:2c:f5: - 19:32:75:0f:7b:54:30:ab:bb:7e:db:43:fb:ed:16:d9:03:81: - 23:8a:8c:7a + 87:ad:98:99:d1:a1:83:2b:b1:f2:4f:54:08:c3:00:69:b3:4a: + fa:9c:5a:10:95:59:dc:2d:64:43:02:95:3c:00:2b:66:34:1b: + 1d:fd:c7:9c:f8:a0:0d:63:4e:29:0b:b0:37:72:3f:99:9b:84: + 23:f6:2a:73:80:bb:73:7d:c6:c9:4e:37:83:18:47:68:e3:01: + 99:12:d7:5e:88:af:7b:8c:45:72:0a:02:56:50:2d:b0:d0:4e: + a4:61:01:c5:4f:2d:0a:cf:c7:07:b7:db:65:f0:50:b2:b4:aa: + 7f:e4:f8:09:eb:8a:3e:4e:05:b9:de:17:83:4c:0d:b9:f0:07: + dc:d4:45:70:97:db:78:c8:b5:0e:ba:a5:29:fe:b4:fa:b2:2b: + fe:0a:31:b5:1b:fb:f4:b4:fb:27:a7:c8:db:97:f3:f2:d2:66: + db:6b:03:71:49:53:10:9e:0a:f9:b3:f6:5b:83:bf:d0:72:a2: + be:b4:84:94:be:a0:fa:1e:9e:bb:09:76:14:d5:8f:ab:75:4a: + 7a:73:20:6f:45:60:84:43:4e:9d:3a:a8:4a:8f:90:75:47:5d: + 78:8a:e2:4d:94:7e:67:2e:6c:2e:47:cf:b4:86:15:2c:ab:2c: + a3:b6:22:cf:e9:0a:12:9c:ff:c6:26:2e:8b:3b:74:1f:09:cf: + be:f2:88:f1 -----BEGIN CERTIFICATE----- -MIIGIzCCBQugAwIBAgISA5JCReZ6ohOE6nx+ztpu1GNnMA0GCSqGSIb3DQEBCwUA +MIIGIDCCBQigAwIBAgISAxCUGlTMsyc8rV/p52Bp7blOMA0GCSqGSIb3DQEBCwUA MDMxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQD -EwNSMTAwHhcNMjQwODIwMTMwNTA0WhcNMjQxMTE4MTMwNTAzWjAkMSIwIAYDVQQD +EwNSMTAwHhcNMjQxMTA0MTIwNzMyWhcNMjUwMjAyMTIwNzMxWjAkMSIwIAYDVQQD DBkqLmc2ZnlpYXEubW9uZ29kYi1kZXYubmV0MIICIjANBgkqhkiG9w0BAQEFAAOC -Ag8AMIICCgKCAgEAwv/gCffRMEfmocGSwrrP3iyGLjQ6Y/RaqUUliUfrROj7Tg2s -mV34QnQHpZW6gYuf8WTwN07n9lnDOghbglXN6oGUtufsethrCUFAQC8ntA4J2GGC -3LNDW1BdA3gI7TVS5jqbrd5Pc7NWLyUN5wthg+g1/nOJ4w7LfcEpANTiqMKcXLD9 -Tb4ahqd/s9PZOzVMgrONVQqbd0Cw3bHmkfSR3FDulpLMrxG7QwqkK1oAq5wXouG/ -fJ6SBAGPvxZfhZzkXDeX/ylQGNcBZsW8URGsiqPeWF0cROT0/neDmT9XcS0ulQfM -eLg8UEvqyqgg6JsFkVxAuqnDh9WV16tnMwMducbX7zpOqn2BvBtS+w60fM6e/9gI -KzPuttNcfBp+o8/NkkJ/G6Q2C9lROcbLwWXB4oRTMLou8cMHcQlp3f+SFiwFHWAo -HK9edoif3+eX+80ZSHqH9iTg4eH/dpWTZXJEKV5pXS0mLPunBmP/fwIpgmFC2ZoL -ROqJyLxKdRdYBYUEYh9wvXlmtrsnpojIJ9tB2ojsTnEKIObjeSruta+ZlnKdysPn -T53N5Gsi5DtUL+LoDN9vFPh0TCEVKCxRXsiMho7gXg4t4SXMR4yetZS7NORDsc1V -Km8fFPrCLzyhumX1CY0cIBINgDM19y/Ri8q4d/Cjffq9Mbo69MdejVWnyWkCAwEA -AaOCAj4wggI6MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI -KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUuG+loNBw116uHlRa8Sba -3cssXm8wHwYDVR0jBBgwFoAUu7zDR6XkvKnGw6RyDBCNojXhyOgwVwYIKwYBBQUH +Ag8AMIICCgKCAgEAvlJBCcAiio2D88PToPiHMX/etSW1bPci6tTCalvtwZ3u5T0V +1YWDGF/OcajAAl+BZXN+zK1xqGPAE+8N49JLe4/0TGRdMxDeXpQBLaCURsW+oYW2 +er+59rZiU3Ir3uCscG3CAnPj5PpXknG5OU/OhR7614lJGIkB76NQ2dhkJd9CdAbR +LLgnmJKxr3e1RtU62BsLM3sk+H38hYAs1S/ENfheXopFbQFt7w6lTms75xSTMJzT +ZkoMrxd8oXVl/0Mo36/oc1nXhHnsM284r5rV84mXu+ugYdXWC8fVzfOe+VIE3VjB +jav6apwaFHiSczLTK4Lxv6B/nsb2AqYLKf/ACihHss8DD00Jmt5X5r+zPcTpwkB1 +UcSe0mpto2TupawicysngaF39UiWA15baH8n+Ovq/2dWwcMQL0EE5u/y3s+eVwQC +wdyvArcHYG4pmH0ki4DowT+TkAF+IQB0eOVotVc3XG0uQFL9Fyal0ibELz6ANTkK +cEr03e2KH0GmbLeQ4K0e8rbskhK6w3h16j5a0JJSEZvW+PBm0dfbBzJ7u7H3FXzB +DmqJuyIxx3NuDk3//jwzkdltexcLiOnZXh7cGPhygbf7KkBL2kcy5F/Q+daeKhUy +Xge4QVm7N+nXdf3JusjutpHQrTazjiAYIfeICmeTdqPsA2E2GTJC4vHA/pECAwEA +AaOCAjswggI3MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI +KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUFqm6xsPug0WvU2LYdlq7 +glyTwkowHwYDVR0jBBgwFoAUu7zDR6XkvKnGw6RyDBCNojXhyOgwVwYIKwYBBQUH AQEESzBJMCIGCCsGAQUFBzABhhZodHRwOi8vcjEwLm8ubGVuY3Iub3JnMCMGCCsG AQUFBzAChhdodHRwOi8vcjEwLmkubGVuY3Iub3JnLzBEBgNVHREEPTA7gh4qLmc2 ZnlpYXEubWVzaC5tb25nb2RiLWRldi5uZXSCGSouZzZmeWlhcS5tb25nb2RiLWRl -di5uZXQwEwYDVR0gBAwwCjAIBgZngQwBAgEwggEFBgorBgEEAdZ5AgQCBIH2BIHz -APEAdwB2/4g/Crb7lVHCYcz1h7o0tKTNuyncaEIKn+ZnTFo6dAAAAZFwGaT3AAAE -AwBIMEYCIQDGkkfYQDq5DL38RT7GKAaAYg9K70RWX4wW38akDkx23QIhAMo2D1Rr -hDfzrVBb6N090uo0gnJQ0iwqR/vsFnZa6J4MAHYA3+FW66oFr7WcD4ZxjajAMk6u -Vtlup/WlagHRwTu+UlwAAAGRcBmlggAABAMARzBFAiAjW+E1EHj5mZ9Cf/5z9HQb -VT25k8mo7ueyYVIS0ckGDAIhAIfqh6M6ssbw6lJasn8CLs9oyKXLVA/LzmrM4DrR -CdOcMA0GCSqGSIb3DQEBCwUAA4IBAQAdEbXGfnFsYzSO1OvmQsrO/Q2afepJQ43e -Rq0nCaanXFiJL0cDaOIZD/Z2vkcKttHtXXETKhLeXkHP4aMsRgeB2rGGZmGwC3AZ -M0yjKeTmefY/G0pRaioMwQcs28yaPxei/6wZdp6i1JvJwnVIXP3UXv/Ma/Dqc9oL -+P3FkkLKykNRmOVLd7YP2tKDM3e8YFy3YBJCEHhb7c2DQmO6lt4O2J6ll2pvcIJ8 -gizK46M0YXpw0QP8iQYetPPttGRfVLjVbjHg+gv2vrdsOHj4uyLyfGtEVD6ROoy9 -TRu1iqbfF5vPOr3cwx7FLPUZMnUPe1Qwq7t+20P77RbZA4Ejiox6 +di5uZXQwEwYDVR0gBAwwCjAIBgZngQwBAgEwggECBgorBgEEAdZ5AgQCBIHzBIHw +AO4AdQDm0jFjQHeMwRBBBtdxuc7B0kD2loSG+7qHMh39HjeOUAAAAZL3SEkwAAAE +AwBGMEQCIFfzwunFUm80+Kt+VfTScBfVXZT/2n+twdFrBL0847CMAiAiZ3vH1cPz +Jl/HyxkwjVJ+7WshGexfF6kdRTI2SvQz5QB1ABNK3xq1mEIJeAxv70x6kaQWtyNJ +zlhXat+u2qfCq+AiAAABkvdISUsAAAQDAEYwRAIgGrLergsf6Nk3B/ICjqer4b2k +DzPglT0lbhA4bvlI+oACIElVmtre7pT767762pXSOfC+QFu/+V81VzsJvG5YTslY +MA0GCSqGSIb3DQEBCwUAA4IBAQCHrZiZ0aGDK7HyT1QIwwBps0r6nFoQlVncLWRD +ApU8ACtmNBsd/cec+KANY04pC7A3cj+Zm4Qj9ipzgLtzfcbJTjeDGEdo4wGZEtde +iK97jEVyCgJWUC2w0E6kYQHFTy0Kz8cHt9tl8FCytKp/5PgJ64o+TgW53heDTA25 +8Afc1EVwl9t4yLUOuqUp/rT6siv+CjG1G/v0tPsnp8jbl/Py0mbbawNxSVMQngr5 +s/Zbg7/QcqK+tISUvqD6Hp67CXYU1Y+rdUp6cyBvRWCEQ06dOqhKj5B1R114iuJN +lH5nLmwuR8+0hhUsqyyjtiLP6QoSnP/GJi6LO3QfCc++8ojx -----END CERTIFICATE----- \ No newline at end of file From 6e624486f46ad35afc43ef1118f006b8c35705dc Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:56:50 +0100 Subject: [PATCH 07/14] RUBY-3554 Update required_ruby_version (#2909) --- gemfiles/standard.rb | 12 ++++-------- mongo.gemspec | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/gemfiles/standard.rb b/gemfiles/standard.rb index 6a11bc3404..0b01687a84 100644 --- a/gemfiles/standard.rb +++ b/gemfiles/standard.rb @@ -28,14 +28,10 @@ def standard_dependencies gem 'yajl-ruby', platforms: :mri, require: false gem 'celluloid', platforms: :mri, require: false - # for static analysis -- ignore ruby < 2.6 because of rubocop - # version incompatibilities - if RUBY_VERSION > '2.5.99' - gem 'rubocop', '~> 1.45.1' - gem 'rubocop-performance', '~> 1.16.0' - gem 'rubocop-rake', '~> 0.6.0' - gem 'rubocop-rspec', '~> 2.18.1' - end + gem 'rubocop', '~> 1.45.1' + gem 'rubocop-performance', '~> 1.16.0' + gem 'rubocop-rake', '~> 0.6.0' + gem 'rubocop-rspec', '~> 2.18.1' platform :mri do # Debugger for VSCode. diff --git a/mongo.gemspec b/mongo.gemspec index a4bec6e854..c7921d3946 100644 --- a/mongo.gemspec +++ b/mongo.gemspec @@ -36,7 +36,7 @@ Gem::Specification.new do |s| s.require_paths = ['lib'] s.bindir = 'bin' - s.required_ruby_version = ">= 2.5" + s.required_ruby_version = ">= 2.7" s.add_dependency 'base64' s.add_dependency 'bson', '>=4.14.1', '<6.0.0' From 0e91a408362720a3ec603ed3e6b2e13b5abce800 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:45:44 +0100 Subject: [PATCH 08/14] Use https for submodules (#2913) --- .gitmodules | 2 +- profile/driver_bench/rake/tasks.rake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6d428f359d..366d171c3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/mongodb-labs/drivers-evergreen-tools [submodule "spec/shared"] path = spec/shared - url = git@github.com:mongodb-labs/mongo-ruby-spec-shared.git + url = https://github.com/mongodb-labs/mongo-ruby-spec-shared diff --git a/profile/driver_bench/rake/tasks.rake b/profile/driver_bench/rake/tasks.rake index dbf480c6b5..07e8dea489 100644 --- a/profile/driver_bench/rake/tasks.rake +++ b/profile/driver_bench/rake/tasks.rake @@ -4,7 +4,7 @@ $LOAD_PATH.unshift File.expand_path('../../../lib', __dir__) task driver_bench: %i[ driver_bench:data driver_bench:run ] -SPECS_REPO_URI = 'git@github.com:mongodb/specifications.git' +SPECS_REPO_URI = 'https://github.com/mongodb/specifications' SPECS_PATH = File.expand_path('../../../specifications', __dir__) DRIVER_BENCH_DATA = File.expand_path('../../data/driver_bench', __dir__) From b0ccd0e28a350834b5ab8a86e3dff64e497d0aa5 Mon Sep 17 00:00:00 2001 From: Joseph Teichman Date: Tue, 21 Jan 2025 14:43:41 -0500 Subject: [PATCH 09/14] RUBY-3604 Fix multithread auth race condition (#2912) Co-authored-by: Dmitry Rybakov --- lib/mongo/auth/credential_cache.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/mongo/auth/credential_cache.rb b/lib/mongo/auth/credential_cache.rb index 16d7962775..c9888ce3dd 100644 --- a/lib/mongo/auth/credential_cache.rb +++ b/lib/mongo/auth/credential_cache.rb @@ -22,19 +22,24 @@ module Auth # # @api private module CredentialCache - class << self attr_reader :store end + MUTEX = Mutex.new + module_function def get(key) - @store ||= {} - @store[key] + MUTEX.synchronize do + @store ||= {} + @store[key] + end end module_function def set(key, value) - @store ||= {} - @store[key] = value + MUTEX.synchronize do + @store ||= {} + @store[key] = value + end end module_function def cache(key) @@ -47,7 +52,9 @@ class << self end module_function def clear - @store = {} + MUTEX.synchronize do + @store = {} + end end end end From 36cfcf24e3c58144b15824c7dfeeb36543250090 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 30 Jan 2025 17:35:44 +0100 Subject: [PATCH 10/14] Tracer --- lib/mongo.rb | 1 + lib/mongo/open_telemetry.rb | 28 +++++++ .../open_telemetry/command_span_builder.rb | 65 +++++++++++++++ .../open_telemetry/operation_span_builder.rb | 34 ++++++++ lib/mongo/open_telemetry/tracer.rb | 79 +++++++++++++++++++ 5 files changed, 207 insertions(+) create mode 100644 lib/mongo/open_telemetry.rb create mode 100644 lib/mongo/open_telemetry/command_span_builder.rb create mode 100644 lib/mongo/open_telemetry/operation_span_builder.rb create mode 100644 lib/mongo/open_telemetry/tracer.rb diff --git a/lib/mongo.rb b/lib/mongo.rb index c866ad1a9e..d4664f8c8a 100644 --- a/lib/mongo.rb +++ b/lib/mongo.rb @@ -79,6 +79,7 @@ require 'mongo/write_concern' require 'mongo/utils' require 'mongo/config' +require 'mongo/open_telemetry' module Mongo diff --git a/lib/mongo/open_telemetry.rb b/lib/mongo/open_telemetry.rb new file mode 100644 index 0000000000..3d75e40391 --- /dev/null +++ b/lib/mongo/open_telemetry.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright (C) 2025-present MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Mongo + # @api private + module OpenTelemetry + def tracer + Tracer.instance + end + module_function :tracer + end +end + +require 'mongo/open_telemetry/operation_span_builder' +require 'mongo/open_telemetry/tracer' diff --git a/lib/mongo/open_telemetry/command_span_builder.rb b/lib/mongo/open_telemetry/command_span_builder.rb new file mode 100644 index 0000000000..fa94b9e6e6 --- /dev/null +++ b/lib/mongo/open_telemetry/command_span_builder.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# Copyright (C) 2025-present MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Mongo + module OpenTelemetry + class CommandSpanBuilder + + def build(command, address) + [ span_name(command), build_attributes(command, address) ] + end + + private + + def span_name(command) + collection = collection_name(command) + command_name = command.keys.first + if collection + "#{collection}.#{command_name}" + else + command_name + end + end + + # @return [ Hash ] The attributes of the span. + def build_attributes(command, address) + command_name = command.keys.first + { + 'db.system' => 'mongodb', + 'db.namespace' => command['$db'], + 'db.command.name' => command_name, + 'server.port' => address.port, + 'net.peer.port' => address.port, + 'server.address' => address.host, + 'net.peer.address' => address.host, + 'db.query.summary' => span_name(command) + }.tap do |attributes| + if (coll_name = collection_name(command)) + attributes['db.collection.name'] = coll_name + end + if command_name == 'getMore' + attributes['db.mongodb.cursor_id'] = command[command_name].value + end + end + end + + # @return [ String | nil] Name of collection the operation is executed on. + def collection_name(command) + command.values.first if command.values.first.is_a?(String) + end + end + end +end diff --git a/lib/mongo/open_telemetry/operation_span_builder.rb b/lib/mongo/open_telemetry/operation_span_builder.rb new file mode 100644 index 0000000000..ddfbc7a714 --- /dev/null +++ b/lib/mongo/open_telemetry/operation_span_builder.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Copyright (C) 2025-present MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Mongo + module OpenTelemetry + class OperationSpanBuilder + + def build(name, operation) + attrs = { + 'db.system' => 'mongodb', + 'db.namespace' => operation.spec[:db_name], + 'db.collection.name' => operation.spec[:coll_name], + 'db.operation.name' => name, + 'db.operation.summary' => "#{name} #{operation.spec[:coll_name]}" + } + + [name, attrs] + end + end + end +end diff --git a/lib/mongo/open_telemetry/tracer.rb b/lib/mongo/open_telemetry/tracer.rb new file mode 100644 index 0000000000..49a4c89682 --- /dev/null +++ b/lib/mongo/open_telemetry/tracer.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# Copyright (C) 2025-present MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'singleton' + +module Mongo + module OpenTelemetry + class Tracer + include Singleton + + # Environment variable that enables otel instrumentation. + ENV_VARIABLE_DISABLED = 'OTEL_RUBY_INSTRUMENTATION_MONGODB_DISABLED' + + # Environment variable that controls the db.statement attribute. + ENV_VARIABLE_QUERY_TEXT = 'OTEL_RUBY_INSTRUMENTATION_MONGODB_QUERY_TEXT' + + # Name of the tracer. + OTEL_TRACER_NAME = 'mongo-ruby-driver' + + # @return [ OpenTelemetry::SDK::Trace::Tracer | nil ] The otel tracer. + attr_reader :ot_tracer + + def initialize + return unless defined?(::OpenTelemetry) + return if %w[ 1 yes true ].include?(ENV[ENV_VARIABLE_DISABLED]) + + @ot_tracer = ::OpenTelemetry.tracer_provider.tracer( + OTEL_TRACER_NAME, + Mongo::VERSION + ) + end + + # @param [ String ] name Name of the span. + # @param [ Hash ] attributes Span attributes. + def in_span(name, attributes: {}, &block) + if enabled? + @ot_tracer.in_span(name, attributes: attributes, kind: :client, &block) + else + yield + end + end + + # @param [ String ] name Name of the span. + # @param [ Hash ] attributes Span attributes. + # @param [ OpenTelemetry::API ] + def start_span(name, attributes: {}, with_parent: nil, &block) + return unless enabled? + + @ot_tracer.start_span(name, with_parent: with_parent, attributes: attributes, &block) + end + + private + + # @return [ Boolean ] whether OpenTelemetry tracing is enabled. + def enabled? + @ot_tracer != nil + end + + # @return [ Boolean ] whether query_text should be added. + def query_text? + %w[ 1 yes true ].include?(ENV[ENV_VARIABLE_QUERY_TEXT]) + end + + end + end +end From 6d7916fb90fcd7536f3cd3146600f3d0e0f5d86e Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Thu, 30 Jan 2025 17:45:20 +0100 Subject: [PATCH 11/14] Current span --- lib/mongo/open_telemetry.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/mongo/open_telemetry.rb b/lib/mongo/open_telemetry.rb index 3d75e40391..6b806386b8 100644 --- a/lib/mongo/open_telemetry.rb +++ b/lib/mongo/open_telemetry.rb @@ -21,6 +21,28 @@ def tracer Tracer.instance end module_function :tracer + + def set_current(span, context) + Thread.current['mongo-ruby-driver-otel-span'] = span + Thread.current['mongo-ruby-driver-otel-context'] = context + end + module_function :set_current + + def current_span + Thread.current['mongo-ruby-driver-otel-span'] + end + module_function :current_span + + def current_context + Thread.current['mongo-ruby-driver-otel-context'] + end + module_function :current_context + + def clear_current + Thread.current['mongo-ruby-driver-otel-context'] = nil + Thread.current['mongo-ruby-driver-otel-span'] = nil + end + module_function :clear_current end end From e639279f4fc64cb07c49917e479d3295edbf078f Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Fri, 31 Jan 2025 15:08:43 +0100 Subject: [PATCH 12/14] Tracing --- lib/mongo/collection/view/iterable.rb | 16 ++++++++++------ lib/mongo/cursor.rb | 14 +++++++++----- lib/mongo/open_telemetry.rb | 1 + lib/mongo/open_telemetry/command_span_builder.rb | 16 ++++++++++++++++ lib/mongo/open_telemetry/tracer.rb | 7 ++++++- lib/mongo/operation/shared/executable.rb | 9 ++++++++- 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/lib/mongo/collection/view/iterable.rb b/lib/mongo/collection/view/iterable.rb index 99133c5e9f..0321a0f4ef 100644 --- a/lib/mongo/collection/view/iterable.rb +++ b/lib/mongo/collection/view/iterable.rb @@ -169,12 +169,16 @@ def initial_query_op(session) def send_initial_query(server, context) operation = initial_query_op(context.session) - if server.load_balancer? - # Connection will be checked in when cursor is drained. - connection = server.pool.check_out(context: context) - operation.execute_with_connection(connection, context: context) - else - operation.execute(server, context: context) + builder = OpenTelemetry::OperationSpanBuilder.new + span_name, span_attrs = builder.build('find', operation) + OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |_span, _context| + if server.load_balancer? + # Connection will be checked in when cursor is drained. + connection = server.pool.check_out(context: context) + operation.execute_with_connection(connection, context: context) + else + operation.execute(server, context: context) + end end end diff --git a/lib/mongo/cursor.rb b/lib/mongo/cursor.rb index 0e0927f02c..1dcb2dfc9e 100644 --- a/lib/mongo/cursor.rb +++ b/lib/mongo/cursor.rb @@ -513,11 +513,15 @@ def unregister end def execute_operation(op, context: nil) - op_context = context || possibly_refreshed_context - if @connection.nil? - op.execute(@server, context: op_context) - else - op.execute_with_connection(@connection, context: op_context) + builder = OpenTelemetry::OperationSpanBuilder.new + span_name, span_attrs = builder.build('getMore', op) + OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |_span, _context| + op_context = context || possibly_refreshed_context + if @connection.nil? + op.execute(@server, context: op_context) + else + op.execute_with_connection(@connection, context: op_context) + end end end diff --git a/lib/mongo/open_telemetry.rb b/lib/mongo/open_telemetry.rb index 6b806386b8..cea489cb89 100644 --- a/lib/mongo/open_telemetry.rb +++ b/lib/mongo/open_telemetry.rb @@ -46,5 +46,6 @@ def clear_current end end +require 'mongo/open_telemetry/command_span_builder' require 'mongo/open_telemetry/operation_span_builder' require 'mongo/open_telemetry/tracer' diff --git a/lib/mongo/open_telemetry/command_span_builder.rb b/lib/mongo/open_telemetry/command_span_builder.rb index fa94b9e6e6..f7514a9a41 100644 --- a/lib/mongo/open_telemetry/command_span_builder.rb +++ b/lib/mongo/open_telemetry/command_span_builder.rb @@ -22,6 +22,22 @@ def build(command, address) [ span_name(command), build_attributes(command, address) ] end + # @param [ OpenTelemetry::Trace::Span | nil ] span + # @param [ Mongo::Operation::Result ] result + def add_attributes_from_result(span, result) + return if span.nil? + + if result.successful? + if (cursor_id = result.cursor_id).positive? + span.add_attributes( + 'db.mongodb.cursor_id' => cursor_id + ) + end + else + span.record_exception(result.error) + end + end + private def span_name(command) diff --git a/lib/mongo/open_telemetry/tracer.rb b/lib/mongo/open_telemetry/tracer.rb index 49a4c89682..f5b6a8bd24 100644 --- a/lib/mongo/open_telemetry/tracer.rb +++ b/lib/mongo/open_telemetry/tracer.rb @@ -47,7 +47,12 @@ def initialize # @param [ Hash ] attributes Span attributes. def in_span(name, attributes: {}, &block) if enabled? - @ot_tracer.in_span(name, attributes: attributes, kind: :client, &block) + @ot_tracer.in_span(name, attributes: attributes, kind: :client) do |span, context| + OpenTelemetry.set_current(span, context) + yield(span, context) if block_given? + ensure + OpenTelemetry.clear_current + end else yield end diff --git a/lib/mongo/operation/shared/executable.rb b/lib/mongo/operation/shared/executable.rb index 041e4d1e5b..80513ed6a5 100644 --- a/lib/mongo/operation/shared/executable.rb +++ b/lib/mongo/operation/shared/executable.rb @@ -104,7 +104,14 @@ def result_class end def get_result(connection, context, options = {}) - result_class.new(*dispatch_message(connection, context, options), context: context, connection: connection) + builder = OpenTelemetry::CommandSpanBuilder.new + span_name, span_attrs = builder.build(command(connection), connection.address) + span = OpenTelemetry.tracer.start_span(span_name, attributes: span_attrs, with_parent: OpenTelemetry.current_context) + result_class.new(*dispatch_message(connection, context, options), context: context, connection: connection).tap do |result| + builder.add_attributes_from_result(span, result) + end + ensure + span&.finish end # Returns a Protocol::Message or nil as reply. From a68ff5c104adf8c2ab8ac587b4027f911316ec68 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Tue, 4 Feb 2025 16:25:31 +0100 Subject: [PATCH 13/14] Tracing --- lib/mongo/collection/view/iterable.rb | 10 +++--- lib/mongo/cursor.rb | 2 +- lib/mongo/open_telemetry.rb | 1 + .../open_telemetry/command_span_builder.rb | 28 +++++++++------- .../open_telemetry/operation_span_builder.rb | 33 ++++++++++++++----- lib/mongo/open_telemetry/shared.rb | 23 +++++++++++++ lib/mongo/open_telemetry/tracer.rb | 2 -- lib/mongo/operation/find/result.rb | 8 +++++ lib/mongo/operation/get_more/result.rb | 7 ++++ lib/mongo/operation/indexes/result.rb | 8 ++++- lib/mongo/operation/insert/op_msg.rb | 3 +- lib/mongo/operation/shared/executable.rb | 9 ++--- 12 files changed, 101 insertions(+), 33 deletions(-) create mode 100644 lib/mongo/open_telemetry/shared.rb diff --git a/lib/mongo/collection/view/iterable.rb b/lib/mongo/collection/view/iterable.rb index 0321a0f4ef..c83adf705a 100644 --- a/lib/mongo/collection/view/iterable.rb +++ b/lib/mongo/collection/view/iterable.rb @@ -170,15 +170,17 @@ def initial_query_op(session) def send_initial_query(server, context) operation = initial_query_op(context.session) builder = OpenTelemetry::OperationSpanBuilder.new - span_name, span_attrs = builder.build('find', operation) - OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |_span, _context| + span_name, span_attrs = builder.build('find', operation, context) + OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |span, _context| if server.load_balancer? # Connection will be checked in when cursor is drained. connection = server.pool.check_out(context: context) - operation.execute_with_connection(connection, context: context) + result = operation.execute_with_connection(connection, context: context) else - operation.execute(server, context: context) + result = operation.execute(server, context: context) end + builder.add_attributes_from_result(span, result) + result end end diff --git a/lib/mongo/cursor.rb b/lib/mongo/cursor.rb index 1dcb2dfc9e..3c1c7ba08a 100644 --- a/lib/mongo/cursor.rb +++ b/lib/mongo/cursor.rb @@ -514,7 +514,7 @@ def unregister def execute_operation(op, context: nil) builder = OpenTelemetry::OperationSpanBuilder.new - span_name, span_attrs = builder.build('getMore', op) + span_name, span_attrs = builder.build('getMore', op, context) OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |_span, _context| op_context = context || possibly_refreshed_context if @connection.nil? diff --git a/lib/mongo/open_telemetry.rb b/lib/mongo/open_telemetry.rb index cea489cb89..265f3ee13e 100644 --- a/lib/mongo/open_telemetry.rb +++ b/lib/mongo/open_telemetry.rb @@ -46,6 +46,7 @@ def clear_current end end +require 'mongo/open_telemetry/shared' require 'mongo/open_telemetry/command_span_builder' require 'mongo/open_telemetry/operation_span_builder' require 'mongo/open_telemetry/tracer' diff --git a/lib/mongo/open_telemetry/command_span_builder.rb b/lib/mongo/open_telemetry/command_span_builder.rb index f7514a9a41..251bbedfe6 100644 --- a/lib/mongo/open_telemetry/command_span_builder.rb +++ b/lib/mongo/open_telemetry/command_span_builder.rb @@ -17,25 +17,19 @@ module Mongo module OpenTelemetry class CommandSpanBuilder + include OpenTelemetry::Shared def build(command, address) [ span_name(command), build_attributes(command, address) ] end - # @param [ OpenTelemetry::Trace::Span | nil ] span - # @param [ Mongo::Operation::Result ] result - def add_attributes_from_result(span, result) + def add_query_text(span, message) return if span.nil? - if result.successful? - if (cursor_id = result.cursor_id).positive? - span.add_attributes( - 'db.mongodb.cursor_id' => cursor_id - ) - end - else - span.record_exception(result.error) - end + query_text = mask(message.payload[:command]) + span.add_attributes( + 'db.query.text' => query_text.as_extended_json.to_s + ) unless query_text.empty? end private @@ -76,6 +70,16 @@ def build_attributes(command, address) def collection_name(command) command.values.first if command.values.first.is_a?(String) end + + private + + def statement(command) + mask(command) + end + + def mask(hash) + hash.reject { |k, v| Mongo::Protocol::Msg::INTERNAL_KEYS.include?(k.to_s) } + end end end end diff --git a/lib/mongo/open_telemetry/operation_span_builder.rb b/lib/mongo/open_telemetry/operation_span_builder.rb index ddfbc7a714..28f2d37663 100644 --- a/lib/mongo/open_telemetry/operation_span_builder.rb +++ b/lib/mongo/open_telemetry/operation_span_builder.rb @@ -17,17 +17,34 @@ module Mongo module OpenTelemetry class OperationSpanBuilder + include OpenTelemetry::Shared - def build(name, operation) - attrs = { + def build(name, operation, context) + [ + build_span_name(name, operation), + build_span_attrs(name, operation, context) + ] + end + + private + + def build_span_name(op_name, op) + if (coll_name = op.spec[:coll_name]) + "#{op_name} #{op.spec[:db_name]}.#{coll_name}" + else + op_name + end + end + + def build_span_attrs(op_name, op, op_context) + pp op_context&.in_transaction? + { 'db.system' => 'mongodb', - 'db.namespace' => operation.spec[:db_name], - 'db.collection.name' => operation.spec[:coll_name], - 'db.operation.name' => name, - 'db.operation.summary' => "#{name} #{operation.spec[:coll_name]}" + 'db.namespace' => op.spec[:db_name], + 'db.collection.name' => op.spec[:coll_name], + 'db.operation.name' => op_name, + 'db.operation.summary' => build_span_name(op_name, op) } - - [name, attrs] end end end diff --git a/lib/mongo/open_telemetry/shared.rb b/lib/mongo/open_telemetry/shared.rb new file mode 100644 index 0000000000..403dea5e43 --- /dev/null +++ b/lib/mongo/open_telemetry/shared.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Mongo + module OpenTelemetry + module Shared + # @param [ OpenTelemetry::Trace::Span | nil ] span + # @param [ Mongo::Operation::Result ] result + def add_attributes_from_result(span, result) + return if span.nil? || result.nil? + + if result.successful? + if result.has_cursor_id? && (cursor_id = result.cursor_id).positive? + span.add_attributes( + 'db.mongodb.cursor_id' => cursor_id + ) + end + else + span.record_exception(result.error) + end + end + end + end +end diff --git a/lib/mongo/open_telemetry/tracer.rb b/lib/mongo/open_telemetry/tracer.rb index f5b6a8bd24..4e03a0fbb8 100644 --- a/lib/mongo/open_telemetry/tracer.rb +++ b/lib/mongo/open_telemetry/tracer.rb @@ -67,8 +67,6 @@ def start_span(name, attributes: {}, with_parent: nil, &block) @ot_tracer.start_span(name, with_parent: with_parent, attributes: attributes, &block) end - private - # @return [ Boolean ] whether OpenTelemetry tracing is enabled. def enabled? @ot_tracer != nil diff --git a/lib/mongo/operation/find/result.rb b/lib/mongo/operation/find/result.rb index 9bae6d6914..1b320c8551 100644 --- a/lib/mongo/operation/find/result.rb +++ b/lib/mongo/operation/find/result.rb @@ -25,6 +25,14 @@ class Find # @api semiprivate class Result < Operation::Result + def has_cursor_id? + if cursor_document && cursor_document.key?(CURSOR_ID) + true + else + super + end + end + # Get the cursor id. # # @example Get the cursor id. diff --git a/lib/mongo/operation/get_more/result.rb b/lib/mongo/operation/get_more/result.rb index df84c458e6..0f14459138 100644 --- a/lib/mongo/operation/get_more/result.rb +++ b/lib/mongo/operation/get_more/result.rb @@ -24,6 +24,13 @@ class GetMore # @since 2.2.0 # @api semiprivate class Result < Operation::Result + def has_cursor_id? + if cursor_document && cursor_document.key?(CURSOR_ID) + true + else + super + end + end # Get the cursor id. # diff --git a/lib/mongo/operation/indexes/result.rb b/lib/mongo/operation/indexes/result.rb index 6211f6b560..2195e09f9f 100644 --- a/lib/mongo/operation/indexes/result.rb +++ b/lib/mongo/operation/indexes/result.rb @@ -25,6 +25,13 @@ class Indexes # @since 2.0.0 # @api semiprivate class Result < Operation::Result + def has_cursor_id? + if cursor_document && cursor_document.key?(CURSOR_ID) + true + else + super + end + end # Get the cursor id for the result. # @@ -102,4 +109,3 @@ def first_document end end end - diff --git a/lib/mongo/operation/insert/op_msg.rb b/lib/mongo/operation/insert/op_msg.rb index 39b299ef76..3d683f99e6 100644 --- a/lib/mongo/operation/insert/op_msg.rb +++ b/lib/mongo/operation/insert/op_msg.rb @@ -35,7 +35,8 @@ class OpMsg < OpMsgBase def get_result(connection, context, options = {}) # This is a Mongo::Operation::Insert::Result - Result.new(*dispatch_message(connection, context), @ids, context: context) + message = build_message(connection, context) + Result.new(*dispatch_message(message, connection, context), @ids, context: context) end def selector(connection) diff --git a/lib/mongo/operation/shared/executable.rb b/lib/mongo/operation/shared/executable.rb index 80513ed6a5..1b7f0af69d 100644 --- a/lib/mongo/operation/shared/executable.rb +++ b/lib/mongo/operation/shared/executable.rb @@ -107,7 +107,9 @@ def get_result(connection, context, options = {}) builder = OpenTelemetry::CommandSpanBuilder.new span_name, span_attrs = builder.build(command(connection), connection.address) span = OpenTelemetry.tracer.start_span(span_name, attributes: span_attrs, with_parent: OpenTelemetry.current_context) - result_class.new(*dispatch_message(connection, context, options), context: context, connection: connection).tap do |result| + message = build_message(connection, context) + builder.add_query_text(span, message) if OpenTelemetry.tracer.query_text? + result_class.new(*dispatch_message(message, connection, context, options), context: context, connection: connection).tap do |result| builder.add_attributes_from_result(span, result) end ensure @@ -115,8 +117,7 @@ def get_result(connection, context, options = {}) end # Returns a Protocol::Message or nil as reply. - def dispatch_message(connection, context, options = {}) - message = build_message(connection, context) + def dispatch_message(message, connection, context, options = {}) message = message.maybe_encrypt(connection, context) reply = connection.dispatch([ message ], context, options) [reply, connection.description, connection.global_id] @@ -127,7 +128,7 @@ def dispatch_message(connection, context, options = {}) # @param [ Mongo::Operation::Context ] context The operation context. def build_message(connection, context) msg = message(connection) - if server_api = context.server_api + if ( server_api = context.server_api ) msg = msg.maybe_add_server_api(server_api) end msg From 75e7b0cccc8e48183b39adf80505357da8047517 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Fri, 7 Feb 2025 08:41:11 +0100 Subject: [PATCH 14/14] Operations and commands --- lib/mongo/collection.rb | 37 +++-- lib/mongo/collection/view/aggregation.rb | 22 ++- lib/mongo/collection/view/iterable.rb | 10 +- lib/mongo/collection/view/readable.rb | 43 ++--- lib/mongo/collection/view/writable.rb | 149 ++++++++++-------- lib/mongo/open_telemetry.rb | 32 ++++ .../open_telemetry/method_span_builder.rb | 50 ++++++ .../open_telemetry/operation_span_builder.rb | 13 +- lib/mongo/operation/shared/specifiable.rb | 4 + 9 files changed, 237 insertions(+), 123 deletions(-) create mode 100644 lib/mongo/open_telemetry/method_span_builder.rb diff --git a/lib/mongo/collection.rb b/lib/mongo/collection.rb index b9cbefee0c..faf7900fb2 100644 --- a/lib/mongo/collection.rb +++ b/lib/mongo/collection.rb @@ -453,25 +453,27 @@ def create(opts = {}) # @since 2.0.0 def drop(opts = {}) client.with_session(opts) do |session| - maybe_drop_emm_collections(opts[:encrypted_fields], client, session) do - temp_write_concern = write_concern - write_concern = if opts[:write_concern] - WriteConcern.get(opts[:write_concern]) - else - temp_write_concern - end - context = Operation::Context.new( + context = Operation::Context.new( client: client, session: session, operation_timeouts: operation_timeouts(opts) ) - operation = Operation::Drop.new({ + temp_write_concern = write_concern + write_concern = if opts[:write_concern] + WriteConcern.get(opts[:write_concern]) + else + temp_write_concern + end + operation = Operation::Drop.new({ selector: { :drop => name }, db_name: database.name, write_concern: write_concern, session: session, }) - do_drop(operation, session, context) + OpenTelemetry.trace_operation(operation, context) do + maybe_drop_emm_collections(opts[:encrypted_fields], client, session) do + do_drop(operation, session, context) + end end end end @@ -865,9 +867,8 @@ def insert_one(document, opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - Operation::Insert.new( - :documents => [ document ], + operation = Operation::Insert.new( + :documents => [document], :db_name => database.name, :coll_name => name, :write_concern => write_concern, @@ -875,9 +876,13 @@ def insert_one(document, opts = {}) :options => opts, :id_generator => client.options[:id_generator], :session => session, - :txn_num => txn_num, :comment => opts[:comment] - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'insert_one') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end @@ -910,7 +915,7 @@ def insert_one(document, opts = {}) def insert_many(documents, options = {}) QueryCache.clear_namespace(namespace) - inserts = documents.map{ |doc| { :insert_one => doc }} + inserts = documents.map { |doc| { :insert_one => doc } } bulk_write(inserts, options) end diff --git a/lib/mongo/collection/view/aggregation.rb b/lib/mongo/collection/view/aggregation.rb index f80a4f491b..69ad5c2cbb 100644 --- a/lib/mongo/collection/view/aggregation.rb +++ b/lib/mongo/collection/view/aggregation.rb @@ -121,22 +121,28 @@ def send_initial_query(server, context) if server.load_balancer? # Connection will be checked in when cursor is drained. connection = server.pool.check_out(context: context) - initial_query_op( + op = initial_query_op( context.session, effective_read_preference(connection) - ).execute_with_connection( - connection, - context: context ) + OpenTelemetry.trace_operation(op, context) do + op.execute_with_connection( + connection, + context: context + ) + end else server.with_connection do |connection| - initial_query_op( + op = initial_query_op( context.session, effective_read_preference(connection) - ).execute_with_connection( - connection, - context: context ) + OpenTelemetry.trace_operation(op, context) do + op.execute_with_connection( + connection, + context: context + ) + end end end end diff --git a/lib/mongo/collection/view/iterable.rb b/lib/mongo/collection/view/iterable.rb index c83adf705a..dd339f2588 100644 --- a/lib/mongo/collection/view/iterable.rb +++ b/lib/mongo/collection/view/iterable.rb @@ -169,18 +169,14 @@ def initial_query_op(session) def send_initial_query(server, context) operation = initial_query_op(context.session) - builder = OpenTelemetry::OperationSpanBuilder.new - span_name, span_attrs = builder.build('find', operation, context) - OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |span, _context| + OpenTelemetry.trace_operation(operation, context) do if server.load_balancer? # Connection will be checked in when cursor is drained. connection = server.pool.check_out(context: context) - result = operation.execute_with_connection(connection, context: context) + operation.execute_with_connection(connection, context: context) else - result = operation.execute(server, context: context) + operation.execute(server, context: context) end - builder.add_attributes_from_result(span, result) - result end end diff --git a/lib/mongo/collection/view/readable.rb b/lib/mongo/collection/view/readable.rb index 05fcc78df7..a2e94008f8 100644 --- a/lib/mongo/collection/view/readable.rb +++ b/lib/mongo/collection/view/readable.rb @@ -192,8 +192,7 @@ def count(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - read_with_retry(session, selector, context) do |server| - Operation::Count.new( + operation = Operation::Count.new( selector: cmd, db_name: database.name, options: {:limit => -1}, @@ -203,10 +202,14 @@ def count(opts = {}) # string key. Note that this isn't documented as valid usage. collation: opts[:collation] || opts['collation'] || collation, comment: opts[:comment], - ).execute( - server, - context: context ) + OpenTelemetry.trace_operation(operation, context) do + read_with_retry(session, selector, context) do |server| + operation.execute( + server, + context: context + ) + end end.n.to_i end end @@ -294,21 +297,23 @@ def estimated_document_count(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - read_with_retry(session, selector, context) do |server| - cmd = { count: collection.name } - cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms] - if read_concern - cmd[:readConcern] = Options::Mapper.transform_values_to_strings(read_concern) - end - result = Operation::Count.new( - selector: cmd, - db_name: database.name, - read: read_pref, - session: session, - comment: opts[:comment], - ).execute(server, context: context) - result.n.to_i + cmd = { count: collection.name } + cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms] + if read_concern + cmd[:readConcern] = Options::Mapper.transform_values_to_strings(read_concern) end + operation = Operation::Count.new( + selector: cmd, + db_name: database.name, + read: read_pref, + session: session, + comment: opts[:comment], + ) + OpenTelemetry.trace_operation(operation, context) do + read_with_retry(session, selector, context) do |server| + operation.execute(server, context: context) + end + end.n.to_i end rescue Error::OperationFailure::Family => exc if exc.code == 26 diff --git a/lib/mongo/collection/view/writable.rb b/lib/mongo/collection/view/writable.rb index 0a1f553b1d..01574a637f 100644 --- a/lib/mongo/collection/view/writable.rb +++ b/lib/mongo/collection/view/writable.rb @@ -90,19 +90,22 @@ def find_one_and_delete(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_4 = connection.server.description.server_version_gte?('4.4') - if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::WriteCommand.new( + operation = Operation::WriteCommand.new( selector: cmd, db_name: database.name, write_concern: write_concern, session: session, - txn_num: txn_num, - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'find_one_and_delete') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_4 = connection.server.description.server_version_gte?('4.4') + if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end.first&.fetch('value', nil) end @@ -211,19 +214,21 @@ def find_one_and_update(document, opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_4 = connection.server.description.server_version_gte?('4.4') - if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::WriteCommand.new( + operation = Operation::WriteCommand.new( selector: cmd, db_name: database.name, write_concern: write_concern, session: session, - txn_num: txn_num, - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'find_one_and_update') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_4 = connection.server.description.server_version_gte?('4.4') + if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end.first&.fetch('value', nil) value unless value.nil? || value.empty? @@ -275,14 +280,8 @@ def delete_many(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_4 = connection.server.description.server_version_gte?('4.4') - if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::Delete.new( - deletes: [ delete_doc ], + operation = Operation::Delete.new( + deletes: [delete_doc], db_name: collection.database.name, coll_name: collection.name, write_concern: write_concern, @@ -290,7 +289,16 @@ def delete_many(opts = {}) session: session, let: opts[:let], comment: opts[:comment], - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'delete_many') do + nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_4 = connection.server.description.server_version_gte?('4.4') + if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end @@ -342,23 +350,25 @@ def delete_one(opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_4 = connection.server.description.server_version_gte?('4.4') - if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::Delete.new( - deletes: [ delete_doc ], + operation = Operation::Delete.new( + deletes: [delete_doc], db_name: collection.database.name, coll_name: collection.name, write_concern: write_concern, bypass_document_validation: !!opts[:bypass_document_validation], session: session, - txn_num: txn_num, let: opts[:let], comment: opts[:comment], - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'delete_one') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_4 = connection.server.description.server_version_gte?('4.4') + if !gte_4_4 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end @@ -420,23 +430,25 @@ def replace_one(replacement, opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_2 = connection.server.description.server_version_gte?('4.2') - if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::Update.new( - updates: [ update_doc ], + operation = Operation::Update.new( + updates: [update_doc], db_name: collection.database.name, coll_name: collection.name, write_concern: write_concern, bypass_document_validation: !!opts[:bypass_document_validation], session: session, - txn_num: txn_num, let: opts[:let], comment: opts[:comment], - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'update_one') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_2 = connection.server.description.server_version_gte?('4.2') + if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end @@ -501,14 +513,8 @@ def update_many(spec, opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_2 = connection.server.description.server_version_gte?('4.2') - if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::Update.new( - updates: [ update_doc ], + operation = Operation::Update.new( + updates: [update_doc], db_name: collection.database.name, coll_name: collection.name, write_concern: write_concern, @@ -516,7 +522,16 @@ def update_many(spec, opts = {}) session: session, let: opts[:let], comment: opts[:comment], - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'update_many') do + nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_2 = connection.server.description.server_version_gte?('4.2') + if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end @@ -580,23 +595,25 @@ def update_one(spec, opts = {}) session: session, operation_timeouts: operation_timeouts(opts) ) - write_with_retry(write_concern, context: context) do |connection, txn_num, context| - gte_4_2 = connection.server.description.server_version_gte?('4.2') - if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? - raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) - end - - Operation::Update.new( - updates: [ update_doc ], + operation = Operation::Update.new( + updates: [update_doc], db_name: collection.database.name, coll_name: collection.name, write_concern: write_concern, bypass_document_validation: !!opts[:bypass_document_validation], session: session, - txn_num: txn_num, let: opts[:let], comment: opts[:comment], - ).execute_with_connection(connection, context: context) + ) + OpenTelemetry.trace_operation(operation, context, 'update_one') do + write_with_retry(write_concern, context: context) do |connection, txn_num, context| + gte_4_2 = connection.server.description.server_version_gte?('4.2') + if !gte_4_2 && opts[:hint] && write_concern && !write_concern.acknowledged? + raise Error::UnsupportedOption.hint_error(unacknowledged_write: true) + end + operation.txn_num = txn_num + operation.execute_with_connection(connection, context: context) + end end end end diff --git a/lib/mongo/open_telemetry.rb b/lib/mongo/open_telemetry.rb index 265f3ee13e..f7101c848c 100644 --- a/lib/mongo/open_telemetry.rb +++ b/lib/mongo/open_telemetry.rb @@ -20,33 +20,65 @@ module OpenTelemetry def tracer Tracer.instance end + module_function :tracer def set_current(span, context) Thread.current['mongo-ruby-driver-otel-span'] = span Thread.current['mongo-ruby-driver-otel-context'] = context end + module_function :set_current def current_span Thread.current['mongo-ruby-driver-otel-span'] end + module_function :current_span def current_context Thread.current['mongo-ruby-driver-otel-context'] end + module_function :current_context def clear_current Thread.current['mongo-ruby-driver-otel-context'] = nil Thread.current['mongo-ruby-driver-otel-span'] = nil end + module_function :clear_current + + def trace_operation(operation, context, operation_name = nil) + builder = OpenTelemetry::OperationSpanBuilder.new + operation_name = operation.class.name.split("::").last.downcase if operation_name.nil? + span_name, span_attrs = builder.build(operation_name, operation, context) + OpenTelemetry.tracer.in_span(span_name, attributes: span_attrs) do |span, _context| + result = yield + builder.add_attributes_from_result(span, result) + result + end + end + + module_function :trace_operation + + def trace_driver_method(db, coll) + name = caller[0][/`.*'/][1..-2] + builder = OpenTelemetry::MethodSpanBuilder.new + span_name, span_attrs = builder.build(name, db, coll) + OpenTelemetry.trace.in_span(span_name, attributes: span_attrs) do |span, _context| + result = yield + builder.add_attributes_from_result(span, result) + result + end + end + + module_function :trace_driver_method end end require 'mongo/open_telemetry/shared' require 'mongo/open_telemetry/command_span_builder' +require 'mongo/open_telemetry/method_span_builder' require 'mongo/open_telemetry/operation_span_builder' require 'mongo/open_telemetry/tracer' diff --git a/lib/mongo/open_telemetry/method_span_builder.rb b/lib/mongo/open_telemetry/method_span_builder.rb new file mode 100644 index 0000000000..12958c6948 --- /dev/null +++ b/lib/mongo/open_telemetry/method_span_builder.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Copyright (C) 2025-present MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Mongo + module OpenTelemetry + class MethodSpanBuilder + include OpenTelemetry::Shared + + def build(name, db, collection) + [ + build_span_name(name, operation), + build_span_attrs(name, operation, context) + ] + end + + private + + def build_span_name(name, db, collection) + if (collection) + "#{name} #{db}.#{collection}" + else + name + end + end + + def build_span_attrs(name, db, collection) + { + 'db.system' => 'mongodb', + 'db.namespace' => db, + 'db.collection.name' => collection, + 'db.operation.name' => name, + 'db.operation.summary' => build_span_name(name, db, collection) + } + end + end + end +end diff --git a/lib/mongo/open_telemetry/operation_span_builder.rb b/lib/mongo/open_telemetry/operation_span_builder.rb index 28f2d37663..c4511ba280 100644 --- a/lib/mongo/open_telemetry/operation_span_builder.rb +++ b/lib/mongo/open_telemetry/operation_span_builder.rb @@ -37,14 +37,13 @@ def build_span_name(op_name, op) end def build_span_attrs(op_name, op, op_context) - pp op_context&.in_transaction? { - 'db.system' => 'mongodb', - 'db.namespace' => op.spec[:db_name], - 'db.collection.name' => op.spec[:coll_name], - 'db.operation.name' => op_name, - 'db.operation.summary' => build_span_name(op_name, op) - } + 'db.system' => 'mongodb', + 'db.namespace' => op.spec[:db_name], + 'db.collection.name' => op.spec[:coll_name], + 'db.operation.name' => op_name, + 'db.operation.summary' => build_span_name(op_name, op) + } end end end diff --git a/lib/mongo/operation/shared/specifiable.rb b/lib/mongo/operation/shared/specifiable.rb index afc799f46e..b2dd45050b 100644 --- a/lib/mongo/operation/shared/specifiable.rb +++ b/lib/mongo/operation/shared/specifiable.rb @@ -526,6 +526,10 @@ def txn_num @spec[:txn_num] end + def txn_num=(value) + @spec[:txn_num] = value + end + # The command. # # @return [ Hash ] The command.