RUBY-3352 Backport RUBY-3268 atlas search index helpers to 2.19-stable (#2804)

* RUBY-3268 search index management helpers (#2777)

* RUBY-3268 Index View API for Search Indexes

* documentation, and align method names with the spec

* return the correct value here

* suppress NamespaceNotFound errors for the drop operation

* prose tests for search indexes

* rubocop

* prose tests pass

* first stab at evergreen config for index management specs

* rubocop

* gah, executable permissions

* don't use FLE for full atlas tests

* make sure MONGODB_URI is set

* set the timeout higher for the search index specs

* pass all aggregation options through to the list indexes command

* use the correct implementation for #empty?

* remove unnecessary validation

* bump drivers-evergreen-tools

* RUBY-3324 bump drivers-evergreen-tools to get updated atlas setup/teardown (#2780)

also, expose task_id expansion as environment variable to those scripts

* RUBY-3328 add `execution` expansion to environment for atlas cluster name uniqueness (#2783)

Author: jamis

.evergreen/config.yml

@@ -469,6 +469,50 @@ post:
   - func: "delete serverless instance"
 
 task_groups:
+  - name: testatlas_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: create expansions
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \
+              DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              MONGODB_VERSION="7.0" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh
+      - command: expansions.update
+        params:
+          file: src/atlas-expansion.yml
+    teardown_group:
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh
+    tasks:
+      - test-full-atlas-task
+
   - name: testgcpkms_task_group
     setup_group_can_fail_task: true
     setup_group_timeout_secs: 1800 # 30 minutes
@@ -586,6 +630,16 @@ tasks:
     commands:
       - func: "export AWS auth credentials"
       - func: "run AWS auth tests"
+  - name: "test-full-atlas-task"
+    commands:
+      - command: shell.exec
+        type: test
+        params:
+          working_dir: "src"
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh
   - name: "testgcpkms-task"
     commands:
       - command: shell.exec
@@ -1612,6 +1666,14 @@ buildvariants:
       - name: testazurekms_task_group
         batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
 
+  - matrix_name: atlas-full
+    matrix_spec:
+      ruby: "ruby-3.2"
+      os: rhel8
+    display_name: "Atlas (Full)"
+    tasks:
+      - name: testatlas_task_group
+
   - matrix_name: "atlas"
     matrix_spec:
       ruby: ["ruby-3.2", "ruby-3.1", "ruby-3.0", "ruby-2.7", "ruby-2.6", "ruby-2.5", "jruby-9.3", "jruby-9.2"]

.evergreen/config/common.yml.erb

@@ -466,6 +466,50 @@ post:
   - func: "delete serverless instance"
 
 task_groups:
+  - name: testatlas_task_group
+    setup_group_can_fail_task: true
+    setup_group_timeout_secs: 1800 # 30 minutes
+    setup_group:
+      - func: fetch source
+      - func: create expansions
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \
+              DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              MONGODB_VERSION="7.0" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh
+      - command: expansions.update
+        params:
+          file: src/atlas-expansion.yml
+    teardown_group:
+      - command: shell.exec
+        params:
+          shell: "bash"
+          working_dir: "src"
+          script: |
+            ${PREPARE_SHELL}
+
+            DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \
+              DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \
+              DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \
+              LAMBDA_STACK_NAME="dbx-ruby-lambda" \
+              task_id="${task_id}" \
+              execution="${execution}" \
+              $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh
+    tasks:
+      - test-full-atlas-task
+
   - name: testgcpkms_task_group
     setup_group_can_fail_task: true
     setup_group_timeout_secs: 1800 # 30 minutes
@@ -583,6 +627,16 @@ tasks:
     commands:
       - func: "export AWS auth credentials"
       - func: "run AWS auth tests"
+  - name: "test-full-atlas-task"
+    commands:
+      - command: shell.exec
+        type: test
+        params:
+          working_dir: "src"
+          shell: "bash"
+          script: |
+            ${PREPARE_SHELL}
+            MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh
   - name: "testgcpkms-task"
     commands:
       - command: shell.exec

.evergreen/config/standard.yml.erb

@@ -497,6 +497,14 @@ buildvariants:
       - name: testazurekms_task_group
         batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
 
+  - matrix_name: atlas-full
+    matrix_spec:
+      ruby: <%= latest_ruby %>
+      os: rhel8
+    display_name: "Atlas (Full)"
+    tasks:
+      - name: testatlas_task_group
+
   - matrix_name: "atlas"
     matrix_spec:
       ruby: <%= supported_rubies %>

.evergreen/run-tests-atlas-full.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+. `dirname "$0"`/../spec/shared/shlib/distro.sh
+. `dirname "$0"`/../spec/shared/shlib/set_env.sh
+. `dirname "$0"`/functions.sh
+
+set_env_vars
+set_env_python
+set_env_ruby
+
+bundle_install
+
+ATLAS_URI=$MONGODB_URI \
+  SERVERLESS=1 \
+  EXAMPLE_TIMEOUT=600 \
+  bundle exec rspec -fd spec/integration/search_indexes_prose_spec.rb
+
+test_status=$?
+
+kill_jruby
+
+exit ${test_status}

.gitignore

@@ -25,3 +25,8 @@ data/*
 gemfiles/*.gemfile.lock
 .env.private*
 .env
+build
+profile/benchmarking/data
+secrets-export.sh
+secrets-expansion.yml
+atlas-expansion.yml

.mod/drivers-evergreen-tools

@@ -56,6 +56,9 @@ Metrics/ModuleLength:
 Metrics/MethodLength:
   Max: 20
 
+Naming/MethodParameterName:
+  AllowedNames: [ id, op ]
+
 RSpec/BeforeAfterAll:
   Enabled: false
 

.rubocop.yml

@@ -64,6 +64,7 @@
 require 'mongo/dbref'
 require 'mongo/grid'
 require 'mongo/index'
+require 'mongo/search_index/view'
 require 'mongo/lint'
 require 'mongo/query_cache'
 require 'mongo/server'

lib/mongo.rb

@@ -725,13 +725,35 @@ def distinct(field_name, filter = nil, options = {})
     #
     # @option options [ Session ] :session The session to use.
     #
-    # @return [ View::Index ] The index view.
+    # @return [ Index::View ] The index view.
     #
     # @since 2.0.0
     def indexes(options = {})
       Index::View.new(self, options)
     end
 
+    # Get a view of all search indexes for this collection. Can be iterated or
+    # operated on directly. If id or name are given, the iterator will return
+    # only the indicated index. For all other operations, id and name are
+    # ignored.
+    #
+    # @note Only one of id or name may be given; it is an error to specify both,
+    #   although both may be omitted safely.
+    #
+    # @param [ Hash ] options The options to use to configure the view.
+    #
+    # @option options [ String ] :id The id of the specific index to query (optional)
+    # @option options [ String ] :name The name of the specific index to query (optional)
+    # @option options [ Hash ] :aggregate The options hash to pass to the
+    #    aggregate command (optional)
+    #
+    # @return [ SearchIndex::View ] The search index view.
+    #
+    # @since 2.0.0
+    def search_indexes(options = {})
+      SearchIndex::View.new(self, options)
+    end
+
     # Get a pretty printed string inspection for the collection.
     #
     # @example Inspect the collection.

lib/mongo/collection.rb

@@ -51,6 +51,9 @@
 require 'mongo/operation/remove_user'
 require 'mongo/operation/create_index'
 require 'mongo/operation/drop_index'
+require 'mongo/operation/create_search_indexes'
+require 'mongo/operation/drop_search_index'
+require 'mongo/operation/update_search_index'
 
 module Mongo