Update payloads to new Queryable Encryption protocol (#1103)

JAVA-4891

Author: jyemin

build.gradle

@@ -55,7 +55,7 @@ ext {
     zstdVersion = '1.5.2-5'
     awsSdkV2Version = '2.18.9'
     awsSdkV1Version = '1.12.337'
-    mongoCryptVersion = '1.7.3'
+    mongoCryptVersion = '1.8.0-SNAPSHOT'
     projectReactorVersion = '2022.0.0'
     junitBomVersion = '5.8.2'
     gitVersion = getGitVersion()
@@ -82,12 +82,12 @@ configure(coreProjects) {
         mavenCentral()
 
         // Uncomment this to test with a snapshot build of mongodb-crypt
-//        maven {
-//            url 'https://oss.sonatype.org/content/repositories/snapshots'
-//            content {
-//                includeGroup "org.mongodb"
-//            }
-//        }
+        maven {
+            url 'https://oss.sonatype.org/content/repositories/snapshots'
+            content {
+                includeGroup "org.mongodb"
+            }
+        }
     }
 }
 

driver-core/src/test/resources/client-side-encryption/README.rst

@@ -163,6 +163,8 @@ Do the following before running spec tests:
 
 Load each YAML (or JSON) file using a Canonical Extended JSON parser.
 
+If the test file name matches the regular expression ``fle2\-Range\-.*\-Correctness``, drivers MAY skip the test on macOS. The ``fle2-Range`` tests are very slow on macOS and do not provide significant additional test coverage.
+
 Then for each element in ``tests``:
 
 #. If the ``skipReason`` field is present, skip this test completely.
@@ -1677,7 +1679,14 @@ Expect no error on construction.
 12. Explicit Encryption
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-The Explicit Encryption tests require MongoDB server 6.0+. The tests must not run against a standalone.
+The Explicit Encryption tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 Before running each of the following test cases, perform the following Test Setup.
 
@@ -2493,7 +2502,14 @@ The following tests that a mongocryptd client is not created when shared library
 21. Automatic Data Encryption Keys
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The Automatic Data Encryption Keys tests require MongoDB server 6.0+. The tests must not run against a standalone.
+The Automatic Data Encryption Keys tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 For each of the following test cases, assume `DB` is a valid open database
 handle, and assume a ClientEncryption_ object `CE` created using the following
@@ -2504,9 +2520,24 @@ options::
       keyVaultNamespace: "keyvault.datakeys",
       kmsProviders: {
          local: { key: base64Decode(LOCAL_MASTERKEY) },
+         aws: {
+            accessKeyId: <set from environment>,
+            secretAccessKey: <set from environment>
+         },
       },
    }
 
+Run each test case with each of these KMS providers: ``aws``, ``local``. The KMS provider name is referred to as ``kmsProvider``.
+When testing ``aws``, use the following as the ``masterKey`` option:
+
+.. code:: javascript
+
+   {
+      region: "us-east-1",
+      key: "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0"
+   }
+
+When testing ``local``, set ``masterKey`` to ``null``.
 
 Case 1: Simple Creation and Validation
 ``````````````````````````````````````
@@ -2533,7 +2564,7 @@ rejects an attempt to insert plaintext in an encrypted fields.
          }
       }
 
-2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, "local", null)`
+2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)`
    to obtain a new collection `Coll`. Expect success.
 3. Attempt to insert the following document into `Coll`::
 
@@ -2557,7 +2588,7 @@ missing.
 
 1. Create a new empty create-collection options `Opts`. (i.e. it must not
    contain any ``encryptedFields`` options.)
-2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, "local", null)`.
+2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)`.
 3. Expect the invocation to fail with an error indicating that
    ``encryptedFields`` is not defined for the collection, and expect that no
    collection was created within the database. It would be *incorrect* for
@@ -2590,7 +2621,7 @@ when attempting to create a collection with such invalid settings.
          }
       }
 
-2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, "local", null)`.
+2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)`.
 3. Expect an error from the server indicating a validation error at
    ``create.encryptedFields.fields.keyId``, which must be a UUID and not a
    boolean value.
@@ -2613,7 +2644,7 @@ with encrypted value.
          }
       }
 
-2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, "local", null)`
+2. Invoke `CreateEncryptedCollection(CE, DB, "testing1", Opts, kmsProvider, masterKey)`
    to obtain a new collection `Coll` and data key `key1`. Expect success.
 3. Use `CE` to explicitly encrypt the string "123-45-6789" using
    algorithm `Unindexed` and data key `key1`. Refer result as `encryptedPayload`.
@@ -2627,7 +2658,14 @@ with encrypted value.
 
 22. Range Explicit Encryption
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The Range Explicit Encryption tests require MongoDB server 6.2+. The tests must not run against a standalone.
+The Range Explicit Encryption tests require MongoDB server 7.0+. The tests must not run against a standalone.
+
+.. note::
+   MongoDB Server 7.0 introduced a backwards breaking change to the Queryable Encryption (QE) protocol: QEv2.
+	libmongocrypt 1.8.0 is configured to use the QEv2 protocol.
+
+.. note::
+   Skip this test on Serverless until MongoDB Serverless enables the QEv2 protocol. Refer: `DRIVERS-2589 <https://jira.mongodb.org/browse/DRIVERS-2589>`_
 
 Each of the following test cases must pass for each of the supported types (``DecimalNoPrecision``, ``DecimalPrecision``, ``DoublePrecision``, ``DoubleNoPrecision``, ``Date``, ``Int``, and ``Long``), unless it is stated the type should be skipped.
 

driver-core/src/test/resources/client-side-encryption/legacy/fle2-BypassQueryAnalysis.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-BypassQueryAnalysis.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",
@@ -75,36 +76,6 @@
       "masterKey": {
         "provider": "local"
       }
-    },
-    {
-      "_id": {
-        "$binary": {
-          "base64": "q83vqxI0mHYSNBI0VniQEg==",
-          "subType": "04"
-        }
-      },
-      "keyMaterial": {
-        "$binary": {
-          "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==",
-          "subType": "00"
-        }
-      },
-      "creationDate": {
-        "$date": {
-          "$numberLong": "1648914851981"
-        }
-      },
-      "updateDate": {
-        "$date": {
-          "$numberLong": "1648914851981"
-        }
-      },
-      "status": {
-        "$numberInt": "0"
-      },
-      "masterKey": {
-        "provider": "local"
-      }
     }
   ],
   "tests": [
@@ -133,7 +104,7 @@
               "_id": 1,
               "encryptedIndexed": {
                 "$binary": {
-                  "base64": "BHEBAAAFZAAgAAAAAHb62aV7+mqmaGcotPLdG3KP7S8diFwWMLM/5rYtqLrEBXMAIAAAAAAVJ6OWHRv3OtCozHpt3ZzfBhaxZirLv3B+G8PuaaO4EgVjACAAAAAAsZXWOWA+UiCBbrJNB6bHflB/cn7pWSvwWN2jw4FPeIUFcABQAAAAAMdD1nV2nqeI1eXEQNskDflCy8I7/HvvqDKJ6XxjhrPQWdLqjz+8GosGUsB7A8ee/uG9/guENuL25XD+Fxxkv1LLXtavHOlLF7iW0u9yabqqBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AE0AAAAAq83vqxI0mHYSNBI0VniQEkzZZBBDgeZh+h+gXEmOrSFtVvkUcnHWj/rfPW7iJ0G3UJ8zpuBmUM/VjOMJCY4+eDqdTiPIwX+/vNXegc8FZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsAA==",
+                  "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==",
                   "subType": "06"
                 }
               }
@@ -150,7 +121,7 @@
           "result": [
             {
               "_id": 1,
-              "encryptedIndexed": "value123"
+              "encryptedIndexed": "123"
             }
           ]
         }
@@ -176,7 +147,7 @@
                   "_id": 1,
                   "encryptedIndexed": {
                     "$binary": {
-                      "base64": "BHEBAAAFZAAgAAAAAHb62aV7+mqmaGcotPLdG3KP7S8diFwWMLM/5rYtqLrEBXMAIAAAAAAVJ6OWHRv3OtCozHpt3ZzfBhaxZirLv3B+G8PuaaO4EgVjACAAAAAAsZXWOWA+UiCBbrJNB6bHflB/cn7pWSvwWN2jw4FPeIUFcABQAAAAAMdD1nV2nqeI1eXEQNskDflCy8I7/HvvqDKJ6XxjhrPQWdLqjz+8GosGUsB7A8ee/uG9/guENuL25XD+Fxxkv1LLXtavHOlLF7iW0u9yabqqBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AE0AAAAAq83vqxI0mHYSNBI0VniQEkzZZBBDgeZh+h+gXEmOrSFtVvkUcnHWj/rfPW7iJ0G3UJ8zpuBmUM/VjOMJCY4+eDqdTiPIwX+/vNXegc8FZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsAA==",
+                      "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==",
                       "subType": "06"
                     }
                   }
@@ -230,39 +201,6 @@
             },
             "command_name": "find"
           }
-        },
-        {
-          "command_started_event": {
-            "command": {
-              "find": "datakeys",
-              "filter": {
-                "$or": [
-                  {
-                    "_id": {
-                      "$in": [
-                        {
-                          "$binary": {
-                            "base64": "q83vqxI0mHYSNBI0VniQEg==",
-                            "subType": "04"
-                          }
-                        }
-                      ]
-                    }
-                  },
-                  {
-                    "keyAltNames": {
-                      "$in": []
-                    }
-                  }
-                ]
-              },
-              "$db": "keyvault",
-              "readConcern": {
-                "level": "majority"
-              }
-            },
-            "command_name": "find"
-          }
         }
       ],
       "outcome": {
@@ -276,7 +214,7 @@
               "__safeContent__": [
                 {
                   "$binary": {
-                    "base64": "ThpoKfQ8AkOzkFfNC1+9PF0pY2nIzfXvRdxQgjkNbBw=",
+                    "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=",
                     "subType": "00"
                   }
                 }

driver-core/src/test/resources/client-side-encryption/legacy/fle2-Compact.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-Compact.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",

driver-core/src/test/resources/client-side-encryption/legacy/fle2-CreateCollection.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-CreateCollection.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",

driver-core/src/test/resources/client-side-encryption/legacy/fle2-DecryptExistingData.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-DecryptExistingData.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",

driver-core/src/test/resources/client-side-encryption/legacy/fle2-Delete.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-Delete.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",
@@ -226,7 +227,7 @@
                     "encryptedIndexed": {
                       "$eq": {
                         "$binary": {
-                          "base64": "BbEAAAAFZAAgAAAAAPtVteJQAlgb2YMa/+7YWH00sbQPyt7L6Rb8OwBdMmL2BXMAIAAAAAAd44hgVKnEnTFlwNVC14oyc9OZOTspeymusqkRQj57nAVjACAAAAAA19X9v9NlWidu/wR5/C/7WUV54DfL5CkNmT5WYrhxdDcFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsEmNtAAAAAAAAAAAAAA==",
+                          "base64": "DIkAAAAFZAAgAAAAAPtVteJQAlgb2YMa/+7YWH00sbQPyt7L6Rb8OwBdMmL2BXMAIAAAAAAd44hgVKnEnTFlwNVC14oyc9OZOTspeymusqkRQj57nAVsACAAAAAAaZ9s3G+4znfxStxeOZwcZy1OhzjMGc5hjmdMN+b/w6kSY20AAAAAAAAAAAAA",
                           "subType": "06"
                         }
                       }
@@ -271,24 +272,6 @@
                       }
                     ]
                   }
-                },
-                "deleteTokens": {
-                  "default.default": {
-                    "encryptedIndexed": {
-                      "e": {
-                        "$binary": {
-                          "base64": "65pz95EthqQpfoHS9nWvdCh05AV+OokP7GUaI+7j8+w=",
-                          "subType": "00"
-                        }
-                      },
-                      "o": {
-                        "$binary": {
-                          "base64": "noN+05JsuO1oDg59yypIGj45i+eFH6HOTXOPpeZ//Mk=",
-                          "subType": "00"
-                        }
-                      }
-                    }
-                  }
                 }
               }
             },

driver-core/src/test/resources/client-side-encryption/legacy/fle2-EncryptedFields-vs-EncryptedFieldsMap.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",

driver-core/src/test/resources/client-side-encryption/legacy/fle2-EncryptedFields-vs-jsonSchema.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-EncryptedFields-vs-jsonSchema.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",
@@ -231,7 +232,7 @@
                 "encryptedIndexed": {
                   "$eq": {
                     "$binary": {
-                      "base64": "BbEAAAAFZAAgAAAAAPGmZcUzdE/FPILvRSyAScGvZparGI2y9rJ/vSBxgCujBXMAIAAAAACi1RjmndKqgnXy7xb22RzUbnZl1sOZRXPOC0KcJkAxmQVjACAAAAAAWuidNu47c9A4Clic3DvFhn1AQJVC+FJtoE5bGZuz6PsFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsEmNtAAAAAAAAAAAAAA==",
+                      "base64": "DIkAAAAFZAAgAAAAAPGmZcUzdE/FPILvRSyAScGvZparGI2y9rJ/vSBxgCujBXMAIAAAAACi1RjmndKqgnXy7xb22RzUbnZl1sOZRXPOC0KcJkAxmQVsACAAAAAApJtKPW4+o9B7gAynNLL26jtlB4+hq5TXResijcYet8USY20AAAAAAAAAAAAA",
                       "subType": "06"
                     }
                   }

driver-core/src/test/resources/client-side-encryption/legacy/fle2-EncryptedFieldsMap-defaults.json renamed to driver-core/src/test/resources/client-side-encryption/legacy/fle2v2-EncryptedFieldsMap-defaults.json

@@ -1,7 +1,8 @@
 {
   "runOn": [
     {
-      "minServerVersion": "6.0.0",
+      "minServerVersion": "7.0.0",
+      "serverless": "forbid",
       "topology": [
         "replicaset",
         "sharded",
@@ -75,7 +76,6 @@
                   "default.default": {
                     "fields": [],
                     "escCollection": "enxcol_.default.esc",
-                    "eccCollection": "enxcol_.default.ecc",
                     "ecocCollection": "enxcol_.default.ecoc"
                   }
                 }