diff --git a/source/collection-management/tests/modifyCollection-errorResponse.json b/source/collection-management/tests/modifyCollection-errorResponse.json new file mode 100644 index 0000000000..aea71eb08f --- /dev/null +++ b/source/collection-management/tests/modifyCollection-errorResponse.json @@ -0,0 +1,118 @@ +{ + "description": "modifyCollection-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "collMod-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "collMod-tests", + "documents": [ + { + "_id": 1, + "x": 1 + }, + { + "_id": 2, + "x": 1 + } + ] + } + ], + "tests": [ + { + "description": "modifyCollection prepareUnique violations are accessible", + "runOnRequirements": [ + { + "minServerVersion": "5.2" + } + ], + "operations": [ + { + "name": "createIndex", + "object": "collection0", + "arguments": { + "keys": { + "x": 1 + } + } + }, + { + "name": "modifyCollection", + "object": "database0", + "arguments": { + "collection": "test", + "index": { + "keyPattern": { + "x": 1 + }, + "prepareUnique": true + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 3, + "x": 1 + } + }, + "expectError": { + "errorCode": 11000 + } + }, + { + "name": "modifyCollection", + "object": "database0", + "arguments": { + "collection": "test", + "index": { + "keyPattern": { + "x": 1 + }, + "unique": true + } + }, + "expectError": { + "isClientError": false, + "errorCode": 359, + "errorResponse": { + "violations": [ + { + "ids": [ + 1, + 2 + ] + } + ] + } + } + } + ] + } + ] +} diff --git a/source/collection-management/tests/modifyCollection-errorResponse.yml b/source/collection-management/tests/modifyCollection-errorResponse.yml new file mode 100644 index 0000000000..e61a01211c --- /dev/null +++ b/source/collection-management/tests/modifyCollection-errorResponse.yml @@ -0,0 +1,59 @@ +description: "modifyCollection-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name collMod-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: &initialData + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 1 } + - { _id: 2, x: 1 } + +tests: + - description: "modifyCollection prepareUnique violations are accessible" + runOnRequirements: + - minServerVersion: "5.2" # SERVER-61158 + operations: + - name: createIndex + object: *collection0 + arguments: + keys: { x: 1 } + - name: modifyCollection + object: *database0 + arguments: + collection: *collection0Name + index: + keyPattern: { x: 1 } + prepareUnique: true + - name: insertOne + object: *collection0 + arguments: + document: { _id: 3, x: 1 } + expectError: + errorCode: 11000 # DuplicateKey + - name: modifyCollection + object: *database0 + arguments: + collection: *collection0Name + index: + keyPattern: { x: 1 } + unique: true + expectError: + isClientError: false + errorCode: 359 # CannotConvertIndexToUnique + errorResponse: + violations: + - { ids: [ 1, 2 ] } diff --git a/source/crud/tests/unified/aggregate-merge-errorResponse.json b/source/crud/tests/unified/aggregate-merge-errorResponse.json new file mode 100644 index 0000000000..bfb232480d --- /dev/null +++ b/source/crud/tests/unified/aggregate-merge-errorResponse.json @@ -0,0 +1,86 @@ +{ + "description": "aggregate-merge-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 1 + }, + { + "_id": 2, + "x": 1 + } + ] + } + ], + "tests": [ + { + "description": "aggregate $merge DuplicateKey error is accessible", + "runOnRequirements": [ + { + "minServerVersion": "5.1" + } + ], + "operations": [ + { + "name": "aggregate", + "object": "database0", + "arguments": { + "pipeline": [ + { + "$documents": [ + { + "_id": 2, + "x": 1 + } + ] + }, + { + "$merge": { + "into": "test", + "whenMatched": "fail" + } + } + ] + }, + "expectError": { + "errorCode": 11000, + "errorResponse": { + "keyPattern": { + "_id": 1 + }, + "keyValue": { + "_id": 2 + } + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/aggregate-merge-errorResponse.yml b/source/crud/tests/unified/aggregate-merge-errorResponse.yml new file mode 100644 index 0000000000..9f611dcf3a --- /dev/null +++ b/source/crud/tests/unified/aggregate-merge-errorResponse.yml @@ -0,0 +1,39 @@ +description: "aggregate-merge-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: &initialData + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 1 } + - { _id: 2, x: 1 } + +tests: + - description: "aggregate $merge DuplicateKey error is accessible" + runOnRequirements: + - minServerVersion: "5.1" # SERVER-59097 + operations: + - name: aggregate + object: *database0 + arguments: + pipeline: + - { $documents: [ { _id: 2, x: 1 } ] } + - { $merge: { into: *collection0Name, whenMatched: "fail" } } + expectError: + errorCode: 11000 # DuplicateKey + errorResponse: + keyPattern: { _id: 1 } + keyValue: { _id: 2 } diff --git a/source/crud/tests/unified/bulkWrite-errorResponse.json b/source/crud/tests/unified/bulkWrite-errorResponse.json new file mode 100644 index 0000000000..b520f24041 --- /dev/null +++ b/source/crud/tests/unified/bulkWrite-errorResponse.json @@ -0,0 +1,87 @@ +{ + "description": "bulkWrite-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "tests": [ + { + "description": "bulkWrite operations support errorResponse assertions", + "runOnRequirements": [ + { + "minServerVersion": "4.0.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.2.0", + "topologies": [ + "sharded" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 8 + } + } + } + }, + { + "name": "bulkWrite", + "object": "collection0", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 1 + } + } + } + ] + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/bulkWrite-errorResponse.yml b/source/crud/tests/unified/bulkWrite-errorResponse.yml new file mode 100644 index 0000000000..11cd0dfc1b --- /dev/null +++ b/source/crud/tests/unified/bulkWrite-errorResponse.yml @@ -0,0 +1,49 @@ +description: "bulkWrite-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +tests: + # This test intentionally executes only a single insert operation in the bulk + # write to make the error code and response assertions less ambiguous. That + # said, some drivers may still need to skip this test because the CRUD spec + # does not prescribe how drivers should formulate a BulkWriteException beyond + # collecting write and write concern errors. + - description: "bulkWrite operations support errorResponse assertions" + runOnRequirements: + - minServerVersion: "4.0.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.2.0" + topologies: [ sharded ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: &errorCode 8 # UnknownError + - name: bulkWrite + object: *collection0 + arguments: + requests: + - insertOne: + document: { _id: 1 } + expectError: + errorCode: *errorCode + errorResponse: + code: *errorCode diff --git a/source/crud/tests/unified/deleteOne-errorResponse.json b/source/crud/tests/unified/deleteOne-errorResponse.json new file mode 100644 index 0000000000..045b27cfcc --- /dev/null +++ b/source/crud/tests/unified/deleteOne-errorResponse.json @@ -0,0 +1,81 @@ +{ + "description": "deleteOne-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "tests": [ + { + "description": "delete operations support errorResponse assertions", + "runOnRequirements": [ + { + "minServerVersion": "4.0.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.2.0", + "topologies": [ + "sharded" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorCode": 8 + } + } + } + }, + { + "name": "deleteOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/deleteOne-errorResponse.yml b/source/crud/tests/unified/deleteOne-errorResponse.yml new file mode 100644 index 0000000000..2ce54f0b10 --- /dev/null +++ b/source/crud/tests/unified/deleteOne-errorResponse.yml @@ -0,0 +1,45 @@ +description: "deleteOne-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +tests: + # Some drivers may still need to skip this test because the CRUD spec does not + # prescribe how drivers should formulate a WriteException beyond collecting a + # write or write concern error. + - description: "delete operations support errorResponse assertions" + runOnRequirements: + - minServerVersion: "4.0.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.2.0" + topologies: [ sharded ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ delete ] + errorCode: &errorCode 8 # UnknownError + - name: deleteOne + object: *collection0 + arguments: + filter: { _id: 1 } + expectError: + errorCode: *errorCode + errorResponse: + code: *errorCode diff --git a/source/crud/tests/unified/findOneAndUpdate-errorResponse.json b/source/crud/tests/unified/findOneAndUpdate-errorResponse.json new file mode 100644 index 0000000000..5023a450f3 --- /dev/null +++ b/source/crud/tests/unified/findOneAndUpdate-errorResponse.json @@ -0,0 +1,132 @@ +{ + "description": "findOneAndUpdate-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": "foo" + } + ] + } + ], + "tests": [ + { + "description": "findOneAndUpdate DuplicateKey error is accessible", + "runOnRequirements": [ + { + "minServerVersion": "4.2" + } + ], + "operations": [ + { + "name": "createIndex", + "object": "collection0", + "arguments": { + "keys": { + "x": 1 + }, + "unique": true + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 2 + }, + "update": { + "$set": { + "x": "foo" + } + }, + "upsert": true + }, + "expectError": { + "errorCode": 11000, + "errorResponse": { + "keyPattern": { + "x": 1 + }, + "keyValue": { + "x": "foo" + } + } + } + } + ] + }, + { + "description": "findOneAndUpdate document validation errInfo is accessible", + "runOnRequirements": [ + { + "minServerVersion": "5.0" + } + ], + "operations": [ + { + "name": "modifyCollection", + "object": "database0", + "arguments": { + "collection": "test", + "validator": { + "x": { + "$type": "string" + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "x": 1 + } + } + }, + "expectError": { + "errorCode": 121, + "errorResponse": { + "errInfo": { + "failingDocumentId": 1, + "details": { + "$$type": "object" + } + } + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/findOneAndUpdate-errorResponse.yml b/source/crud/tests/unified/findOneAndUpdate-errorResponse.yml new file mode 100644 index 0000000000..8faed76809 --- /dev/null +++ b/source/crud/tests/unified/findOneAndUpdate-errorResponse.yml @@ -0,0 +1,69 @@ +description: "findOneAndUpdate-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: &initialData + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: "foo" } + +tests: + - description: "findOneAndUpdate DuplicateKey error is accessible" + runOnRequirements: + - minServerVersion: "4.2" # SERVER-37124 + operations: + - name: createIndex + object: *collection0 + arguments: + keys: { x: 1 } + unique: true + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 2 } + update: { $set: { x: "foo" } } + upsert: true + expectError: + errorCode: 11000 # DuplicateKey + errorResponse: + keyPattern: { x: 1 } + keyValue: { x: "foo" } + + - description: "findOneAndUpdate document validation errInfo is accessible" + runOnRequirements: + - minServerVersion: "5.0" + operations: + - name: modifyCollection + object: *database0 + arguments: + collection: *collection0Name + validator: + x: { $type: "string" } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $set: { x: 1 } } + expectError: + errorCode: 121 # DocumentValidationFailure + errorResponse: + # Avoid asserting the exact contents of errInfo as it may vary by + # server version. Likewise, this is why drivers do not model the + # document. The following is sufficient to test that validation + # details are accessible. See SERVER-20547 for more context. + errInfo: + failingDocumentId: 1 + details: { $$type: "object" } diff --git a/source/crud/tests/unified/insertOne-errorResponse.json b/source/crud/tests/unified/insertOne-errorResponse.json new file mode 100644 index 0000000000..98f8830f62 --- /dev/null +++ b/source/crud/tests/unified/insertOne-errorResponse.json @@ -0,0 +1,81 @@ +{ + "description": "insertOne-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "tests": [ + { + "description": "insert operations support errorResponse assertions", + "runOnRequirements": [ + { + "minServerVersion": "4.0.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.2.0", + "topologies": [ + "sharded" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 8 + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 1 + } + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/insertOne-errorResponse.yml b/source/crud/tests/unified/insertOne-errorResponse.yml new file mode 100644 index 0000000000..9dbb73ac4d --- /dev/null +++ b/source/crud/tests/unified/insertOne-errorResponse.yml @@ -0,0 +1,45 @@ +description: "insertOne-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +tests: + # Some drivers may still need to skip this test because the CRUD spec does not + # prescribe how drivers should formulate a WriteException beyond collecting a + # write or write concern error. + - description: "insert operations support errorResponse assertions" + runOnRequirements: + - minServerVersion: "4.0.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.2.0" + topologies: [ sharded ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: &errorCode 8 # UnknownError + - name: insertOne + object: *collection0 + arguments: + document: { _id: 1 } + expectError: + errorCode: *errorCode + errorResponse: + code: *errorCode diff --git a/source/crud/tests/unified/updateOne-errorResponse.json b/source/crud/tests/unified/updateOne-errorResponse.json new file mode 100644 index 0000000000..070d66dcfb --- /dev/null +++ b/source/crud/tests/unified/updateOne-errorResponse.json @@ -0,0 +1,86 @@ +{ + "description": "updateOne-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "tests": [ + { + "description": "update operations support errorResponse assertions", + "runOnRequirements": [ + { + "minServerVersion": "4.0.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.2.0", + "topologies": [ + "sharded" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "update" + ], + "errorCode": 8 + } + } + } + }, + { + "name": "updateOne", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$set": { + "x": 1 + } + } + }, + "expectError": { + "errorCode": 8, + "errorResponse": { + "code": 8 + } + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/updateOne-errorResponse.yml b/source/crud/tests/unified/updateOne-errorResponse.yml new file mode 100644 index 0000000000..4e657ce4a3 --- /dev/null +++ b/source/crud/tests/unified/updateOne-errorResponse.yml @@ -0,0 +1,46 @@ +description: "updateOne-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +tests: + # Some drivers may still need to skip this test because the CRUD spec does not + # prescribe how drivers should formulate a WriteException beyond collecting a + # write or write concern error. + - description: "update operations support errorResponse assertions" + runOnRequirements: + - minServerVersion: "4.0.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.2.0" + topologies: [ sharded ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ update ] + errorCode: &errorCode 8 # UnknownError + - name: updateOne + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $set: { x: 1 } } + expectError: + errorCode: *errorCode + errorResponse: + code: *errorCode diff --git a/source/unified-test-format/schema-1.12.json b/source/unified-test-format/schema-1.12.json new file mode 100644 index 0000000000..91acf4fb69 --- /dev/null +++ b/source/unified-test-format/schema-1.12.json @@ -0,0 +1,624 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": ["description", "schemaVersion", "tests"], + "properties": { + "description": { "type": "string" }, + "schemaVersion": { "$ref": "#/definitions/version" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/entity" } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/test" } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { "$ref": "#/definitions/version" }, + "minServerVersion": { "$ref": "#/definitions/version" }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": ["single", "replicaset", "sharded", "sharded-replicaset", "load-balanced"] + } + }, + "serverless": { + "type": "string", + "enum": ["require", "forbid", "allow"] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { "type": "boolean" }, + "csfle": { "type": "boolean" } + } + }, + + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { "type": "string" }, + "uriOptions": { "type": "object" }, + "useMultipleMongoses": { "type": "boolean" }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "storeEventsAsEntities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/storeEventsAsEntity" } + }, + "serverApi": { "$ref": "#/definitions/serverApi" }, + "observeSensitiveCommands": { "type": "boolean" } + } + }, + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": ["id", "clientEncryptionOpts"], + "properties": { + "id": { "type": "string" }, + "clientEncryptionOpts": { "$ref": "#/definitions/clientEncryptionOpts" } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client", "databaseName"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "databaseName": { "type": "string" }, + "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database", "collectionName"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "collectionName": { "type": "string" }, + "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "sessionOptions": { "type": "object" } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "bucketOptions": { "type": "object" } + } + }, + "thread": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { "type": "string" } + } + } + } + }, + + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": ["keyVaultClient", "keyVaultNamespace", "kmsProviders"], + "properties": { + "keyVaultClient": { "type": "string" }, + "keyVaultNamespace": { "type": "string" }, + "kmsProviders": { "$ref": "#/definitions/kmsProviders" } + } + }, + + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": ["$$placeholder"], + "properties": { + "$$placeholder": {} + } + } + ] + } + }, + "type": "object", + "additionalProperties": false, + "properties": { + "aws": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "secretAccessKey": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "sessionToken": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } + }, + "azure": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "clientId": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "clientSecret": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "identityPlatformEndpoint": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } + }, + "gcp": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "privateKey": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" }, + "endpoint": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } + }, + "kmip": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } + }, + "local": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" } + } + } + } + }, + + "storeEventsAsEntity": { + "type": "object", + "additionalProperties": false, + "required": ["id", "events"], + "properties": { + "id": { "type": "string" }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent" + ] + } + } + } + }, + + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": ["collectionName", "databaseName", "documents"], + "properties": { + "collectionName": { "type": "string" }, + "databaseName": { "type": "string" }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { "type": "object" } + } + } + }, + + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": ["client", "events"], + "properties": { + "client": { "type": "string" }, + "eventType": { + "type": "string", + "enum": ["command", "cmap", "sdam"] + }, + "events": { "type": "array" }, + "ignoreExtraEvents": { "type": "boolean" } + }, + "oneOf": [ + { + "required": ["eventType"], + "properties": { + "eventType": { "const": "command" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedCommandEvent" } + } + } + }, + { + "required": ["eventType"], + "properties": { + "eventType": { "const": "cmap" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedCmapEvent" } + } + } + }, + { + "required": ["eventType"], + "properties": { + "eventType": { "const": "sdam" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedSdamEvent" } + } + } + }, + { + "additionalProperties": false, + "properties": { + "client": { "type": "string" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedCommandEvent" } + }, + "ignoreExtraEvents": { "type": "boolean" } + } + } + ] + }, + + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { "type": "object" }, + "commandName": { "type": "string" }, + "databaseName": { "type": "string" }, + "hasServiceId": { "type": "boolean" }, + "hasServerConnectionId": { "type": "boolean" } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { "type": "object" }, + "commandName": { "type": "string" }, + "hasServiceId": { "type": "boolean" }, + "hasServerConnectionId": { "type": "boolean" } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { "type": "string" }, + "hasServiceId": { "type": "boolean" }, + "hasServerConnectionId": { "type": "boolean" } + } + } + } + }, + + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { "type": "boolean" }, + "interruptInUseConnections": { "type": "boolean" } + } + }, + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { "type": "string" } + } + }, + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { "type": "string" } + } + }, + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { "$ref": "#/definitions/serverDescription" }, + "newDescription": { "$ref": "#/definitions/serverDescription" } + } + } + } + }, + + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { "type": "object" }, + "readPreference": { "type": "object" }, + "writeConcern": { "type": "object" }, + "timeoutMS": { "type": "integer" } + } + }, + + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": ["version"], + "properties": { + "version": { "type": "string" }, + "strict": { "type": "boolean" }, + "deprecationErrors": { "type": "boolean" } + } + }, + + "operation": { + "type": "object", + "additionalProperties": false, + "required": ["name", "object"], + "properties": { + "name": { "type": "string" }, + "object": { "type": "string" }, + "arguments": { "type": "object" }, + "ignoreResultAndError": { "type": "boolean" }, + "expectError": { "$ref": "#/definitions/expectedError" }, + "expectResult": {}, + "saveResultAsEntity": { "type": "string" } + }, + "allOf": [ + { "not": { "required": ["expectError", "expectResult"] } }, + { "not": { "required": ["expectError", "saveResultAsEntity"] } }, + { "not": { "required": ["ignoreResultAndError", "expectResult"] } }, + { "not": { "required": ["ignoreResultAndError", "expectError"] } }, + { "not": { "required": ["ignoreResultAndError", "saveResultAsEntity"] } } + ] + }, + + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { "type": "boolean" }, + "isTimeoutError": { "type": "boolean" }, + "errorContains": { "type": "string" }, + "errorCode": { "type": "integer" }, + "errorCodeName": { "type": "string" }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "errorResponse": { "type": "object" }, + "expectResult": {} + } + }, + + "test": { + "type": "object", + "additionalProperties": false, + "required": ["description", "operations"], + "properties": { + "description": { "type": "string" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "skipReason": { "type": "string" }, + "operations": { + "type": "array", + "items": { "$ref": "#/definitions/operation" } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/expectedEventsForClient" } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + } + } + } + } +} diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile index 6d825fa899..c149481160 100644 --- a/source/unified-test-format/tests/Makefile +++ b/source/unified-test-format/tests/Makefile @@ -1,4 +1,4 @@ -SCHEMA=../schema-1.9.json +SCHEMA=../schema-1.12.json .PHONY: all invalid valid-fail valid-pass versioned-api load-balancers gridfs transactions crud collection-management sessions command-monitoring client-side-operations-timeout HAS_AJV diff --git a/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.json b/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.json new file mode 100644 index 0000000000..6eb66d9b0b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorResponse-type", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorResponse": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.yml new file mode 100644 index 0000000000..e63f6ce892 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorResponse-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorResponse-type" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorResponse: 0 diff --git a/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.json b/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.json new file mode 100644 index 0000000000..177b1baf56 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.json @@ -0,0 +1,70 @@ +{ + "description": "expectedError-errorResponse", + "schemaVersion": "1.12", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + } + ], + "tests": [ + { + "description": "Unsupported command", + "operations": [ + { + "name": "runCommand", + "object": "database0", + "arguments": { + "commandName": "unsupportedCommand", + "command": { + "unsupportedCommand": 1 + } + }, + "expectError": { + "errorResponse": { + "errmsg": { + "$$type": "string" + } + } + } + } + ] + }, + { + "description": "Unsupported query operator", + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "$unsupportedQueryOperator": 1 + } + }, + "expectError": { + "errorResponse": { + "errmsg": { + "$$type": "string" + } + } + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.yml b/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.yml new file mode 100644 index 0000000000..e10c25a1ed --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/expectedError-errorResponse.yml @@ -0,0 +1,39 @@ +description: "expectedError-errorResponse" + +schemaVersion: "1.12" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name test + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + +tests: + - description: "Unsupported command" + operations: + - name: runCommand + object: *database0 + arguments: + commandName: unsupportedCommand + command: { unsupportedCommand: 1 } + expectError: + # Avoid asserting the exact error since it may vary by server version + errorResponse: + errmsg: { $$type: "string" } + + - description: "Unsupported query operator" + operations: + - name: find + object: *collection0 + arguments: + filter: { $unsupportedQueryOperator: 1 } + expectError: + # Avoid asserting the exact error since it may vary by server version + errorResponse: + errmsg: { $$type: "string" } diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index bf6f457e3b..80450fe0d5 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1079,6 +1079,20 @@ The structure of this object is as follows: assert that the error does not contain any of the specified labels (e.g. using the ``hasErrorLabel`` method). +.. _expectedError_errorResponse: + +- ``errorResponse``: Optional document. A value corresponding to the expected + server response. The test runner MUST assert that the error includes a server + response that matches this value as a root-level document according to the + rules in `Evaluating Matches`_. + + Note that some drivers may not be able to evaluate ``errorResponse`` for write + commands (i.e. insert, update, delete) and bulk write operations. For example, + a BulkWriteException is derived from potentially multiple server responses and + may not provide direct access to a single response. Tests SHOULD avoid using + ``errorResponse`` for such operations if possible; otherwise, affected drivers + SHOULD skip such tests if necessary. + .. _expectedError_expectResult: - ``expectResult``: Optional mixed type. This field follows the same rules as @@ -2900,7 +2914,8 @@ root-level documents include, but are not limited to: - ``command`` for `CommandStartedEvent `_ - ``reply`` for `CommandSucceededEvent `_ - `expectResult`_ for ``findOneAndUpdate`` `Collection Operations`_ -- `expectResult`_ for `iterateUntilDocumentOrError`_. +- `expectResult`_ for `iterateUntilDocumentOrError`_ +- `expectedError_errorResponse`_ - each array element in `expectResult`_ for `find`_ or `collection_aggregate`_ `Collection Operations`_ @@ -3749,6 +3764,8 @@ Changelog .. Please note schema version bumps in changelog entries where applicable. +:2022-10-14: **Schema version 1.12.** + Add ``errorResponse`` to ``expectedError``. :2022-10-05: Remove spec front matter, add "Current Schema Version" field, and reformat changelog. Add comment to remind editors to note schema version bumps in changelog updates (where applicable).