From e42036f7e00d8f18fa67ac5f53b254e72cf9ae88 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Mon, 16 Jun 2025 17:08:08 -0700 Subject: [PATCH 1/7] pull over bulkwrite changes --- src/libmongoc/src/mongoc/mongoc-bulkwrite.c | 14 +- src/libmongoc/src/mongoc/mongoc-crypt.c | 25 ++-- .../limits-encryptedFields.json | 14 ++ .../limits-qe-doc.json | 3 + .../test-mongoc-client-side-encryption.c | 129 +++++++++++++++--- src/libmongoc/tests/test-mongoc-crud.c | 8 -- 6 files changed, 148 insertions(+), 45 deletions(-) create mode 100644 src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json create mode 100644 src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json diff --git a/src/libmongoc/src/mongoc/mongoc-bulkwrite.c b/src/libmongoc/src/mongoc/mongoc-bulkwrite.c index 9f16fadcdbc..5012dff29a7 100644 --- a/src/libmongoc/src/mongoc/mongoc-bulkwrite.c +++ b/src/libmongoc/src/mongoc/mongoc-bulkwrite.c @@ -1561,15 +1561,6 @@ mongoc_bulkwrite_execute (mongoc_bulkwrite_t *self, const mongoc_bulkwriteopts_t goto fail; } - if (_mongoc_cse_is_enabled (self->client)) { - _mongoc_set_error (&error, - MONGOC_ERROR_COMMAND, - MONGOC_ERROR_COMMAND_INVALID_ARG, - "bulkWrite does not currently support automatic encryption"); - _bulkwriteexception_set_error (ret.exc, &error); - goto fail; - } - const mongoc_ss_log_context_t ss_log_context = { .operation = "bulkWrite", .has_operation_id = true, .operation_id = self->operation_id}; @@ -1695,8 +1686,11 @@ mongoc_bulkwrite_execute (mongoc_bulkwrite_t *self, const mongoc_bulkwriteopts_t } } - int32_t maxWriteBatchSize = mongoc_server_stream_max_write_batch_size (ss); + const int32_t maxWriteBatchSize = mongoc_server_stream_max_write_batch_size (ss); int32_t maxMessageSizeBytes = mongoc_server_stream_max_msg_size (ss); + if (_mongoc_cse_is_enabled (self->client)) { + maxMessageSizeBytes = MONGOC_REDUCED_MAX_MSG_SIZE_FOR_FLE; + } // `ops_doc_offset` is an offset into the `ops` document sequence. Counts the number of documents sent. size_t ops_doc_offset = 0; // `ops_byte_offset` is an offset into the `ops` document sequence. Counts the number of bytes sent. diff --git a/src/libmongoc/src/mongoc/mongoc-crypt.c b/src/libmongoc/src/mongoc/mongoc-crypt.c index abce3878fc1..870c9f8d7fe 100644 --- a/src/libmongoc/src/mongoc/mongoc-crypt.c +++ b/src/libmongoc/src/mongoc/mongoc-crypt.c @@ -310,9 +310,9 @@ _state_machine_destroy (_state_machine_t *state_machine) bson_free (state_machine); } -/* State handler MONGOCRYPT_CTX_NEED_MONGO_COLLINFO */ +/* State handler MONGOCRYPT_CTX_NEED_MONGO_COLLINFO{_WITH_DB} */ static bool -_state_need_mongo_collinfo (_state_machine_t *state_machine, bson_error_t *error) +_state_need_mongo_collinfo (_state_machine_t *state_machine, bool with_db, bson_error_t *error) { mongoc_database_t *db = NULL; mongoc_cursor_t *cursor = NULL; @@ -336,7 +336,13 @@ _state_need_mongo_collinfo (_state_machine_t *state_machine, bson_error_t *error } bson_append_document (&opts, "filter", -1, &filter_bson); - db = mongoc_client_get_database (state_machine->collinfo_client, state_machine->db_name); + const char *db_name = with_db ? mongocrypt_ctx_mongo_db (state_machine->ctx) : state_machine->db_name; + if (!db_name) { + _ctx_check_error (state_machine->ctx, error, true); + goto fail; + } + db = mongoc_client_get_database (state_machine->collinfo_client, db_name); + cursor = mongoc_database_find_collections_with_opts (db, &opts); if (mongoc_cursor_error (cursor, error)) { goto fail; @@ -1078,7 +1084,7 @@ _state_machine_run (_state_machine_t *state_machine, bson_t *result, bson_error_ _ctx_check_error (state_machine->ctx, error, true); goto fail; case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: - if (!_state_need_mongo_collinfo (state_machine, error)) { + if (!_state_need_mongo_collinfo (state_machine, false, error)) { goto fail; } break; @@ -1112,12 +1118,9 @@ _state_machine_run (_state_machine_t *state_machine, bson_t *result, bson_error_ goto success; break; case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB: - _mongoc_set_error (error, - MONGOC_ERROR_CLIENT_SIDE_ENCRYPTION, - MONGOC_ERROR_CLIENT_INVALID_ENCRYPTION_STATE, - "MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB is " - "unimplemented"); - goto fail; + if (!_state_need_mongo_collinfo (state_machine, true, error)) { + goto fail; + } break; } } @@ -1400,6 +1403,8 @@ _mongoc_crypt_new (const bson_t *kms_providers, crypt->kmsid_to_tlsopts = mcd_mapof_kmsid_to_tlsopts_new (); crypt->handle = mongocrypt_new (); mongocrypt_setopt_retry_kms (crypt->handle, true); + mongocrypt_setopt_use_need_mongo_collinfo_with_db_state (crypt->handle); + if (!mongocrypt_setopt_enable_multiple_collinfo (crypt->handle)) { _crypt_check_error (crypt->handle, error, true); goto fail; diff --git a/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json b/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json new file mode 100644 index 00000000000..c52a0271e16 --- /dev/null +++ b/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json @@ -0,0 +1,14 @@ +{ + "fields": [ + { + "keyId": { + "$binary": { + "base64": "LOCALAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "path": "foo", + "bsonType": "string" + } + ] +} \ No newline at end of file diff --git a/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json b/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json new file mode 100644 index 00000000000..71efbf40682 --- /dev/null +++ b/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json @@ -0,0 +1,3 @@ +{ + "foo": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} \ No newline at end of file diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index 5d1f24ff25d..aea7471e7df 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -14,7 +14,11 @@ * limitations under the License. */ +#include "TestSuite.h" +#include "bson/bson.h" #include "json-test.h" +#include + #include "test-libmongoc.h" #include @@ -347,7 +351,7 @@ _command_started (const mongoc_apm_command_started_t *event) /* Prose Test 4: BSON Size Limits and Batch Splitting */ static void -test_bson_size_limits_and_batch_splitting (void *unused) +test_bson_size_limits_and_batch_splitting (bool with_qe) { /* Expect an insert of two documents over 2MiB to split into two inserts but * still succeed. */ @@ -370,8 +374,6 @@ test_bson_size_limits_and_batch_splitting (void *unused) const int exceeds_2mib_after_encryption = size_2mib - 2000; const int exceeds_16mib_after_encryption = size_16mib - 2000; - BSON_UNUSED (unused); - /* Do the test setup. */ /* Drop and create db.coll configured with limits-schema.json */ @@ -423,6 +425,7 @@ test_bson_size_limits_and_batch_splitting (void *unused) coll = mongoc_client_get_collection (client, "db", "coll"); /* End of setup */ + /* Case 1 */ /* Insert { "_id": "over_2mib_under_16mib", "unencrypted": } */ docs[0] = BCON_NEW ("_id", "over_2mib_under_16mib"); @@ -441,6 +444,23 @@ test_bson_size_limits_and_batch_splitting (void *unused) ASSERT_OR_PRINT (mongoc_collection_insert_one (coll, docs[0], NULL /* opts */, NULL /* reply */, &error), error); bson_destroy (docs[0]); + /* Check that inserting close to, but not exceeding, 16MiB, passes */ + docs[0] = bson_new (); + bson_append_utf8 (docs[0], "_id", -1, "under_16mib", -1); + bson_append_utf8 (docs[0], "unencrypted", -1, as, exceeds_16mib_after_encryption); + ASSERT_OR_PRINT (mongoc_collection_insert_one (coll, docs[0], NULL /* opts */, NULL /* reply */, &error), error); + bson_destroy (docs[0]); + + /* but.. exceeding 16 MiB fails */ + docs[0] = get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-doc.json"); + bson_append_utf8 (docs[0], "_id", -1, "under_16mib", -1); + bson_append_utf8 (docs[0], "unencrypted", -1, as, exceeds_16mib_after_encryption); + BSON_ASSERT (!mongoc_collection_insert_one (coll, docs[0], NULL /* opts */, NULL /* reply */, &error)); + ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_SERVER, 2, "too large"); + bson_destroy (docs[0]); + + /* Case 2: collection bulkWrite */ + /* Insert two documents that each exceed 2MiB but no encryption occurs. * Expect the bulk write to succeed and run as two separate inserts. */ @@ -474,20 +494,73 @@ test_bson_size_limits_and_batch_splitting (void *unused) bson_destroy (docs[0]); bson_destroy (docs[1]); - /* Check that inserting close to, but not exceeding, 16MiB, passes */ - docs[0] = bson_new (); - bson_append_utf8 (docs[0], "_id", -1, "under_16mib", -1); - bson_append_utf8 (docs[0], "unencrypted", -1, as, exceeds_16mib_after_encryption); - ASSERT_OR_PRINT (mongoc_collection_insert_one (coll, docs[0], NULL /* opts */, NULL /* reply */, &error), error); - bson_destroy (docs[0]); + if (with_qe) { + /* Case 3: client bulkWrite */ + mongoc_bulkwriteopts_t *bw_opts = mongoc_bulkwriteopts_new (); + mongoc_bulkwriteopts_set_verboseresults (bw_opts, true); - /* but.. exceeding 16 MiB fails */ - docs[0] = get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-doc.json"); - bson_append_utf8 (docs[0], "_id", -1, "under_16mib", -1); - bson_append_utf8 (docs[0], "unencrypted", -1, as, exceeds_16mib_after_encryption); - BSON_ASSERT (!mongoc_collection_insert_one (coll, docs[0], NULL /* opts */, NULL /* reply */, &error)); - ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_SERVER, 2, "too large"); - bson_destroy (docs[0]); + bson_t *corpus_encryptedFields = + get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json"); + bson_t *coll_opts = BCON_NEW ("encryptedFields", BCON_DOCUMENT (corpus_encryptedFields)); + mongoc_database_t *db = mongoc_client_get_database (client, "db"); + (void) mongoc_collection_drop (coll, NULL); + // Create a newly named collection to avoid cached previous JSON Schema. + mongoc_collection_t *coll2 = mongoc_database_create_collection (db, "coll2", coll_opts, &error); + ASSERT_OR_PRINT (coll2, error); + mongoc_collection_destroy(coll2); + + /* Insert two documents that each exceed 2MiB but no encryption occurs. + * Expect two separate bulkWrite commands. + */ + docs[0] = BCON_NEW ("_id", "over_2mib_3"); + bson_append_utf8 (docs[0], "unencrypted", -1, as, size_2mib - 1500); + docs[1] = BCON_NEW ("_id", "over_2mib_4"); + bson_append_utf8 (docs[1], "unencrypted", -1, as, size_2mib - 1500); + + ctx.num_inserts = 0; + mongoc_bulkwrite_t *bw = mongoc_client_bulkwrite_new (client); + ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[0], NULL, &error), error); + ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[1], NULL, &error), error); + + mongoc_bulkwritereturn_t bwr = mongoc_bulkwrite_execute (bw, bw_opts); + ASSERT_NO_BULKWRITEEXCEPTION (bwr); + ASSERT_CMPINT (ctx.num_inserts, ==, 2); + bson_destroy (docs[0]); + bson_destroy (docs[1]); + mongoc_bulkwrite_destroy (bw); + mongoc_bulkwriteresult_destroy (bwr.res); + mongoc_bulkwriteexception_destroy (bwr.exc); + + /* Insert two documents that each exceed 2MiB after encryption occurs. Expect + * the bulk write to succeed and run as two separate inserts. + */ + + + docs[0] = get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json"); + bson_append_utf8 (docs[0], "_id", -1, "encryption_exceeds_2mib_3", -1); + bson_append_utf8 (docs[0], "foo", -1, as, exceeds_2mib_after_encryption - 1500); + docs[1] = get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json"); + bson_append_utf8 (docs[1], "_id", -1, "encryption_exceeds_2mib_4", -1); + bson_append_utf8 (docs[1], "foo", -1, as, exceeds_2mib_after_encryption - 1500); + + ctx.num_inserts = 0; + bw = mongoc_client_bulkwrite_new (client); + ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[0], NULL, &error), error); + ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[1], NULL, &error), error); + + bwr = mongoc_bulkwrite_execute (bw, bw_opts); + ASSERT_NO_BULKWRITEEXCEPTION (bwr); + ASSERT_CMPINT (ctx.num_inserts, ==, 2); + bson_destroy (docs[0]); + bson_destroy (docs[1]); + mongoc_bulkwrite_destroy (bw); + mongoc_bulkwriteresult_destroy (bwr.res); + mongoc_bulkwriteexception_destroy (bwr.exc); + mongoc_bulkwriteopts_destroy (bw_opts); + bson_destroy (corpus_encryptedFields); + bson_destroy (coll_opts); + mongoc_database_destroy (db); + } bson_free (as); bson_destroy (kms_providers); @@ -501,6 +574,20 @@ test_bson_size_limits_and_batch_splitting (void *unused) mongoc_auto_encryption_opts_destroy (opts); } +static void +test_bson_size_limits_and_batch_splitting_no_qe (void *unused) +{ + BSON_UNUSED (unused); + test_bson_size_limits_and_batch_splitting (false); +} + +static void +test_bson_size_limits_and_batch_splitting_qe (void *unused) +{ + BSON_UNUSED (unused); + test_bson_size_limits_and_batch_splitting (true); +} + typedef struct { bson_t *last_cmd; } _datakey_and_double_encryption_ctx_t; @@ -6902,11 +6989,19 @@ test_client_side_encryption_install (TestSuite *suite) test_framework_skip_if_no_auth /* requires auth for error check */); TestSuite_AddFull (suite, "/client_side_encryption/bson_size_limits_and_batch_splitting", - test_bson_size_limits_and_batch_splitting, + test_bson_size_limits_and_batch_splitting_no_qe, NULL, NULL, test_framework_skip_if_no_client_side_encryption, TestSuite_CheckLive); + TestSuite_AddFull (suite, + "/client_side_encryption/bson_size_limits_and_batch_splitting_qe", + test_bson_size_limits_and_batch_splitting_qe, + NULL, + NULL, + test_framework_skip_if_no_client_side_encryption, + test_framework_skip_if_max_wire_version_less_than_25, + test_framework_skip_if_single); TestSuite_AddFull (suite, "/client_side_encryption/views_are_prohibited", test_views_are_prohibited, diff --git a/src/libmongoc/tests/test-mongoc-crud.c b/src/libmongoc/tests/test-mongoc-crud.c index f6d3fee4343..43aaad48822 100644 --- a/src/libmongoc/tests/test-mongoc-crud.c +++ b/src/libmongoc/tests/test-mongoc-crud.c @@ -1518,14 +1518,6 @@ test_crud_install (TestSuite *suite) test_framework_skip_if_max_wire_version_less_than_25 // require server 8.0 ); - TestSuite_AddFull (suite, - "/crud/prose_test_13", - prose_test_13, - NULL /* dtor */, - NULL /* ctx */, - test_framework_skip_if_max_wire_version_less_than_25, // require server 8.0 - test_framework_skip_if_no_client_side_encryption); - TestSuite_AddFull (suite, "/crud/prose_test_15", prose_test_15, From 7513b23ff6d252802530791cb01c921d80f8a2ab Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:12:39 -0700 Subject: [PATCH 2/7] count bulkwrite commands in test --- src/libmongoc/tests/test-mongoc-client-side-encryption.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index aea7471e7df..2f68f8ce3c7 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -344,7 +344,8 @@ _command_started (const mongoc_apm_command_started_t *event) limits_apm_ctx_t *ctx; ctx = (limits_apm_ctx_t *) mongoc_apm_command_started_get_context (event); - if (0 == strcmp ("insert", mongoc_apm_command_started_get_command_name (event))) { + const char *cmd_name = mongoc_apm_command_started_get_command_name(event); + if (0 == strcmp ("insert", cmd_name) || 0 == strcmp("bulkWrite", cmd_name)) { ctx->num_inserts++; } } From 7ea616249d2ae367eb792a99885e1b77d855dc89 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:15:09 -0700 Subject: [PATCH 3/7] sync bulkwrite test --- .../crud/unified/client-bulkWrite-qe.json | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json diff --git a/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json b/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json new file mode 100644 index 00000000000..dad3f3950a0 --- /dev/null +++ b/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json @@ -0,0 +1,296 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + }, + { + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ] + } + ] +} From 4eb2705922ddb221ca6d9edbcf25c2a4c95b0fe8 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Tue, 17 Jun 2025 17:26:54 -0700 Subject: [PATCH 4/7] delete prose test 13 --- src/libmongoc/tests/test-mongoc-crud.c | 51 -------------------------- 1 file changed, 51 deletions(-) diff --git a/src/libmongoc/tests/test-mongoc-crud.c b/src/libmongoc/tests/test-mongoc-crud.c index 43aaad48822..5d5ea7a0010 100644 --- a/src/libmongoc/tests/test-mongoc-crud.c +++ b/src/libmongoc/tests/test-mongoc-crud.c @@ -1258,57 +1258,6 @@ prose_test_12 (void *ctx) mongoc_client_destroy (client); } -static void -prose_test_13 (void *ctx) -{ - /* - 13. `MongoClient.bulkWrite` errors if configured with automatic encryption. - */ - mongoc_client_t *client; - BSON_UNUSED (ctx); - bool ok; - bson_error_t error; - - client = test_framework_new_default_client (); - mongoc_auto_encryption_opts_t *aeo = mongoc_auto_encryption_opts_new (); - mongoc_auto_encryption_opts_set_keyvault_namespace (aeo, "db", "coll"); - mongoc_auto_encryption_opts_set_kms_providers ( - aeo, tmp_bson (BSON_STR ({"aws" : {"accessKeyId" : "foo", "secretAccessKey" : "bar"}}))); - ok = mongoc_client_enable_auto_encryption (client, aeo, &error); - ASSERT_OR_PRINT (ok, error); - - // Try to to a bulk write. - { - mongoc_bulkwrite_t *bw = mongoc_client_bulkwrite_new (client); - - // Create bulk write. - { - ok = mongoc_bulkwrite_append_insertone (bw, "db.coll", tmp_bson ("{'a': 'b'}"), NULL, &error); - ASSERT_OR_PRINT (ok, error); - } - - // Execute. - { - mongoc_bulkwritereturn_t bwr = mongoc_bulkwrite_execute (bw, NULL); - ASSERT (!bwr.res); // No result due to no successful writes. - ASSERT (bwr.exc); - if (!mongoc_bulkwriteexception_error (bwr.exc, &error)) { - test_error ("Expected top-level error but got:\n%s", test_bulkwriteexception_str (bwr.exc)); - } - ASSERT_ERROR_CONTAINS (error, - MONGOC_ERROR_COMMAND, - MONGOC_ERROR_COMMAND_INVALID_ARG, - "bulkWrite does not currently support automatic encryption"); - mongoc_bulkwriteresult_destroy (bwr.res); - mongoc_bulkwriteexception_destroy (bwr.exc); - } - mongoc_bulkwrite_destroy (bw); - } - - mongoc_auto_encryption_opts_destroy (aeo); - mongoc_client_destroy (client); -} - static void prose_test_15 (void *ctx) { From d5a7c2c356b8284b694569530c14e1efa623b46f Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 16 Jul 2025 22:16:32 -0700 Subject: [PATCH 5/7] Apply suggestions from code review Co-authored-by: Kevin Albertson --- src/libmongoc/tests/test-mongoc-client-side-encryption.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index 2f68f8ce3c7..2aab2c5f094 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -533,7 +533,7 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) mongoc_bulkwriteexception_destroy (bwr.exc); /* Insert two documents that each exceed 2MiB after encryption occurs. Expect - * the bulk write to succeed and run as two separate inserts. + * the bulk write to succeed and run as two separate bulkWrite commands. */ From c320e6c138d1451b7fd29893b8aef60c814c667a Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 16 Jul 2025 22:55:49 -0700 Subject: [PATCH 6/7] track bulkwrite separately --- .../test-mongoc-client-side-encryption.c | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index 2aab2c5f094..098745489b3 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -14,28 +14,23 @@ * limitations under the License. */ -#include "TestSuite.h" -#include "bson/bson.h" -#include "json-test.h" -#include - -#include "test-libmongoc.h" - +#include #include -#include +#include +#include /* _mongoc_host_list_from_string_with_err */ -#include #include +#include /* MONGOC_SERVER_ERR_NS_NOT_FOUND */ -#include - #include +#include +#include #include -#include + #include static void @@ -336,6 +331,7 @@ _make_kms_masterkey (char const *provider) typedef struct { int num_inserts; + int num_bulk_writes; } limits_apm_ctx_t; static void @@ -345,9 +341,12 @@ _command_started (const mongoc_apm_command_started_t *event) ctx = (limits_apm_ctx_t *) mongoc_apm_command_started_get_context (event); const char *cmd_name = mongoc_apm_command_started_get_command_name(event); - if (0 == strcmp ("insert", cmd_name) || 0 == strcmp("bulkWrite", cmd_name)) { + if (0 == strcmp ("insert", cmd_name)) { ctx->num_inserts++; } + if (0 == strcmp("bulkWrite", cmd_name)) { + ctx->num_bulk_writes++; + } } /* Prose Test 4: BSON Size Limits and Batch Splitting */ @@ -426,7 +425,6 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) coll = mongoc_client_get_collection (client, "db", "coll"); /* End of setup */ - /* Case 1 */ /* Insert { "_id": "over_2mib_under_16mib", "unencrypted": } */ docs[0] = BCON_NEW ("_id", "over_2mib_under_16mib"); @@ -460,8 +458,6 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) ASSERT_ERROR_CONTAINS (error, MONGOC_ERROR_SERVER, 2, "too large"); bson_destroy (docs[0]); - /* Case 2: collection bulkWrite */ - /* Insert two documents that each exceed 2MiB but no encryption occurs. * Expect the bulk write to succeed and run as two separate inserts. */ @@ -523,9 +519,10 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[0], NULL, &error), error); ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[1], NULL, &error), error); + ctx.num_bulk_writes = 0; mongoc_bulkwritereturn_t bwr = mongoc_bulkwrite_execute (bw, bw_opts); ASSERT_NO_BULKWRITEEXCEPTION (bwr); - ASSERT_CMPINT (ctx.num_inserts, ==, 2); + ASSERT_CMPINT (ctx.num_bulk_writes, ==, 2); bson_destroy (docs[0]); bson_destroy (docs[1]); mongoc_bulkwrite_destroy (bw); @@ -535,8 +532,6 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) /* Insert two documents that each exceed 2MiB after encryption occurs. Expect * the bulk write to succeed and run as two separate bulkWrite commands. */ - - docs[0] = get_bson_from_json_file ("./src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json"); bson_append_utf8 (docs[0], "_id", -1, "encryption_exceeds_2mib_3", -1); bson_append_utf8 (docs[0], "foo", -1, as, exceeds_2mib_after_encryption - 1500); @@ -549,9 +544,10 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[0], NULL, &error), error); ASSERT_OR_PRINT (mongoc_bulkwrite_append_insertone (bw, "db.coll2", docs[1], NULL, &error), error); + ctx.num_bulk_writes = 0; bwr = mongoc_bulkwrite_execute (bw, bw_opts); ASSERT_NO_BULKWRITEEXCEPTION (bwr); - ASSERT_CMPINT (ctx.num_inserts, ==, 2); + ASSERT_CMPINT (ctx.num_bulk_writes, ==, 2); bson_destroy (docs[0]); bson_destroy (docs[1]); mongoc_bulkwrite_destroy (bw); From 831709d479c88cc86c674555f45decaf19ca57fa Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 16 Jul 2025 23:02:43 -0700 Subject: [PATCH 7/7] format --- src/libmongoc/tests/test-mongoc-client-side-encryption.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index 098745489b3..90d358ce549 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -340,11 +340,11 @@ _command_started (const mongoc_apm_command_started_t *event) limits_apm_ctx_t *ctx; ctx = (limits_apm_ctx_t *) mongoc_apm_command_started_get_context (event); - const char *cmd_name = mongoc_apm_command_started_get_command_name(event); + const char *cmd_name = mongoc_apm_command_started_get_command_name (event); if (0 == strcmp ("insert", cmd_name)) { ctx->num_inserts++; } - if (0 == strcmp("bulkWrite", cmd_name)) { + if (0 == strcmp ("bulkWrite", cmd_name)) { ctx->num_bulk_writes++; } } @@ -504,7 +504,7 @@ test_bson_size_limits_and_batch_splitting (bool with_qe) // Create a newly named collection to avoid cached previous JSON Schema. mongoc_collection_t *coll2 = mongoc_database_create_collection (db, "coll2", coll_opts, &error); ASSERT_OR_PRINT (coll2, error); - mongoc_collection_destroy(coll2); + mongoc_collection_destroy (coll2); /* Insert two documents that each exceed 2MiB but no encryption occurs. * Expect two separate bulkWrite commands.