From 1ccf38ed9b604cc00a2beececf2ff11cc549ec20 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 19 Nov 2024 13:26:20 -0500 Subject: [PATCH 01/29] temp --- .github/workflows/encryption-tests.yml | 74 ++++++++++++++++++++++++++ test/encryption/encryption.test.js | 11 ++++ 2 files changed, 85 insertions(+) create mode 100644 .github/workflows/encryption-tests.yml create mode 100644 test/encryption/encryption.test.js diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml new file mode 100644 index 00000000000..87ffa37fcc4 --- /dev/null +++ b/.github/workflows/encryption-tests.yml @@ -0,0 +1,74 @@ +on: + push: + branches: ['main'] + workflow_dispatch: {} + +permissions: + contents: write + pull-requests: write + id-token: write + +name: encryption-tests + +jobs: + ssdlc: + permissions: + # required for all workflows + security-events: write + id-token: write + contents: write + environment: release + runs-on: ubuntu-latest + steps: + - name: Install mongodb-client-encryption + uses: mongodb-client-encryption + + - name: Set-up cluster + + + + - name: Run tests + uses: mongodb-labs/drivers-github-tools/node/get_version_info@v2 + with: + npm_package_name: mongodb + + - name: actions/compress_sign_and_upload + uses: mongodb-labs/drivers-github-tools/node/sign_node_package@v2 + with: + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: us-east-1 + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + npm_package_name: mongodb + dry_run: ${{ needs.release_please.outputs.release_created == '' }} + + - name: Copy sbom file to release assets + shell: bash + if: ${{ '' == '' }} + run: cp sbom.json ${{ env.S3_ASSETS }}/sbom.json + + # only used for mongodb-client-encryption + - name: Augment SBOM and copy to release assets + if: ${{ '' != '' }} + uses: mongodb-labs/drivers-github-tools/sbom@v2 + with: + silk_asset_group: '' + sbom_file_name: sbom.json + + - name: Generate authorized pub report + uses: mongodb-labs/drivers-github-tools/full-report@v2 + with: + release_version: ${{ env.package_version }} + product_name: mongodb + sarif_report_target_ref: 'main' + third_party_dependency_tool: n/a + dist_filenames: artifacts/* + token: ${{ github.token }} + sbom_file_name: sbom.json + evergreen_project: mongo-node-driver-next + evergreen_commit: ${{ env.commit }} + + - uses: mongodb-labs/drivers-github-tools/upload-s3-assets@v2 + with: + version: ${{ env.package_version }} + product_name: mongodb + dry_run: ${{ needs.release_please.outputs.release_created == '' }} \ No newline at end of file diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js new file mode 100644 index 00000000000..cb9d3be2d66 --- /dev/null +++ b/test/encryption/encryption.test.js @@ -0,0 +1,11 @@ +'use strict'; + +const assert = require('assert'); +const EJSON = require('bson').EJSON; + +describe('setup check', () => { + it('environment variables are set', async function() { + const { local } = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}'); + assert.ok(local); + }); +}); From aeda36be987adfaf29fa6ce9c5d3ca03aea13e11 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 20 Nov 2024 02:14:22 -0500 Subject: [PATCH 02/29] temp --- .github/workflows/encryption-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 87ffa37fcc4..82214877e62 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -17,14 +17,15 @@ jobs: security-events: write id-token: write contents: write - environment: release runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - name: Set-up cluster - + - shell: bash + + - name: Run tests From e5d8cadbf86e924f8db30317cddd659366dad390 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 20 Nov 2024 16:24:58 -0500 Subject: [PATCH 03/29] temp 2 --- .github/scripts/run-kms-servers.sh | 11 +++ .github/scripts/run-orchestration.sh | 36 +++++++++ .github/workflows/encryption-tests.yml | 103 ++++++++++++++----------- 3 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 .github/scripts/run-kms-servers.sh create mode 100644 .github/scripts/run-orchestration.sh diff --git a/.github/scripts/run-kms-servers.sh b/.github/scripts/run-kms-servers.sh new file mode 100644 index 00000000000..988b07bfef1 --- /dev/null +++ b/.github/scripts/run-kms-servers.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +cd ${DRIVERS_TOOLS}/.evergreen/csfle +. ./prepare-kmsvenv.sh + +echo "$PYTHON_EXEC" + +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert & \ No newline at end of file diff --git a/.github/scripts/run-orchestration.sh b/.github/scripts/run-orchestration.sh new file mode 100644 index 00000000000..3708c7bc33d --- /dev/null +++ b/.github/scripts/run-orchestration.sh @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +# This is a thin wrapper around drivers-tools run orchestration meant to print each of the configuration settings we make use of +# Additionally it ensures the downloaded binaries are in the PATH for the script to find (namely, the legacy shell for server set up) + +export MONGODB_VERSION=${VERSION} +echo "MONGODB_VERSION=${VERSION}" + +export TOPOLOGY=${TOPOLOGY} +echo "TOPOLOGY=${TOPOLOGY}" + +export AUTH=${AUTH} +echo "AUTH=${AUTH}" + +export SSL=${SSL} +echo "SSL=${SSL}" + +export ORCHESTRATION_FILE=${ORCHESTRATION_FILE} +echo "ORCHESTRATION_FILE=${ORCHESTRATION_FILE}" + +export REQUIRE_API_VERSION=${REQUIRE_API_VERSION} +echo "REQUIRE_API_VERSION=${REQUIRE_API_VERSION}" + +export LOAD_BALANCER=${LOAD_BALANCER} +echo "LOAD_BALANCER=${LOAD_BALANCER}" + +export COMPRESSOR=${COMPRESSOR} +echo "COMPRESSOR=${COMPRESSOR}" + +export PATH="$MONGODB_BINARIES:$PATH" +echo "MONGODB_BINARIES=${MONGODB_BINARIES}" + +export SKIP_LEGACY_SHELL="true" +echo "SKIP_LEGACY_SHELL=${SKIP_LEGACY_SHELL}" + +bash "${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh" \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 82214877e62..e8afec2d560 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -11,7 +11,7 @@ permissions: name: encryption-tests jobs: - ssdlc: + set-up: permissions: # required for all workflows security-events: write @@ -24,52 +24,61 @@ jobs: - name: Set-up cluster - shell: bash - - - - - name: Run tests - uses: mongodb-labs/drivers-github-tools/node/get_version_info@v2 - with: - npm_package_name: mongodb - - - name: actions/compress_sign_and_upload - uses: mongodb-labs/drivers-github-tools/node/sign_node_package@v2 - with: - aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} - aws_region_name: us-east-1 - aws_secret_id: ${{ secrets.AWS_SECRET_ID }} - npm_package_name: mongodb - dry_run: ${{ needs.release_please.outputs.release_created == '' }} - - - name: Copy sbom file to release assets - shell: bash - if: ${{ '' == '' }} - run: cp sbom.json ${{ env.S3_ASSETS }}/sbom.json - - # only used for mongodb-client-encryption - - name: Augment SBOM and copy to release assets - if: ${{ '' != '' }} - uses: mongodb-labs/drivers-github-tools/sbom@v2 - with: - silk_asset_group: '' - sbom_file_name: sbom.json - - - name: Generate authorized pub report - uses: mongodb-labs/drivers-github-tools/full-report@v2 + run-tests: + needs: + - set-up + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + node: [20] + os: [ubuntu-latest] + mongodb: [8.0.0] + #clone drivers-evergreen-tools (will delete itself once the run is over) + git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git /data/mci/ead9f9a67ef900991e09bd9589dfee0e/drivers-tools + #set path + #calling run orchestration sets up a server on your computer + # it outputs relevants uri in expansions.yml + # put uri into an environment variable + - name: set-up-orchestration + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: ${VERSION} + TOPOLOGY: ${TOPOLOGY} + AUTH: ${AUTH} + SSL: ${SSL} + ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + LOAD_BALANCER: ${LOAD_BALANCER} + COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_BINARIES: ${MONGODB_BINARIES} + args: + - .github/scripts/run-orchestration.sh + name: Encryption tests + env: + FORCE_COLOR: true + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Setup node + uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: - release_version: ${{ env.package_version }} - product_name: mongodb - sarif_report_target_ref: 'main' - third_party_dependency_tool: n/a - dist_filenames: artifacts/* - token: ${{ github.token }} - sbom_file_name: sbom.json - evergreen_project: mongo-node-driver-next - evergreen_commit: ${{ env.commit }} - - - uses: mongodb-labs/drivers-github-tools/upload-s3-assets@v2 + node-version: latest + - name: Load MongoDB binary cache + id: cache-mongodb-binaries + uses: actions/cache@v4 with: - version: ${{ env.package_version }} - product_name: mongodb - dry_run: ${{ needs.release_please.outputs.release_created == '' }} \ No newline at end of file + path: ~/.cache/mongodb-binaries + key: ${{ matrix.os }}-${{ matrix.mongodb }} + - name: Install Dependencies + run: npm install + - name: Install drivers-evergreen-tools + + - name: Test + run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file From a041206260d287d0746df5c4973f7735c9238418 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 16:59:50 -0500 Subject: [PATCH 04/29] test --- .github/workflows/encryption-tests.yml | 62 ++++++++++++++++---------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index e8afec2d560..586cc450dae 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -10,6 +10,28 @@ permissions: name: encryption-tests +functions: + "bootstrap mongo-orchestration": + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: ${VERSION} + TOPOLOGY: ${TOPOLOGY} + AUTH: ${AUTH} + SSL: ${SSL} + ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + LOAD_BALANCER: ${LOAD_BALANCER} + COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_BINARIES: ${MONGODB_BINARIES} + args: + - ./workflows/run-orchestration.sh + jobs: set-up: permissions: @@ -21,7 +43,6 @@ jobs: steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - - name: Set-up cluster - shell: bash @@ -36,31 +57,10 @@ jobs: os: [ubuntu-latest] mongodb: [8.0.0] #clone drivers-evergreen-tools (will delete itself once the run is over) - git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git /data/mci/ead9f9a67ef900991e09bd9589dfee0e/drivers-tools #set path #calling run orchestration sets up a server on your computer # it outputs relevants uri in expansions.yml # put uri into an environment variable - - name: set-up-orchestration - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: ${VERSION} - TOPOLOGY: ${TOPOLOGY} - AUTH: ${AUTH} - SSL: ${SSL} - ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - LOAD_BALANCER: ${LOAD_BALANCER} - COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ${DRIVERS_TOOLS} - MONGODB_BINARIES: ${MONGODB_BINARIES} - args: - - .github/scripts/run-orchestration.sh name: Encryption tests env: FORCE_COLOR: true @@ -79,6 +79,22 @@ jobs: - name: Install Dependencies run: npm install - name: Install drivers-evergreen-tools - + run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools + - name: Set up cluster + + run: bash .github/workflows/run-orchestration.sh + with: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: 8.0.0 + TOPOLOGY: sharded + AUTH: auth + # ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + # REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + # LOAD_BALANCER: ${LOAD_BALANCER} + # COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools + MONGODB_BINARIES: ~/.cache/mongodb-binaries + - - name: Test run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file From 0c978a283aa6bce08293ff3e82489b98f0d16990 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 17:51:23 -0500 Subject: [PATCH 05/29] edit --- .github/workflows/encryption-tests.yml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 586cc450dae..69580c06f40 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,3 +1,5 @@ +name: encryption-tests + on: push: branches: ['main'] @@ -8,30 +10,6 @@ permissions: pull-requests: write id-token: write -name: encryption-tests - -functions: - "bootstrap mongo-orchestration": - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: ${VERSION} - TOPOLOGY: ${TOPOLOGY} - AUTH: ${AUTH} - SSL: ${SSL} - ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - LOAD_BALANCER: ${LOAD_BALANCER} - COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ${DRIVERS_TOOLS} - MONGODB_BINARIES: ${MONGODB_BINARIES} - args: - - ./workflows/run-orchestration.sh - jobs: set-up: permissions: From f8cbb9aef63b9c8f261c81f5a86383ce828a027c Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:13:16 -0500 Subject: [PATCH 06/29] removed errors --- .github/workflows/encryption-tests.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 69580c06f40..a39ed25b083 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,4 +1,4 @@ -name: encryption-tests +name: Encryption Tests on: push: @@ -21,8 +21,6 @@ jobs: steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - - name: Set-up cluster - - shell: bash run-tests: needs: @@ -59,9 +57,8 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - - run: bash .github/workflows/run-orchestration.sh - with: + run: bash /.github/scripts/run-orchestration.sh + env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 TOPOLOGY: sharded @@ -73,6 +70,5 @@ jobs: # These are paths necessary for launching mongodb DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools MONGODB_BINARIES: ~/.cache/mongodb-binaries - - - name: Test - run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file + run: mocha --exit ./test/encryption/*.test.js From 2fa3417ce20c45f3398b7b91e2951d5240b1c005 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:16:21 -0500 Subject: [PATCH 07/29] on push --- .github/workflows/encryption-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index a39ed25b083..24c0fd23626 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,9 +1,8 @@ name: Encryption Tests on: - push: - branches: ['main'] - workflow_dispatch: {} + push + #workflow_dispatch: {} permissions: contents: write From c71d3c287fd94a21de683005ae97920c9b585e42 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:21:34 -0500 Subject: [PATCH 08/29] test --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 24c0fd23626..6bb982c03b3 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-client-encryption + uses: mongodb-client-encryption@v6.1 run-tests: needs: From e6facfe19ce58bc2ebba3e6334406f7ee77cceff Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:27:21 -0500 Subject: [PATCH 09/29] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 6bb982c03b3..a5d87177559 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-client-encryption@v6.1 + uses: mongodb-js/mongodb-client-encryption@v6.1.0 run-tests: needs: From 06f9758cc2be1a62c196c1d09a05de613c518318 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:33:59 -0500 Subject: [PATCH 10/29] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index a5d87177559..788a57d1aaa 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-js/mongodb-client-encryption@v6.1.0 + run: npm install mongodb-client-encryption run-tests: needs: From 86f7874c5898ff298167666e25c24b4b047f3f3f Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:35:59 -0500 Subject: [PATCH 11/29] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 788a57d1aaa..90a10f1aa21 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash /.github/scripts/run-orchestration.sh + run: bash ../scripts/run-orchestration.sh env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 From 7a30734ed0959f673b2046a5e19ba503f167a01a Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:37:54 -0500 Subject: [PATCH 12/29] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 90a10f1aa21..00e3324072c 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash ../scripts/run-orchestration.sh + run: bash .github/scripts/run-orchestration.sh env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 From b39c75442305b51598efa78966c3c16e7150046e Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:34:08 -0500 Subject: [PATCH 13/29] fixed --- .github/workflows/encryption-tests.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 00e3324072c..9234f2b5083 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -53,21 +53,15 @@ jobs: key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - - name: Install drivers-evergreen-tools - run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash .github/scripts/run-orchestration.sh - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: 8.0.0 - TOPOLOGY: sharded - AUTH: auth - # ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - # REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - # LOAD_BALANCER: ${LOAD_BALANCER} - # COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools - MONGODB_BINARIES: ~/.cache/mongodb-binaries - - name: Test + id: setup-cluster + uses: mongodb-labs/drivers-evergreen-tools@master + with: + version: ${{ matrix.mongodb }} + topology: sharded + auth: auth + - name: Run Tests run: mocha --exit ./test/encryption/*.test.js + env: + MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} + CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From ccb726edbb9fcd6a253b2759967a5f3285ed6813 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:47:20 -0500 Subject: [PATCH 14/29] typo --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 9234f2b5083..c83217958dd 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -58,7 +58,7 @@ jobs: uses: mongodb-labs/drivers-evergreen-tools@master with: version: ${{ matrix.mongodb }} - topology: sharded + topology: sharded_cluster auth: auth - name: Run Tests run: mocha --exit ./test/encryption/*.test.js From b0a1c3c08e186925d179dc05a952bd073e768c92 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:51:24 -0500 Subject: [PATCH 15/29] install mocha --- .github/workflows/encryption-tests.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index c83217958dd..c4ee9e31727 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -10,20 +10,12 @@ permissions: id-token: write jobs: - set-up: + run-tests: permissions: # required for all workflows security-events: write id-token: write contents: write - runs-on: ubuntu-latest - steps: - - name: Install mongodb-client-encryption - run: npm install mongodb-client-encryption - - run-tests: - needs: - - set-up runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -40,17 +32,19 @@ jobs: env: FORCE_COLOR: true steps: + # - name: Install mongodb-client-encryption + # run: npm install mongodb-client-encryption - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup node uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: node-version: latest - - name: Load MongoDB binary cache - id: cache-mongodb-binaries - uses: actions/cache@v4 - with: - path: ~/.cache/mongodb-binaries - key: ${{ matrix.os }}-${{ matrix.mongodb }} + #- name: Load MongoDB binary cache + # id: cache-mongodb-binaries + # uses: actions/cache@v4 + # with: + # path: ~/.cache/mongodb-binaries + # key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - name: Set up cluster @@ -60,6 +54,8 @@ jobs: version: ${{ matrix.mongodb }} topology: sharded_cluster auth: auth + - name: Install mocha + run: brew install mocha - name: Run Tests run: mocha --exit ./test/encryption/*.test.js env: From 6198ade074d21b0d0d94df92d01930ca7ac9d995 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:56:31 -0500 Subject: [PATCH 16/29] fixed? --- .github/scripts/run-orchestration.sh | 36 -------------------------- .github/workflows/encryption-tests.yml | 17 +----------- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 .github/scripts/run-orchestration.sh diff --git a/.github/scripts/run-orchestration.sh b/.github/scripts/run-orchestration.sh deleted file mode 100644 index 3708c7bc33d..00000000000 --- a/.github/scripts/run-orchestration.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env bash - -# This is a thin wrapper around drivers-tools run orchestration meant to print each of the configuration settings we make use of -# Additionally it ensures the downloaded binaries are in the PATH for the script to find (namely, the legacy shell for server set up) - -export MONGODB_VERSION=${VERSION} -echo "MONGODB_VERSION=${VERSION}" - -export TOPOLOGY=${TOPOLOGY} -echo "TOPOLOGY=${TOPOLOGY}" - -export AUTH=${AUTH} -echo "AUTH=${AUTH}" - -export SSL=${SSL} -echo "SSL=${SSL}" - -export ORCHESTRATION_FILE=${ORCHESTRATION_FILE} -echo "ORCHESTRATION_FILE=${ORCHESTRATION_FILE}" - -export REQUIRE_API_VERSION=${REQUIRE_API_VERSION} -echo "REQUIRE_API_VERSION=${REQUIRE_API_VERSION}" - -export LOAD_BALANCER=${LOAD_BALANCER} -echo "LOAD_BALANCER=${LOAD_BALANCER}" - -export COMPRESSOR=${COMPRESSOR} -echo "COMPRESSOR=${COMPRESSOR}" - -export PATH="$MONGODB_BINARIES:$PATH" -echo "MONGODB_BINARIES=${MONGODB_BINARIES}" - -export SKIP_LEGACY_SHELL="true" -echo "SKIP_LEGACY_SHELL=${SKIP_LEGACY_SHELL}" - -bash "${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh" \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index c4ee9e31727..158cde2e10b 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -23,28 +23,15 @@ jobs: node: [20] os: [ubuntu-latest] mongodb: [8.0.0] - #clone drivers-evergreen-tools (will delete itself once the run is over) - #set path - #calling run orchestration sets up a server on your computer - # it outputs relevants uri in expansions.yml - # put uri into an environment variable name: Encryption tests env: FORCE_COLOR: true steps: - # - name: Install mongodb-client-encryption - # run: npm install mongodb-client-encryption - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup node uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: node-version: latest - #- name: Load MongoDB binary cache - # id: cache-mongodb-binaries - # uses: actions/cache@v4 - # with: - # path: ~/.cache/mongodb-binaries - # key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - name: Set up cluster @@ -54,10 +41,8 @@ jobs: version: ${{ matrix.mongodb }} topology: sharded_cluster auth: auth - - name: Install mocha - run: brew install mocha - name: Run Tests - run: mocha --exit ./test/encryption/*.test.js + run: npx mocha --exit ./test/encryption/*.test.js env: MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From a135e79c93a1836acc0639c7dfd3304565c11eda Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 15:02:53 -0500 Subject: [PATCH 17/29] fix tests --- test/encryption/encryption.test.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index cb9d3be2d66..66088cf0c51 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -3,9 +3,14 @@ const assert = require('assert'); const EJSON = require('bson').EJSON; -describe('setup check', () => { - it('environment variables are set', async function() { - const { local } = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}'); - assert.ok(local); +describe('environmental variables', () => { + it('MONGODB_URI is set', async function() { + const uri = process.env.MONGODB_URI; + assert.ok(uri); + }); + + it('CRYPT_SHARED_LIB_PATH is set', async function() { + const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; + assert.ok(shared_library_path); }); }); From ca258684b1db84b5cfb13a9b8f8eb9b75dc95279 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 15:59:31 -0500 Subject: [PATCH 18/29] run tests --- .github/workflows/encryption-tests.yml | 4 ++- mongocryptd.pid | 0 package.json | 1 + test/encryption/encryption.test.js | 35 +++++++++++++++++++++++--- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 mongocryptd.pid diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 158cde2e10b..09dc64df74b 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -34,6 +34,8 @@ jobs: node-version: latest - name: Install Dependencies run: npm install + - name: Install mongodb-client-encryption + run: npm install mongodb-client-encryption - name: Set up cluster id: setup-cluster uses: mongodb-labs/drivers-evergreen-tools@master @@ -44,5 +46,5 @@ jobs: - name: Run Tests run: npx mocha --exit ./test/encryption/*.test.js env: - MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} + MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/mongocryptd.pid b/mongocryptd.pid new file mode 100644 index 00000000000..e69de29bb2d diff --git a/package.json b/package.json index 085d0655a7a..c24f2444972 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "bson": "^6.7.0", "kareem": "2.6.3", "mongodb": "~6.10.0", + "mongodb-client-encryption": "^6.1.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index 66088cf0c51..e6aa2f1d7b0 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -1,11 +1,11 @@ 'use strict'; const assert = require('assert'); -const EJSON = require('bson').EJSON; +const mdb = require('mongodb'); describe('environmental variables', () => { - it('MONGODB_URI is set', async function() { - const uri = process.env.MONGODB_URI; + it('MONGODB_TEST_URI is set', async function() { + const uri = process.env.MONGOOSE_TEST_URI; assert.ok(uri); }); @@ -14,3 +14,32 @@ describe('environmental variables', () => { assert.ok(shared_library_path); }); }); + +describe('basic integration', () => { + it('supports mongodb csfle auto-encryption integration', async() => { + // 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) + const client = new mdb.MongoClient( + process.env.MONGOOSE_TEST_URI, + { + autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: Buffer.from( + 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', + 'base64' + ) + } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + } + ); + await client.connect(); + const insertResult = await client + .db('db') + .collection('coll') + .insertOne({ unencrypted: 'test' }); + assert.ok(insertResult.insertedId); + }); +}); From 270d151fcb14ef8bfff2be5b4420d9e69c368bb4 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:54:17 -0500 Subject: [PATCH 19/29] ready for rereview --- .github/scripts/run-kms-servers.sh | 11 ---- .github/workflows/encryption-tests.yml | 17 ++---- package.json | 1 + test/encryption/encryption.test.js | 80 +++++++++++++++++++++----- 4 files changed, 72 insertions(+), 37 deletions(-) delete mode 100644 .github/scripts/run-kms-servers.sh diff --git a/.github/scripts/run-kms-servers.sh b/.github/scripts/run-kms-servers.sh deleted file mode 100644 index 988b07bfef1..00000000000 --- a/.github/scripts/run-kms-servers.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail - -cd ${DRIVERS_TOOLS}/.evergreen/csfle -. ./prepare-kmsvenv.sh - -echo "$PYTHON_EXEC" - -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert & \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 09dc64df74b..2151b4eb3ea 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,8 +1,9 @@ name: Encryption Tests on: - push - #workflow_dispatch: {} + push: + branches: ['master'] + workflow_dispatch: {} permissions: contents: write @@ -16,13 +17,7 @@ jobs: security-events: write id-token: write contents: write - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - node: [20] - os: [ubuntu-latest] - mongodb: [8.0.0] + runs-on: ubuntu-latest name: Encryption tests env: FORCE_COLOR: true @@ -40,11 +35,11 @@ jobs: id: setup-cluster uses: mongodb-labs/drivers-evergreen-tools@master with: - version: ${{ matrix.mongodb }} + version: 8.0.0 topology: sharded_cluster auth: auth - name: Run Tests - run: npx mocha --exit ./test/encryption/*.test.js + run: npm run encryption-test env: MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/package.json b/package.json index c24f2444972..405af204ade 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", + "test-encryption": "mocha --exit ./test/encryption/*.test.js", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index e6aa2f1d7b0..a1b082433a7 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -2,9 +2,12 @@ const assert = require('assert'); const mdb = require('mongodb'); +const isBsonType = require('../../lib/helpers/isBsonType'); + +const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); describe('environmental variables', () => { - it('MONGODB_TEST_URI is set', async function() { + it('MONGOOSE_TEST_URI is set', async function() { const uri = process.env.MONGOOSE_TEST_URI; assert.ok(uri); }); @@ -16,18 +19,44 @@ describe('environmental variables', () => { }); describe('basic integration', () => { - it('supports mongodb csfle auto-encryption integration', async() => { - // 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - const client = new mdb.MongoClient( + let keyVaultClient; + let dataKey; + let encryptedClient; + let dummyClient; + + beforeEach(async function() { + keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + dataKey = await clientEncryption.createDataKey('local'); + + encryptedClient = new mdb.MongoClient( process.env.MONGOOSE_TEST_URI, { autoEncryption: { keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: Buffer.from( - 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', - 'base64' - ) - } }, + kmsProviders: { local: { key: LOCAL_KEY } }, + schemaMap: { + 'db.coll': { + bsonType: 'object', + encryptMetadata: { + keyId: [new mdb.UUID(dataKey)] + }, + properties: { + a: { + encrypt: { + bsonType: 'int', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', + keyId: [new mdb.UUID(dataKey)] + } + } + } + } + }, extraOptions: { cryptdSharedLibRequired: true, cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH @@ -35,11 +64,32 @@ describe('basic integration', () => { } } ); - await client.connect(); - const insertResult = await client - .db('db') - .collection('coll') - .insertOne({ unencrypted: 'test' }); - assert.ok(insertResult.insertedId); + + dummyClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); + + afterEach(async function() { + await keyVaultClient.close(); + await encryptedClient.close(); + await dummyClient.close(); + }); + + it('supports mongodb csfle auto-encryption integration', async() => { + await encryptedClient.connect(); + await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + + // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded + const encryptedCursor = await dummyClient.db('db').collection('coll').find(); + const encryptedResult = await encryptedCursor.next(); + assert.ok(encryptedResult); + assert.ok(encryptedResult.a); + assert.ok(isBsonType(encryptedResult.a, 'Binary')); + assert.ok(encryptedResult.a.sub_type === 6); + + // when the encryptedClient runs a find, the original unencrypted value is returned + const unencryptedCursor = await encryptedClient.db('db').collection('coll').find(); + const unencryptedResult = await unencryptedCursor.next(); + assert.ok(unencryptedResult); + assert.ok(unencryptedResult.a === 1); }); }); From 69dbda600aaaee77b07d497c43b9830f4a78f13b Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:55:28 -0500 Subject: [PATCH 20/29] ready for rereview 2 --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 2151b4eb3ea..dc586d31118 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -2,7 +2,7 @@ name: Encryption Tests on: push: - branches: ['master'] + branches: ['master', 'NODE-6505/ci-setup'] workflow_dispatch: {} permissions: From 3ce14a41f8ba476bef63c0018d20e5fdd9258126 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:58:55 -0500 Subject: [PATCH 21/29] typo --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index dc586d31118..17ffbe3ca22 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -39,7 +39,7 @@ jobs: topology: sharded_cluster auth: auth - name: Run Tests - run: npm run encryption-test + run: npm run test-encryption env: MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From eac708f94c7318adcd86e9214e0630fe7b5a4222 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:24:32 -0500 Subject: [PATCH 22/29] ready for review --- .github/workflows/encryption-tests.yml | 4 +- .gitignore | 2 + CONTRIBUTING.md | 1 + mongocryptd.pid | 0 package.json | 2 + scripts/encrypted-cluster.sh | 57 ++++++++++++++++++++++++++ test/encryption/encryption.test.js | 10 +++-- 7 files changed, 71 insertions(+), 5 deletions(-) delete mode 100644 mongocryptd.pid create mode 100755 scripts/encrypted-cluster.sh diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 17ffbe3ca22..e605778920b 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -2,7 +2,9 @@ name: Encryption Tests on: push: - branches: ['master', 'NODE-6505/ci-setup'] + branches: ['master'] + pull_request: + branches: [ 'master' ] workflow_dispatch: {} permissions: diff --git a/.gitignore b/.gitignore index 47c0742bb12..f88cd4f6ad9 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,5 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out + +encrypted-cluster \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ba098d3897..a6d1cfac102 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/encrypted-cluster.sh` file ## Documentation diff --git a/mongocryptd.pid b/mongocryptd.pid deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/package.json b/package.json index 405af204ade..660698caaab 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "license": "MIT", "dependencies": { "bson": "^6.7.0", + "hugo": "^0.0.3", "kareem": "2.6.3", "mongodb": "~6.10.0", "mongodb-client-encryption": "^6.1.0", @@ -106,6 +107,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", + "test-encryption-local": "chmod +x scripts/encrypted-cluster.sh && scripts/encrypted-cluster.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/encrypted-cluster.sh b/scripts/encrypted-cluster.sh new file mode 100755 index 00000000000..e919f1870c6 --- /dev/null +++ b/scripts/encrypted-cluster.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# sets up an encrypted mongodb cluster + +if [ -d "encrypted-cluster" ]; then + cd encrypted-cluster +else + mkdir encrypted-cluster + cd encrypted-cluster + + if [-d drivers-evergreen-tools]; then + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" + fi + + export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + export MONGODB_VERSION=8.0 + export AUTH=true + export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin + export NODE_DRIVER=~/dev/node-mongodb-native + export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo + export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration + export TOPOLOGY=sharded_cluster + export SSL=nossl + + cd $DRIVERS_TOOLS + rm -rf mongosh mongodb mo + mkdir mo + cd - + + rm expansions.sh 2> /dev/null + + bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh +fi + +# IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output + +read -r -d '' SOURCE_SCRIPT << EOM +const fs = require('fs'); +const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) + .trim().split('\\n'); +const regex = /^(?.*): "(?.*)"$/; +const variables = file.map( + (line) => regex.exec(line.trim()).groups +).map( + ({key, value}) => \`export \${key}='\${value}'\` +).join('\n'); + +process.stdout.write(variables); +process.stdout.write('\n'); +EOM + +node --eval "$SOURCE_SCRIPT" | tee expansions.sh +source expansions.sh + +export MONGOOSE_TEST_URI=$MONGODB_URI + +npm run test-encryption diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index a1b082433a7..ba29506965b 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -44,14 +44,14 @@ describe('basic integration', () => { 'db.coll': { bsonType: 'object', encryptMetadata: { - keyId: [new mdb.UUID(dataKey)] + keyId: [dataKey] }, properties: { a: { encrypt: { bsonType: 'int', algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [new mdb.UUID(dataKey)] + keyId: [dataKey] } } } @@ -78,9 +78,11 @@ describe('basic integration', () => { await encryptedClient.connect(); await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedCursor = await dummyClient.db('db').collection('coll').find(); - const encryptedResult = await encryptedCursor.next(); + const encryptedResult = await dummyClient.db('db').collection('coll').findOne({ _id: insertedId }); + assert.ok(encryptedResult); assert.ok(encryptedResult.a); assert.ok(isBsonType(encryptedResult.a, 'Binary')); From f38366bd1ea3fcb9d8e3f0e1c0e07cb51159e17c Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:41:46 -0500 Subject: [PATCH 23/29] change to two files for local testing --- .gitignore | 2 +- CONTRIBUTING.md | 2 +- package.json | 2 +- ...uster.sh => run-encryption-tests-local.sh} | 28 ++----------------- scripts/start-encrypted-cluster.sh | 27 ++++++++++++++++++ 5 files changed, 33 insertions(+), 28 deletions(-) rename scripts/{encrypted-cluster.sh => run-encryption-tests-local.sh} (51%) create mode 100644 scripts/start-encrypted-cluster.sh diff --git a/.gitignore b/.gitignore index f88cd4f6ad9..88f5a87efba 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,4 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out -encrypted-cluster \ No newline at end of file +encrypted-cluster diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6d1cfac102..086e2734a22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/encrypted-cluster.sh` file +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file ## Documentation diff --git a/package.json b/package.json index 660698caaab..06adb0af08b 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "chmod +x scripts/encrypted-cluster.sh && scripts/encrypted-cluster.sh", + "test-encryption-local": "chmod +x scripts/run-encryption-tests-local.sh && scripts/run-encryption-tests-local.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/encrypted-cluster.sh b/scripts/run-encryption-tests-local.sh similarity index 51% rename from scripts/encrypted-cluster.sh rename to scripts/run-encryption-tests-local.sh index e919f1870c6..dd29eb98800 100755 --- a/scripts/encrypted-cluster.sh +++ b/scripts/run-encryption-tests-local.sh @@ -2,34 +2,12 @@ # sets up an encrypted mongodb cluster +export CWD=$(pwd); + if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else - mkdir encrypted-cluster - cd encrypted-cluster - - if [-d drivers-evergreen-tools]; then - git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" - fi - - export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - export MONGODB_VERSION=8.0 - export AUTH=true - export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin - export NODE_DRIVER=~/dev/node-mongodb-native - export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo - export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration - export TOPOLOGY=sharded_cluster - export SSL=nossl - - cd $DRIVERS_TOOLS - rm -rf mongosh mongodb mo - mkdir mo - cd - - - rm expansions.sh 2> /dev/null - - bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh + source $CWD/scripts/start-encrypted-cluster.sh fi # IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output diff --git a/scripts/start-encrypted-cluster.sh b/scripts/start-encrypted-cluster.sh new file mode 100644 index 00000000000..b04c4b655c1 --- /dev/null +++ b/scripts/start-encrypted-cluster.sh @@ -0,0 +1,27 @@ + +export CWD=$(pwd); +mkdir encrypted-cluster +cd encrypted-cluster + +if [ ! -d "drivers-evergreen-tools/" ]; then + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" +fi + +export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools +export MONGODB_VERSION=8.0 +export AUTH=true +export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin +export NODE_DRIVER=~/dev/node-mongodb-native +export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo +export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration +export TOPOLOGY=sharded_cluster +export SSL=nossl + +cd $DRIVERS_TOOLS +rm -rf mongosh mongodb mo +mkdir mo +cd - + +rm expansions.sh 2> /dev/null + +bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh \ No newline at end of file From 62d18d830a74d24218b24cef26253317c38a2086 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:43:43 -0500 Subject: [PATCH 24/29] fixed deps --- CONTRIBUTING.md | 2 +- package.json | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 086e2734a22..97f084447dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file. ## Documentation diff --git a/package.json b/package.json index 06adb0af08b..0d0fe6f99c3 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,8 @@ "license": "MIT", "dependencies": { "bson": "^6.7.0", - "hugo": "^0.0.3", "kareem": "2.6.3", "mongodb": "~6.10.0", - "mongodb-client-encryption": "^6.1.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", From 40858d490b529dac1eec8b3e4d69fcb98e9e6b49 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 11 Dec 2024 14:25:03 -0500 Subject: [PATCH 25/29] requested changes --- .eslintrc.js | 3 +- mongocryptd.pid | 0 package.json | 2 +- scripts/run-encryption-tests-local.sh | 8 +- scripts/start-encrypted-cluster.sh | 13 ++- test/encryption/encryption.test.js | 139 +++++++++++++------------- 6 files changed, 88 insertions(+), 77 deletions(-) create mode 100644 mongocryptd.pid diff --git a/.eslintrc.js b/.eslintrc.js index b4d7d1652d9..7670926d241 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -14,7 +14,8 @@ module.exports = { '**/docs/js/native.js', '!.*', 'node_modules', - '.git' + '.git', + 'encrypted-cluster' ], overrides: [ { diff --git a/mongocryptd.pid b/mongocryptd.pid new file mode 100644 index 00000000000..e69de29bb2d diff --git a/package.json b/package.json index 0d0fe6f99c3..5c726e144c4 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "chmod +x scripts/run-encryption-tests-local.sh && scripts/run-encryption-tests-local.sh", + "test-encryption-local": "bash scripts/run-encryption-tests-local.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index dd29eb98800..ff4332370bd 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash -# sets up an encrypted mongodb cluster +# sets up an encrypted mongodb cluster, adds relevant variables to the environment, and runs encryption tests export CWD=$(pwd); +# set up encrypted mongodb cluster if the encrypted-cluster folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/start-encrypted-cluster.sh' script if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else source $CWD/scripts/start-encrypted-cluster.sh fi -# IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output - +# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run read -r -d '' SOURCE_SCRIPT << EOM const fs = require('fs'); const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) @@ -32,4 +33,5 @@ source expansions.sh export MONGOOSE_TEST_URI=$MONGODB_URI +# run encryption tests npm run test-encryption diff --git a/scripts/start-encrypted-cluster.sh b/scripts/start-encrypted-cluster.sh index b04c4b655c1..1fb8a4aa22c 100644 --- a/scripts/start-encrypted-cluster.sh +++ b/scripts/start-encrypted-cluster.sh @@ -1,17 +1,25 @@ +# creates a encrypted cluster (sharded on 8.0 server) export CWD=$(pwd); mkdir encrypted-cluster cd encrypted-cluster +# note: + # we're using drivers-evergreen-tools which is a repo that handles cluster set-up for us. + # if you'd like to make changes to the cluster settings, edit the exported variables below. + # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh + # after this script is run, the encrypted-cluster/ folder will notably contain the following: + # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path + # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod if [ ! -d "drivers-evergreen-tools/" ]; then git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" fi +# configure cluster settings export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools export MONGODB_VERSION=8.0 export AUTH=true export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin -export NODE_DRIVER=~/dev/node-mongodb-native export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration export TOPOLOGY=sharded_cluster @@ -24,4 +32,5 @@ cd - rm expansions.sh 2> /dev/null -bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh \ No newline at end of file +# start cluster +bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index ba29506965b..14e18306d94 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -6,92 +6,91 @@ const isBsonType = require('../../lib/helpers/isBsonType'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); -describe('environmental variables', () => { - it('MONGOOSE_TEST_URI is set', async function() { - const uri = process.env.MONGOOSE_TEST_URI; - assert.ok(uri); - }); +describe('ci', () => { + describe('environmental variables', () => { + it('MONGOOSE_TEST_URI is set', async function() { + const uri = process.env.MONGOOSE_TEST_URI; + assert.ok(uri); + }); - it('CRYPT_SHARED_LIB_PATH is set', async function() { - const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; - assert.ok(shared_library_path); + it('CRYPT_SHARED_LIB_PATH is set', async function() { + const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; + assert.ok(shared_library_path); + }); }); -}); -describe('basic integration', () => { - let keyVaultClient; - let dataKey; - let encryptedClient; - let dummyClient; + describe('basic integration', () => { + let keyVaultClient; + let dataKey; + let encryptedClient; + let unencryptedClient; - beforeEach(async function() { - keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); - await keyVaultClient.connect(); - await keyVaultClient.db('keyvault').collection('datakeys'); - const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } } - }); - dataKey = await clientEncryption.createDataKey('local'); + beforeEach(async function() { + keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + dataKey = await clientEncryption.createDataKey('local'); - encryptedClient = new mdb.MongoClient( - process.env.MONGOOSE_TEST_URI, - { - autoEncryption: { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } }, - schemaMap: { - 'db.coll': { - bsonType: 'object', - encryptMetadata: { - keyId: [dataKey] - }, - properties: { - a: { - encrypt: { - bsonType: 'int', - algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [dataKey] + encryptedClient = new mdb.MongoClient( + process.env.MONGOOSE_TEST_URI, + { + autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + schemaMap: { + 'db.coll': { + bsonType: 'object', + encryptMetadata: { + keyId: [dataKey] + }, + properties: { + a: { + encrypt: { + bsonType: 'int', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', + keyId: [dataKey] + } } } } + }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } - }, - extraOptions: { - cryptdSharedLibRequired: true, - cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } } - } - ); + ); - dummyClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); - }); - - afterEach(async function() { - await keyVaultClient.close(); - await encryptedClient.close(); - await dummyClient.close(); - }); + unencryptedClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); - it('supports mongodb csfle auto-encryption integration', async() => { - await encryptedClient.connect(); - await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + afterEach(async function() { + await keyVaultClient.close(); + await encryptedClient.close(); + await unencryptedClient.close(); + }); - const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + it('ci set-up should support basic mongodb auto-encryption integration', async() => { + await encryptedClient.connect(); + const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); - // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedResult = await dummyClient.db('db').collection('coll').findOne({ _id: insertedId }); + // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded + const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); - assert.ok(encryptedResult); - assert.ok(encryptedResult.a); - assert.ok(isBsonType(encryptedResult.a, 'Binary')); - assert.ok(encryptedResult.a.sub_type === 6); + assert.ok(encryptedResult); + assert.ok(encryptedResult.a); + assert.ok(isBsonType(encryptedResult.a, 'Binary')); + assert.ok(encryptedResult.a.sub_type === 6); - // when the encryptedClient runs a find, the original unencrypted value is returned - const unencryptedCursor = await encryptedClient.db('db').collection('coll').find(); - const unencryptedResult = await unencryptedCursor.next(); - assert.ok(unencryptedResult); - assert.ok(unencryptedResult.a === 1); + // when the encryptedClient runs a find, the original unencrypted value is returned + const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); + assert.ok(unencryptedResult); + assert.ok(unencryptedResult.a === 1); + }); }); }); From d6044e79358fd574a1b7190b593f7452d614069b Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 11 Dec 2024 15:28:33 -0500 Subject: [PATCH 26/29] fix --- mongocryptd.pid | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mongocryptd.pid diff --git a/mongocryptd.pid b/mongocryptd.pid deleted file mode 100644 index e69de29bb2d..00000000000 From 08f4c2345a89a9d70dbe089b05f56a38a809cd40 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 18 Dec 2024 10:36:05 -0500 Subject: [PATCH 27/29] fix wording --- .gitignore | 1 + CONTRIBUTING.md | 2 +- ...ed-cluster.sh => configure-cluster-with-encryption.sh} | 8 ++++++-- scripts/run-encryption-tests-local.sh | 8 ++++---- 4 files changed, 12 insertions(+), 7 deletions(-) rename scripts/{start-encrypted-cluster.sh => configure-cluster-with-encryption.sh} (76%) diff --git a/.gitignore b/.gitignore index 88f5a87efba..e78311260a1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ notes.md list.out encrypted-cluster +*.pid \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97f084447dd..a94baf554bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-cluster-with-encryption.sh` file. ## Documentation diff --git a/scripts/start-encrypted-cluster.sh b/scripts/configure-cluster-with-encryption.sh similarity index 76% rename from scripts/start-encrypted-cluster.sh rename to scripts/configure-cluster-with-encryption.sh index 1fb8a4aa22c..1a77feee78f 100644 --- a/scripts/start-encrypted-cluster.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -1,11 +1,15 @@ -# creates a encrypted cluster (sharded on 8.0 server) +# note: in order to use FLE with mongodb, we must +# have mongocryptd or the shared library downloaded +# have an enterprise server >= 4.2 + +# this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server) export CWD=$(pwd); mkdir encrypted-cluster cd encrypted-cluster # note: - # we're using drivers-evergreen-tools which is a repo that handles cluster set-up for us. + # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. # if you'd like to make changes to the cluster settings, edit the exported variables below. # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh # after this script is run, the encrypted-cluster/ folder will notably contain the following: diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index ff4332370bd..1b83c324033 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -# sets up an encrypted mongodb cluster, adds relevant variables to the environment, and runs encryption tests +# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests export CWD=$(pwd); -# set up encrypted mongodb cluster if the encrypted-cluster folder does not exist -# note: for tooling, cluster set-up and configuration look into the 'scripts/start-encrypted-cluster.sh' script +# set up mongodb cluster and encryption configuration if the encrypted-cluster folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/start-cluster-with-encryption.sh' script if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else - source $CWD/scripts/start-encrypted-cluster.sh + source $CWD/scripts/start-cluster-with-encryption.sh fi # extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run From 1098636943915c39c2f381aa69d071b565fd85e2 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 26 Dec 2024 13:00:42 -0500 Subject: [PATCH 28/29] change all occurences of encrypted-cluster to data --- .eslintrc.js | 2 +- .gitignore | 2 +- scripts/configure-cluster-with-encryption.sh | 8 ++++---- scripts/run-encryption-tests-local.sh | 11 ++++++----- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 7670926d241..4b4f2f312cc 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -15,7 +15,7 @@ module.exports = { '!.*', 'node_modules', '.git', - 'encrypted-cluster' + 'data' ], overrides: [ { diff --git a/.gitignore b/.gitignore index e78311260a1..c66816fcd42 100644 --- a/.gitignore +++ b/.gitignore @@ -68,5 +68,5 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out -encrypted-cluster +data *.pid \ No newline at end of file diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 1a77feee78f..4584920ed40 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -5,14 +5,14 @@ # this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server) export CWD=$(pwd); -mkdir encrypted-cluster -cd encrypted-cluster +mkdir data +cd data # note: # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. # if you'd like to make changes to the cluster settings, edit the exported variables below. # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh - # after this script is run, the encrypted-cluster/ folder will notably contain the following: + # after this script is run, the data/ folder will notably contain the following: # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod if [ ! -d "drivers-evergreen-tools/" ]; then @@ -20,7 +20,7 @@ if [ ! -d "drivers-evergreen-tools/" ]; then fi # configure cluster settings -export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools +export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools export MONGODB_VERSION=8.0 export AUTH=true export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index 1b83c324033..244c1254591 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -4,12 +4,13 @@ export CWD=$(pwd); -# set up mongodb cluster and encryption configuration if the encrypted-cluster folder does not exist -# note: for tooling, cluster set-up and configuration look into the 'scripts/start-cluster-with-encryption.sh' script -if [ -d "encrypted-cluster" ]; then - cd encrypted-cluster +# set up mongodb cluster and encryption configuration if the data/ folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script + +if [ -d "data" ]; then + cd data else - source $CWD/scripts/start-cluster-with-encryption.sh + source $CWD/scripts/configure-cluster-with-encryption.sh fi # extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run From 955cedf05b0db8cff1f72165c540a4c353d4efa0 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 27 Dec 2024 13:22:00 -0500 Subject: [PATCH 29/29] remove extra gha call - use local script instead lint typo lint --- .github/workflows/encryption-tests.yml | 10 ---------- .gitignore | 2 +- CONTRIBUTING.md | 2 +- package.json | 3 +-- ...cryption-tests-local.sh => run-encryption-tests.sh} | 3 ++- 5 files changed, 5 insertions(+), 15 deletions(-) rename scripts/{run-encryption-tests-local.sh => run-encryption-tests.sh} (95%) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index e605778920b..263ebaedc11 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -33,15 +33,5 @@ jobs: run: npm install - name: Install mongodb-client-encryption run: npm install mongodb-client-encryption - - name: Set up cluster - id: setup-cluster - uses: mongodb-labs/drivers-evergreen-tools@master - with: - version: 8.0.0 - topology: sharded_cluster - auth: auth - name: Run Tests run: npm run test-encryption - env: - MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} - CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/.gitignore b/.gitignore index c66816fcd42..9a52110981e 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,4 @@ notes.md list.out data -*.pid \ No newline at end of file +*.pid diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a94baf554bd..06073758d97 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-cluster-with-encryption.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. ## Documentation diff --git a/package.json b/package.json index 5c726e144c4..39d4451eeb7 100644 --- a/package.json +++ b/package.json @@ -104,8 +104,7 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", - "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "bash scripts/run-encryption-tests-local.sh", + "test-encryption": "bash scripts/run-encryption-tests.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests.sh similarity index 95% rename from scripts/run-encryption-tests-local.sh rename to scripts/run-encryption-tests.sh index 244c1254591..0209292168d 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests.sh @@ -35,4 +35,5 @@ source expansions.sh export MONGOOSE_TEST_URI=$MONGODB_URI # run encryption tests -npm run test-encryption +cd .. +npx mocha --exit ./test/encryption/*.test.js