INTPYTHON-527 Add Queryable Encryption support

.evergreen/config.yml

@@ -12,6 +12,17 @@ post_error_fails_task: true
 post_timeout_secs: 1800 # 5 minutes
 
 functions:
+  "start csfle servers":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
+      params:
+        binary: bash
+        include_expansions_in_env: ["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_SESSION_TOKEN"]
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
+
   "setup":
     - command: git.get_project
       params:
@@ -44,7 +55,7 @@ functions:
       params:
         binary: bash
         working_dir: "src"
-        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
+        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI", "DJANGO_SETTINGS_MODULE", "CRYPT_SHARED_LIB_PATH"]
         args:
           - ./.evergreen/run-tests.sh
 
@@ -54,9 +65,11 @@ functions:
         binary: bash
         args:
           - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
 
 pre:
   - func: setup
+  - func: start csfle servers
   - func: bootstrap mongo-orchestration
 
 post:
@@ -111,3 +124,13 @@ buildvariants:
       SSL: "ssl"
     tasks:
       - name: run-tests
+
+  - name: tests-8-qe
+    display_name: Run Tests 8.2 QE
+    run_on: rhel87-small
+    expansions:
+      MONGODB_VERSION: "8.2"
+      TOPOLOGY: replica_set
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+    tasks:
+      - name: run-tests

.evergreen/run-tests.sh

@@ -2,11 +2,14 @@
 
 set -eux
 
+# Export secrets as environment variables
+. ../secrets-export.sh
+
 # Install django-mongodb-backend
 /opt/python/3.10/bin/python3 -m venv venv
 . venv/bin/activate
 python -m pip install -U pip
-pip install -e .
+pip install -e '.[encryption]'
 
 # Install django and test dependencies
 git clone --branch mongodb-5.2.x https://github.com/mongodb-forks/django django_repo

.evergreen/setup.sh

@@ -16,8 +16,8 @@ DRIVERS_TOOLS="$(dirname "$(pwd)")/drivers-tools"
 PROJECT_DIRECTORY="$(pwd)"
 
 if [ "Windows_NT" = "${OS:-}" ]; then
-    DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
-    PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+    DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS")
+    PROJECT_DIRECTORY=$(cygpath -m "$PROJECT_DIRECTORY")
 fi
 export PROJECT_DIRECTORY
 export DRIVERS_TOOLS
@@ -37,8 +37,8 @@ PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
 EOT
 
 # Set up drivers-tools with a .env file.
-git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git ${DRIVERS_TOOLS}
-cat <<EOT > ${DRIVERS_TOOLS}/.env
+git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git "${DRIVERS_TOOLS}"
+cat <<EOT > "${DRIVERS_TOOLS}/.env"
 CURRENT_VERSION="$CURRENT_VERSION"
 DRIVERS_TOOLS="$DRIVERS_TOOLS"
 MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"

.github/workflows/encrypted_settings.py

@@ -0,0 +1,64 @@
+# Settings for django_mongodb_backend/tests when encryption is supported.
+import os
+from pathlib import Path
+
+from mongodb_settings import *  # noqa: F403
+from pymongo.encryption import AutoEncryptionOpts
+
+os.environ["LD_LIBRARY_PATH"] = str(Path(os.environ["CRYPT_SHARED_LIB_PATH"]).parent)
+
+AWS_CREDS = {
+    "accessKeyId": os.environ.get("FLE_AWS_KEY", ""),
+    "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""),
+}
+
+_USE_AWS_KMS = any(AWS_CREDS.values())
+
+if _USE_AWS_KMS:
+    _AWS_REGION = os.environ.get("FLE_AWS_KMS_REGION", "us-east-1")
+    _AWS_KEY_ARN = os.environ.get(
+        "FLE_AWS_KMS_KEY_ARN",
+        "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
+    )
+    KMS_PROVIDERS = {"aws": AWS_CREDS}
+    KMS_CREDENTIALS = {"aws": {"key": _AWS_KEY_ARN, "region": _AWS_REGION}}
+else:
+    KMS_PROVIDERS = {"local": {"key": os.urandom(96)}}
+    KMS_CREDENTIALS = {"local": {}}
+
+DATABASES["encrypted"] = {  # noqa: F405
+    "ENGINE": "django_mongodb_backend",
+    "NAME": "djangotests_encrypted",
+    "OPTIONS": {
+        "auto_encryption_opts": AutoEncryptionOpts(
+            key_vault_namespace="djangotests_encrypted.__keyVault",
+            kms_providers=KMS_PROVIDERS,
+            crypt_shared_lib_path=os.environ["CRYPT_SHARED_LIB_PATH"],
+            crypt_shared_lib_required=True,
+        ),
+        "directConnection": True,
+    },
+    "KMS_CREDENTIALS": KMS_CREDENTIALS,
+}
+
+
+class EncryptedRouter:
+    def db_for_read(self, model, **hints):
+        if model._meta.app_label == "encryption_":
+            return "encrypted"
+        return None
+
+    db_for_write = db_for_read
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        # The encryption_ app's models are only created in the encrypted
+        # database.
+        if app_label == "encryption_":
+            return db == "encrypted"
+        # Don't create other app's models in the encrypted database.
+        if db == "encrypted":
+            return False
+        return None
+
+
+DATABASE_ROUTERS.append(EncryptedRouter())  # noqa: F405

.github/workflows/mongodb_settings.py

@@ -1,4 +1,5 @@
-# Settings for django_mongodb_backend/tests.
+# Settings for django_mongodb_backend/tests when encryption isn't supported.
 from django_settings import *  # noqa: F403
 
+DATABASES["encrypted"] = {}  # noqa: F405
 DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"]

.github/workflows/runtests.py

@@ -6,151 +6,6 @@
 from django.core.exceptions import ImproperlyConfigured
 
 test_apps = [
-    "admin_changelist",
-    "admin_checks",
-    "admin_custom_urls",
-    "admin_docs",
-    "admin_filters",
-    "admin_inlines",
-    "admin_ordering",
-    "admin_scripts",
-    "admin_utils",
-    "admin_views",
-    "admin_widgets",
-    "aggregation",
-    "aggregation_regress",
-    "annotations",
-    "apps",
-    "async",
-    "auth_tests",
-    "backends",
-    "basic",
-    "bulk_create",
-    "cache",
-    "check_framework",
-    "constraints",
-    "contenttypes_tests",
-    "context_processors",
-    "custom_columns",
-    "custom_lookups",
-    "custom_managers",
-    "custom_pk",
-    "datatypes",
-    "dates",
-    "datetimes",
-    "db_functions",
-    "defer",
-    "defer_regress",
-    "delete",
-    "delete_regress",
-    "empty",
-    "empty_models",
-    "expressions",
-    "expressions_case",
-    "field_defaults",
-    "file_storage",
-    "file_uploads",
-    "fixtures",
-    "fixtures_model_package",
-    "fixtures_regress",
-    "flatpages_tests",
-    "force_insert_update",
-    "foreign_object",
-    "forms_tests",
-    "from_db_value",
-    "generic_inline_admin",
-    "generic_relations",
-    "generic_relations_regress",
-    "generic_views",
-    "get_earliest_or_latest",
-    "get_object_or_404",
-    "get_or_create",
-    "i18n",
-    "indexes",
-    "inline_formsets",
-    "introspection",
-    "invalid_models_tests",
-    "known_related_objects",
-    "lookup",
-    "m2m_and_m2o",
-    "m2m_intermediary",
-    "m2m_multiple",
-    "m2m_recursive",
-    "m2m_regress",
-    "m2m_signals",
-    "m2m_through",
-    "m2m_through_regress",
-    "m2o_recursive",
-    "managers_regress",
-    "many_to_many",
-    "many_to_one",
-    "many_to_one_null",
-    "max_lengths",
-    "messages_tests",
-    "migrate_signals",
-    "migration_test_data_persistence",
-    "migrations",
-    "model_fields",
-    "model_forms",
-    "model_formsets",
-    "model_formsets_regress",
-    "model_indexes",
-    "model_inheritance",
-    "model_inheritance_regress",
-    "model_options",
-    "model_package",
-    "model_regress",
-    "model_utils",
-    "modeladmin",
-    "multiple_database",
-    "mutually_referential",
-    "nested_foreign_keys",
-    "null_fk",
-    "null_fk_ordering",
-    "null_queries",
-    "one_to_one",
-    "or_lookups",
-    "order_with_respect_to",
-    "ordering",
-    "pagination",
-    "prefetch_related",
-    "proxy_model_inheritance",
-    "proxy_models",
-    "queries",
-    "queryset_pickle",
-    "redirects_tests",
-    "reserved_names",
-    "reverse_lookup",
-    "save_delete_hooks",
-    "schema",
-    "select_for_update",
-    "select_related",
-    "select_related_onetoone",
-    "select_related_regress",
-    "serializers",
-    "servers",
-    "sessions_tests",
-    "shortcuts",
-    "signals",
-    "sitemaps_tests",
-    "sites_framework",
-    "sites_tests",
-    "string_lookup",
-    "swappable_models",
-    "syndication_tests",
-    "test_client",
-    "test_client_regress",
-    "test_runner",
-    "test_utils",
-    "timezones",
-    "transactions",
-    "unmanaged_models",
-    "update",
-    "update_only_fields",
-    "user_commands",
-    "validation",
-    "view_tests",
-    "xor_lookups",
     # Add directories in django_mongodb_backend/tests
     *sorted(
         [

.github/workflows/test-python-atlas.yml

@@ -28,7 +28,7 @@ jobs:
       - name: install django-mongodb-backend
         run: |
           pip3 install --upgrade pip
-          pip3 install -e .
+          pip3 install -e .[encryption]
       - name: Checkout Django
         uses: actions/checkout@v6
         with:
@@ -51,8 +51,15 @@ jobs:
         run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py
       - name: Start local Atlas
         working-directory: .
-        run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7
+        run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15
+      - name: Download crypt shared
+        run: |
+          wget https://downloads.mongodb.com/linux/mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz
+          tar -xvzf mongo_crypt_shared_v1-linux-x86_64-enterprise-ubuntu2404-8.2.1.tgz lib/mongo_crypt_v1.so
       - name: Run tests
         run: python3 django_repo/tests/runtests_.py
     permissions:
         contents: read
+    env:
+      DJANGO_SETTINGS_MODULE: "encrypted_settings"
+      CRYPT_SHARED_LIB_PATH: "${{ github.workspace }}/lib/mongo_crypt_v1.so"