From 1db5cfa2f5ea53705dfba0707a9d74f6222c324b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 13 Mar 2024 07:33:43 -0500 Subject: [PATCH 1/7] delay another import --- pymongo/_azure_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymongo/_azure_helpers.py b/pymongo/_azure_helpers.py index 661e4ce37a..1ce00c0603 100644 --- a/pymongo/_azure_helpers.py +++ b/pymongo/_azure_helpers.py @@ -17,12 +17,14 @@ import json from typing import Any, Optional -from urllib.request import Request, urlopen def _get_azure_response( resource: str, object_id: Optional[str] = None, timeout: float = 5 ) -> dict[str, Any]: + # Deferred import to save overall import time. + from urllib.request import Request, urlopen + url = "http://169.254.169.254/metadata/identity/oauth2/token" url += "?api-version=2018-02-01" url += f"&resource={resource}" From 4eca4973987eae56a8baadb7155b9470babcfe8c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 09:24:57 -0500 Subject: [PATCH 2/7] clean up test suite scripts and add docs --- .evergreen/run-tests.sh | 6 ----- .evergreen/scripts/run_server.py | 5 ---- .evergreen/scripts/setup_tests.py | 32 +++++++++++++++-------- .evergreen/scripts/utils.py | 43 ++++++++++++++++++++++++++----- CONTRIBUTING.md | 23 +++++++++++++++-- 5 files changed, 79 insertions(+), 30 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 6f53ced61c..04dd16d34f 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -24,12 +24,6 @@ else exit 1 fi -# Source the local secrets export file if available. -if [ -f "./secrets-export.sh" ]; then - echo "Sourcing local secrets file" - . "./secrets-export.sh" -fi - # List the packages. uv sync ${UV_ARGS} --reinstall uv pip list diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f85207daa4..5d9aa54e11 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -28,11 +28,6 @@ def start_server(): elif test_name == "load_balancer": set_env("LOAD_BALANCER") - elif test_name == "auth_oidc": - raise ValueError( - "OIDC auth does not use run-orchestration directly, do not use run-server!" - ) - elif test_name == "ocsp": opts.ssl = True if "ORCHESTRATION_FILE" not in os.environ: diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 17f9de1a71..33d774ab56 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -115,8 +115,16 @@ def setup_libmongocrypt(): run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") +def load_config_from_file(path: str | Path): + config = read_env(path) + for key, value in config.items(): + write_env(key, value) + + def get_secrets(name: str) -> None: - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh {name}") + secrets_dir = Path("{DRIVERS_TOOLS}/.evergreen/secrets_handling") + run_command(f"bash ${secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) + load_config_from_file(secrets_dir / "secrets-export.sh") def handle_test_env() -> None: @@ -158,7 +166,8 @@ def handle_test_env() -> None: # Handle pass through env vars. for var in PASS_THROUGH_ENV: - if is_set(var): + opt_name = var.lower().replace("_", "-") + if is_set(var) or getattr(opts, opt_name): write_env(var, os.environ[var]) if extra := EXTRAS_MAP.get(test_name, ""): @@ -233,12 +242,11 @@ def handle_test_env() -> None: if is_set("MONGODB_URI"): write_env("PYMONGO_MUST_CONNECT", "true") - if is_set("DISABLE_TEST_COMMANDS"): + if is_set("DISABLE_TEST_COMMANDS") or opts.disable_test_commands: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if test_name == "enterprise_auth": get_secrets("drivers/enterprise_auth") - config = read_env(f"{ROOT}/secrets-export.sh") if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", config["SASL_PASS"]) @@ -316,7 +324,7 @@ def handle_test_env() -> None: write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") - compressors = os.environ.get("COMPRESSORS") + compressors = os.environ.get("COMPRESSORS") or opts.compressor if compressors == "snappy": UV_ARGS.append("--extra snappy") elif compressors == "zstd": @@ -349,13 +357,15 @@ def handle_test_env() -> None: if test_name == "encryption": if not DRIVERS_TOOLS: raise RuntimeError("Missing DRIVERS_TOOLS") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh") - run_command(f"bash {DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh") + csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") + run_command(f"bash {csfle_dir}/setup-secrets.sh", cwd=csfle_dir) + load_config_from_file(csfle_dir / "secrets-export.sh") + run_command(f"bash {csfle_dir}/start-servers.sh") if sub_test_name == "pyopenssl": UV_ARGS.append("--extra ocsp") - if is_set("TEST_CRYPT_SHARED"): + if is_set("TEST_CRYPT_SHARED") or opts.crypt_shared: config = read_env(f"{DRIVERS_TOOLS}/mo-expansion.sh") CRYPT_SHARED_DIR = Path(config["CRYPT_SHARED_LIB_PATH"]).parent.as_posix() LOGGER.info("Using crypt_shared_dir %s", CRYPT_SHARED_DIR) @@ -414,15 +424,15 @@ def handle_test_env() -> None: # Add coverage if requested. # Only cover CPython. PyPy reports suspiciously low coverage. - if is_set("COVERAGE") and platform.python_implementation() == "CPython": + if (is_set("COVERAGE") or opts.cov) and platform.python_implementation() == "CPython": # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") TEST_ARGS = f"{TEST_ARGS} --cov" write_env("COVERAGE") - if is_set("GREEN_FRAMEWORK"): - framework = os.environ["GREEN_FRAMEWORK"] + if is_set("GREEN_FRAMEWORK") or opts.green_framework: + framework = opts.green_framework or os.environ["GREEN_FRAMEWORK"] UV_ARGS.append(f"--group {framework}") else: diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 0ff3b76a5f..dc009f74de 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -52,7 +52,10 @@ class Distro: # Tests that require a sub test suite. SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"] -EXTRA_TESTS = ["mod_wsgi", "aws_lambda", "search_index"] +EXTRA_TESTS = ["mod_wsgi", "aws_lambda"] + +# Tests that do not use run-orchestration. +NO_RUN_ORCHESTRATION = ["auth_oidc", "atlas_connect", "data_lake", "mockupdb", "serverless"] def get_test_options( @@ -75,19 +78,47 @@ def get_test_options( else: parser.add_argument( "test_name", - choices=sorted(TEST_SUITE_MAP), + choices=set(TEST_SUITE_MAP) - set(NO_RUN_ORCHESTRATION), nargs="?", default="default", help="The optional name of the test suite to be run, which informs the server configuration.", ) parser.add_argument( - "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level" + "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level." ) parser.add_argument( - "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level" + "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level." ) - parser.add_argument("--auth", action="store_true", help="Whether to add authentication") - parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration") + parser.add_argument("--auth", action="store_true", help="Whether to add authentication.") + parser.add_argument("--ssl", action="store_true", help="Whether to add TLS configuration.") + + # Add the test modifiers. + if require_sub_test_name: + parser.add_argument( + "--debug-log", action="store_true", help="Enable pymongo standard logging." + ) + parser.add_argument("--cov", action="store_true", help="Add test coverage.") + parser.add_argument( + "--green-framework", + nargs=1, + choices=["eventlet", "gevent"], + help="Optional green framework to test against.", + ) + parser.add_argument( + "--compressor", + nargs=1, + choices=["zlib", "zstd", "snappy"], + help="Optional compression algorithm.", + ) + parser.add_argument("--crypt-shared", action="store_true", help="Test with crypt_shared.") + parser.add_argument("--no-ext", action="store_true", help="Run without c extensions.") + parser.add_argument( + "--mongodb-api-version", choices=["1"], help="MongoDB stable API version to use." + ) + parser.add_argument( + "--disable-test-commands", action="store_true", help="Disable test commands." + ) + # Get the options. if not allow_extra_opts: opts, extra_opts = parser.parse_args(), [] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e75510171e..d8cc8c8bcd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -217,9 +217,11 @@ the pages will re-render and the browser will automatically refresh. ### Usage -- Run `just run-server` with optional args to set up the server. - All given flags will be passed to `run-orchestration.sh` in `$DRIVERS_TOOLS`. +- Run `just run-server` with optional args to set up the server. All given options will be passed to + `run-orchestration.sh` in `$DRIVERS_TOOLS`. See `$DRIVERS_TOOLS/evergreen/run-orchestration.sh -h` + for a full list of options. - Run `just setup-tests` with optional args to set up the test environment, secrets, etc. + See `just setup-tests -h` for a full list of available options. - Run `just run-tests` to run the tests in an appropriate Python environment. - When done, run `just teardown-tests` to clean up and `just stop-server` to stop the server. @@ -346,11 +348,28 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Run the tests: `just run-tests`. ## Enable Debug Logs + - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest`. - Add `log_cli_level = "DEBUG` and `log_cli = 1` to the `tool.pytest.ini_options` section in `pyproject.toml` for Evergreen patches or to enable debug logs by default on your machine. - You can also set `DEBUG_LOG=1` and run either `just setup-tests` or `just-test`. +- Finally, you can use `just setup-tests --debug-log`. - For evergreen patch builds, you can use `evergreen patch --param DEBUG_LOG=1` to enable debug logs for the patch. +## Adding a new test suite + +- If adding new tests files that should only be run for that test suite, add a pytest marker to the file and add + to the list of pytest markers in `pyproject.toml`. Then add the test suite to the `TEST_SUITE_MAP` in `.evergreen/scripts/utils.py`. If for some reason it is not a pytest-runnable test, add it to the list of `EXTRA_TESTS` instead. +- If the test uses Atlas or otherwise doesn't use `run-orchestration.sh`, add it to the `NO_RUN_ORCHESTRATION` list in + `.evergreen/scripts/utils.py`. +- If there is something special required to run the local server or there is an extra flag that should always be set + like `AUTH`, add that logic to `.evergreen/scripts/run_server.py`. +- The bulk of the logic will typically be in `.evergreen/scripts/setup_tests.py`. This is where you should fetch secrets and make them available using `write_env`, start services, and write other env vars needed using `write_env`. +- If there are any special test considerations, including not running `pytest` at all, handle it in `.evergreen/scripts/run_tests.py`. +- If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. +- Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. +- Regenerate the test variants and tasks using the instructions in `.evergreen/scripts/generate_config.py`. +- Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. + ## Re-sync Spec Tests If you would like to re-sync the copy of the specification tests in the From 45fc3386cee332fe2b6063a6cbfca803ed7b75b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 09:33:31 -0500 Subject: [PATCH 3/7] fix pass through handling --- .evergreen/scripts/setup_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 33d774ab56..b26259c94a 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -166,8 +166,7 @@ def handle_test_env() -> None: # Handle pass through env vars. for var in PASS_THROUGH_ENV: - opt_name = var.lower().replace("_", "-") - if is_set(var) or getattr(opts, opt_name): + if is_set(var) or getattr(opts, var): write_env(var, os.environ[var]) if extra := EXTRAS_MAP.get(test_name, ""): From a30b6d125ca2fe3dbfbaff5db440028817e021f8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 10:36:28 -0500 Subject: [PATCH 4/7] fix pass through handling --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index b26259c94a..22537c4fec 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -166,7 +166,7 @@ def handle_test_env() -> None: # Handle pass through env vars. for var in PASS_THROUGH_ENV: - if is_set(var) or getattr(opts, var): + if is_set(var) or getattr(opts, var.lower()): write_env(var, os.environ[var]) if extra := EXTRAS_MAP.get(test_name, ""): From a5a25e2b260d369d1443455b1bf366999a86c96a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 11:15:07 -0500 Subject: [PATCH 5/7] fix get_secrets --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 22537c4fec..be338e930e 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -122,7 +122,7 @@ def load_config_from_file(path: str | Path): def get_secrets(name: str) -> None: - secrets_dir = Path("{DRIVERS_TOOLS}/.evergreen/secrets_handling") + secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling") run_command(f"bash ${secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) load_config_from_file(secrets_dir / "secrets-export.sh") From e771ee2f9bcd452b9d1dd41f47fb123d0f75cc23 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 11:17:50 -0500 Subject: [PATCH 6/7] fix get_secrets --- .evergreen/scripts/setup_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index be338e930e..68e85346c6 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -123,7 +123,7 @@ def load_config_from_file(path: str | Path): def get_secrets(name: str) -> None: secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling") - run_command(f"bash ${secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) + run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) load_config_from_file(secrets_dir / "secrets-export.sh") From 81d3ae947e9f28615309b72940a9ebd13617c8b2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 22 Mar 2025 11:25:46 -0500 Subject: [PATCH 7/7] fix get_secrets --- .evergreen/scripts/setup_tests.py | 9 +++++---- .evergreen/scripts/utils.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 68e85346c6..74971bca76 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -115,16 +115,17 @@ def setup_libmongocrypt(): run_command("chmod +x libmongocrypt/nocrypto/bin/mongocrypt.dll") -def load_config_from_file(path: str | Path): +def load_config_from_file(path: str | Path) -> dict[str, str]: config = read_env(path) for key, value in config.items(): write_env(key, value) + return config -def get_secrets(name: str) -> None: +def get_secrets(name: str) -> dict[str, str]: secrets_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/secrets_handling") run_command(f"bash {secrets_dir}/setup-secrets.sh {name}", cwd=secrets_dir) - load_config_from_file(secrets_dir / "secrets-export.sh") + return load_config_from_file(secrets_dir / "secrets-export.sh") def handle_test_env() -> None: @@ -245,7 +246,7 @@ def handle_test_env() -> None: write_env("PYMONGO_DISABLE_TEST_COMMANDS", "1") if test_name == "enterprise_auth": - get_secrets("drivers/enterprise_auth") + config = get_secrets("drivers/enterprise_auth") if PLATFORM == "windows": LOGGER.info("Setting GSSAPI_PASS") write_env("GSSAPI_PASS", config["SASL_PASS"]) diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index dc009f74de..3eb44f2ab9 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -144,7 +144,7 @@ def get_test_options( return opts, extra_opts -def read_env(path: Path | str) -> dict[str, Any]: +def read_env(path: Path | str) -> dict[str, str]: config = dict() with Path(path).open() as fid: for line in fid.readlines():