From cbc2db80db94c2afbc9a51936a2d9260e5b69361 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 13:56:27 -0600 Subject: [PATCH 01/18] PYTHON-4975 Use justfile as the task runner --- .evergreen/scripts/ensure-hatch.sh | 98 +++++++++++++++--------------- .github/workflows/test-python.yml | 24 ++++---- CONTRIBUTING.md | 35 +++++------ hatch.toml | 13 ---- justfile | 64 +++++++++++++++++++ 5 files changed, 144 insertions(+), 90 deletions(-) create mode 100644 justfile diff --git a/.evergreen/scripts/ensure-hatch.sh b/.evergreen/scripts/ensure-hatch.sh index e63d98bb6d..76eb2b7ab7 100755 --- a/.evergreen/scripts/ensure-hatch.sh +++ b/.evergreen/scripts/ensure-hatch.sh @@ -5,55 +5,57 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) pushd "$(dirname "$(dirname $HERE)")" > /dev/null -# Ensure hatch is available. -if [ ! -x "$(command -v hatch)" ]; then - # Install a virtual env with "hatch" - # Ensure there is a python venv. - . .evergreen/utils.sh - - if [ -z "${PYTHON_BINARY:-}" ]; then - PYTHON_BINARY=$(find_python3) - fi - VENV_DIR=.venv - if [ ! -d $VENV_DIR ]; then - echo "Creating virtual environment..." - createvirtualenv "$PYTHON_BINARY" .venv - echo "Creating virtual environment... done." - fi - if [ -f $VENV_DIR/Scripts/activate ]; then - . $VENV_DIR/Scripts/activate - else - . $VENV_DIR/bin/activate - fi +# Ensure hatch is available locally. +VENV_DIR=.venv +if [ -f $VENV_DIR/Scripts/hatch ]; then + return 0 +fi +if [ -f $VENV_DIR/bin/hatch ]; then + return 0 +fi + +# Install a virtual env with "hatch" +# Ensure there is a python venv. +. .evergreen/utils.sh - python --version - - echo "Installing hatch..." - python -m pip install -U pip - python -m pip install hatch || { - # Install rust and try again. - CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} - # Handle paths on Windows. - if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - CARGO_HOME=$(cygpath -m $CARGO_HOME) - fi - export RUSTUP_HOME="${CARGO_HOME}/.rustup" - ${DRIVERS_TOOLS}/.evergreen/install-rust.sh - source "${CARGO_HOME}/env" - python -m pip install hatch - } - # Ensure hatch does not write to user or global locations. - touch hatch_config.toml - HATCH_CONFIG=$(pwd)/hatch_config.toml +if [ -z "${PYTHON_BINARY:-}" ]; then + PYTHON_BINARY=$(find_python3) +fi + +if [ ! -d $VENV_DIR ]; then + echo "Creating virtual environment..." + createvirtualenv "$PYTHON_BINARY" .venv + echo "Creating virtual environment... done." +fi +if [ -f $VENV_DIR/Scripts/activate ]; then + . $VENV_DIR/Scripts/activate +else + . $VENV_DIR/bin/activate +fi + +echo "Installing hatch..." +python -m pip install -U pip +python -m pip install hatch || { + # Install rust and try again. + CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} + # Handle paths on Windows. if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") + CARGO_HOME=$(cygpath -m $CARGO_HOME) fi - export HATCH_CONFIG - hatch config restore - hatch config set dirs.data "$(pwd)/.hatch/data" - hatch config set dirs.cache "$(pwd)/.hatch/cache" - - echo "Installing hatch... done." + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + source "${CARGO_HOME}/env" + python -m pip install hatch +} +# Ensure hatch does not write to user or global locations. +touch hatch_config.toml +HATCH_CONFIG=$(pwd)/hatch_config.toml +if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin + HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") fi -hatch --version -popd > /dev/null +export HATCH_CONFIG +hatch config restore +hatch config set dirs.data "$(pwd)/.hatch/data" +hatch config set dirs.cache "$(pwd)/.hatch/cache" + +echo "Installing hatch... done." diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 2310b7698d..5d3f7665a8 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -29,10 +29,10 @@ jobs: cache-dependency-path: 'pyproject.toml' - name: Install Python dependencies run: | - python -m pip install -U pip hatch + python -m pip install -U pip hatch rust-just - name: Run linters run: | - hatch run lint:run-manual + just-manual - name: Run compilation run: | export PYMONGO_C_EXT_MUST_BUILD=1 @@ -40,7 +40,7 @@ jobs: python tools/fail_if_no_c.py - name: Run typecheck run: | - hatch run typing:check + just typing - run: | sudo apt-get install -y cppcheck - run: | @@ -75,7 +75,7 @@ jobs: allow-prereleases: true - name: Install dependencies run: | - pip install -U pip + pip install -U pip rust-just if [[ "${{ matrix.python-version }}" == "3.13" ]]; then pip install --pre cffi setuptools pip install --no-build-isolation hatch @@ -95,7 +95,7 @@ jobs: if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then pytest -v --durations=5 --maxfail=10 else - hatch run test:test + just test fi doctest: @@ -113,14 +113,14 @@ jobs: cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | - pip install -U hatch pip + pip install -U hatch pip rust-just - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 with: mongodb-version: '8.0.0-rc4' - name: Run tests run: | - hatch run doctest:test + just doctest docs: name: Docs Checks @@ -137,10 +137,10 @@ jobs: python-version: '3.9' - name: Install dependencies run: | - pip install -U pip hatch + pip install -U pip hatch rust-just - name: Build docs run: | - hatch run doc:build + just docs linkcheck: name: Link Check @@ -160,7 +160,7 @@ jobs: pip install -U pip hatch - name: Build docs run: | - hatch run doc:linkcheck + just docs-linkcheck typing: name: Typing Tests @@ -179,10 +179,10 @@ jobs: cache-dependency-path: 'pyproject.toml' - name: Install dependencies run: | - pip install -U pip hatch + pip install -U pip hatch rust-just - name: Run typecheck run: | - hatch run typing:check + just typing make_sdist: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 814e040048..11a04e8190 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,9 +28,10 @@ including 4 space indents and 79 character line limits. - Avoid backward breaking changes if at all possible. - Write inline documentation for new classes and methods. -- We use [hatch](https://hatch.pypa.io/dev/) for our script runner and packaging tool. +- We use [hatch](https://hatch.pypa.io/dev/) for python environment management and packaging. +- We use [just](https://just.systems/man/en/) as our task runner. - Write tests and make sure they pass (make sure you have a mongod - running on the default port, then execute `hatch run test:test` from the cmd + running on the default port, then execute `just test` from the cmd line to run the test suite). - Add yourself to doc/contributors.rst `:)` @@ -148,17 +149,17 @@ To run `pre-commit` manually, run: pre-commit run --all-files ``` -To run a manual hook like `mypy` manually, run: +To run a manual hook like `ruff` manually, run: ```bash -pre-commit run --all-files --hook-stage manual mypy +pre-commit run --all-files --hook-stage manual ruff ``` -Typically we use `hatch` to run the linters, e.g. +Typically we use `just` to run the linters, e.g. ```bash -hatch run typing:check-mypy -hatch run lint:build-manual +just typing-mypy +just run lint-manual ``` ## Documentation @@ -176,13 +177,13 @@ documentation including narrative docs, and the [Sphinx docstring format](https: You can build the documentation locally by running: ```bash -hatch run doc:build +just docs-build ``` When updating docs, it can be helpful to run the live docs server as: ```bash -hatch run doc:serve +just docs-serve ``` Browse to the link provided, and then as you make changes to docstrings or narrative docs, @@ -192,13 +193,13 @@ the pages will re-render and the browser will automatically refresh. ## Running Tests Locally - Ensure you have started the appropriate Mongo Server(s). -- Run `pip install hatch` to use `hatch` for testing or run - `pip install -e ".[test]"` to run `pytest` directly. -- Run `hatch run test:test` or `pytest` to run all of the tests. +- Run `just install` to set up `hatch` in a local virtual environment, or you can manually + create a virtual environment and run `pytest` directly. +- Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class and the `` to test a full module. For example: - `hatch run test:test -- test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`. + `just test test/test_change_stream.py::TestUnifiedChangeStreamsErrors::test_change_stream_errors_on_ElectionInProgress`. - Use the `-k` argument to select tests by pattern. ## Running Load Balancer Tests Locally @@ -211,15 +212,15 @@ the pages will re-render and the browser will automatically refresh. - Start the load balancer using: `MONGODB_URI='mongodb://localhost:27017,localhost:27018/' $PWD/drivers-evergreen-tools/.evergreen/run-load-balancer.sh start`. - Run the tests from the `pymongo` checkout directory using: - `TEST_LOADBALANCER=1 hatch run test:test-eg`. + `TEST_LOADBALANCER=1 just test-eg`. ## Running Encryption Tests Locally - Clone `drivers-evergreen-tools`: `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. - Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools` -- Run `AWS_PROFILE= hatch run encryption:setup` after setting up your AWS profile with `aws configure sso`. -- Run the tests with `TEST_ENCRYPTION=1 hatch run test:test-eg`. -- When done, run `hatch run encryption:teardown` to clean up. +- Run `AWS_PROFILE= just setup-encryption` after setting up your AWS profile with `aws configure sso`. +- Run the tests with `TEST_ENCRYPTION=1 just test-eg`. +- When done, run `just teardown-encryption` to clean up. ## Re-sync Spec Tests diff --git a/hatch.toml b/hatch.toml index 60bd0af014..15d0f25f07 100644 --- a/hatch.toml +++ b/hatch.toml @@ -30,13 +30,6 @@ check-strict-pyright = [ ] check = ["check-mypy", "check-pyright", "check-strict-pyright"] -[envs.lint] -skip-install = true -dependencies = ["pre-commit"] -[envs.lint.scripts] -run = "pre-commit run --all-files" -run-manual = "pre-commit run --all-files --hook-stage manual" - [envs.test] features = ["test"] [envs.test.scripts] @@ -44,9 +37,3 @@ test = "pytest -v --durations=5 --maxfail=10 {args}" test-eg = "bash ./.evergreen/run-tests.sh {args}" test-async = "pytest -v --durations=5 --maxfail=10 -m default_async {args}" test-mockupdb = ["pip install -U git+https://github.com/mongodb-labs/mongo-mockup-db@master", "test -m mockupdb"] - -[envs.encryption] -skip-install = true -[envs.encryption.scripts] -setup = "bash .evergreen/setup-encryption.sh" -teardown = "bash .evergreen/teardown-encryption.sh" diff --git a/justfile b/justfile new file mode 100644 index 0000000000..b53ba2354c --- /dev/null +++ b/justfile @@ -0,0 +1,64 @@ +# See https://just.systems/man/en/ for instructions +set shell := ["bash", "-c"] +set dotenv-load +set dotenv-filename := "./.evergreen/scripts/env.sh" + +# Make the default recipe private so it doesn't show up in the list. +[private] +default: + @just --list + +install: + bash .evergreen/scripts/ensure-hatch.sh + +[group('docs')] +docs-build: + source .evergreen/scripts/ensure-hatch.sh && hatch run doc:build + +[group('docs')] +docs-serve: + source .evergreen/scripts/ensure-hatch.sh && hatch run doc:serve + +[group('docs')] +docs-linkcheck: + hatch run doc:linkcheck + +[group('docs')] +docs-test: + source .evergreen/scripts/ensure-hatch.sh && hatch run doctest:test + +[group('typing')] +typing: + source .evergreen/scripts/ensure-hatch.sh && hatch run typing:check + +[group('typing')] +typing-mypy: + source .evergreen/scripts/ensure-hatch.sh && hatch run typing:mypy + +[group('lint')] +lint: + pre-commit run --all-files + +[group('lint')] +lint-manual: + pre-commit run --all-files --hook-stage manual + +[group('test')] +test *args: + source .evergreen/scripts/ensure-hatch.sh && hatch run test:test {{args}} + +[group('test')] +test-mockupdb: + source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-mockupdb + +[group('test')] +test-eg *args: + source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-eg {{args}} + +[group('encryption')] +setup-encryption: + bash .evergreen/setup-encryption.sh + +[group('encryption')] +teardown-encryption: + bash .evergreen/teardown-encryption.sh From 7f99c3bdc9664ae028c2efbfbb5d5921e86518bd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 14:10:42 -0600 Subject: [PATCH 02/18] fix reactivation --- .evergreen/scripts/ensure-hatch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/scripts/ensure-hatch.sh b/.evergreen/scripts/ensure-hatch.sh index 76eb2b7ab7..81cff6f186 100755 --- a/.evergreen/scripts/ensure-hatch.sh +++ b/.evergreen/scripts/ensure-hatch.sh @@ -8,9 +8,11 @@ pushd "$(dirname "$(dirname $HERE)")" > /dev/null # Ensure hatch is available locally. VENV_DIR=.venv if [ -f $VENV_DIR/Scripts/hatch ]; then + . $VENV_DIR/Scripts/activate return 0 fi if [ -f $VENV_DIR/bin/hatch ]; then + . $VENV_DIR/bin/activate return 0 fi From b249dee68f730011c019a5ea1d134d336f34ce20 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 14:14:42 -0600 Subject: [PATCH 03/18] fix encryption handling --- .evergreen/scripts/cleanup.sh | 2 +- .evergreen/scripts/setup-encryption.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index 9e583e4f1e..a1fd92f04d 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then - . .evergreen/hatch.sh encryption:teardown + bash .evergreen/teardown-encryption.sh fi rm -rf "${DRIVERS_TOOLS}" || true rm -f ./secrets-export.sh || true diff --git a/.evergreen/scripts/setup-encryption.sh b/.evergreen/scripts/setup-encryption.sh index 2f167cd20b..5b73240205 100755 --- a/.evergreen/scripts/setup-encryption.sh +++ b/.evergreen/scripts/setup-encryption.sh @@ -1,5 +1,5 @@ #!/bin/bash if [ -n "${test_encryption}" ]; then - ./.evergreen/hatch.sh encryption:setup + bash .evergreen/setup-encryption.sh fi From 42e60b11504838f2bc6557abb81034f2eeed3c18 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 15:24:07 -0600 Subject: [PATCH 04/18] fix invocation --- .github/workflows/test-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5d3f7665a8..ddb40beabc 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -32,7 +32,7 @@ jobs: python -m pip install -U pip hatch rust-just - name: Run linters run: | - just-manual + just lint-manual - name: Run compilation run: | export PYMONGO_C_EXT_MUST_BUILD=1 From ed041ae5d4d5cb33179edc5d188e7d4f20bd0599 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 15:28:10 -0600 Subject: [PATCH 05/18] cleanup invocations --- .github/workflows/test-python.yml | 6 +++--- justfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index ddb40beabc..9a0c033d90 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -29,7 +29,7 @@ jobs: cache-dependency-path: 'pyproject.toml' - name: Install Python dependencies run: | - python -m pip install -U pip hatch rust-just + python -m pip install -U pip hatch rust-just pre-commit - name: Run linters run: | just lint-manual @@ -120,7 +120,7 @@ jobs: mongodb-version: '8.0.0-rc4' - name: Run tests run: | - just doctest + just docs-test docs: name: Docs Checks @@ -157,7 +157,7 @@ jobs: python-version: '3.9' - name: Install dependencies run: | - pip install -U pip hatch + pip install -U pip hatch rust-just - name: Build docs run: | just docs-linkcheck diff --git a/justfile b/justfile index b53ba2354c..c679e12c58 100644 --- a/justfile +++ b/justfile @@ -12,7 +12,7 @@ install: bash .evergreen/scripts/ensure-hatch.sh [group('docs')] -docs-build: +docs: source .evergreen/scripts/ensure-hatch.sh && hatch run doc:build [group('docs')] From 3651496f291416b7a588c6ccdb39f8fb2a4d57fc Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 09:00:19 -0600 Subject: [PATCH 06/18] Simplify justfile usage --- .evergreen/hatch.sh | 2 +- .evergreen/scripts/configure-env.sh | 4 +- .evergreen/scripts/ensure-hatch.sh | 63 -------------------------- .evergreen/scripts/setup-dev-env.sh | 70 +++++++++++++++++++++++++++++ .evergreen/setup-spawn-host.sh | 2 +- .evergreen/utils.sh | 4 +- .github/workflows/test-python.yml | 31 +++++++------ .gitignore | 2 + CONTRIBUTING.md | 4 +- justfile | 29 +++++++----- 10 files changed, 117 insertions(+), 94 deletions(-) delete mode 100755 .evergreen/scripts/ensure-hatch.sh create mode 100755 .evergreen/scripts/setup-dev-env.sh diff --git a/.evergreen/hatch.sh b/.evergreen/hatch.sh index c01dfcd19e..9bfc930382 100755 --- a/.evergreen/hatch.sh +++ b/.evergreen/hatch.sh @@ -1,5 +1,5 @@ #!/bin/bash set -eu -. .evergreen/scripts/ensure-hatch.sh +. .evergreen/scripts/setup-dev-env.sh hatch run "$@" diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index e0c845a333..5c7dff82f7 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -14,12 +14,14 @@ fi PROJECT_DIRECTORY="$(pwd)" DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} +HATCH_CONFIG=$PROJECT_DIRECTORY/hatch_config.toml # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY) CARGO_HOME=$(cygpath -m $CARGO_HOME) + HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") fi SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts" @@ -34,7 +36,6 @@ export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" cat < "$SCRIPT_DIR"/env.sh -set -o errexit export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export CURRENT_VERSION="$CURRENT_VERSION" export SKIP_LEGACY_SHELL=1 @@ -59,6 +60,7 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" +export HATCH_CONFIG="$HATCH_CONFIG" export PATH="$MONGODB_BINARIES:$PATH" # shellcheck disable=SC2154 export PROJECT="${project:-mongo-python-driver}" diff --git a/.evergreen/scripts/ensure-hatch.sh b/.evergreen/scripts/ensure-hatch.sh deleted file mode 100755 index 81cff6f186..0000000000 --- a/.evergreen/scripts/ensure-hatch.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -set -eu - -HERE=$(dirname ${BASH_SOURCE:-$0}) -pushd "$(dirname "$(dirname $HERE)")" > /dev/null - -# Ensure hatch is available locally. -VENV_DIR=.venv -if [ -f $VENV_DIR/Scripts/hatch ]; then - . $VENV_DIR/Scripts/activate - return 0 -fi -if [ -f $VENV_DIR/bin/hatch ]; then - . $VENV_DIR/bin/activate - return 0 -fi - -# Install a virtual env with "hatch" -# Ensure there is a python venv. -. .evergreen/utils.sh - -if [ -z "${PYTHON_BINARY:-}" ]; then - PYTHON_BINARY=$(find_python3) -fi - -if [ ! -d $VENV_DIR ]; then - echo "Creating virtual environment..." - createvirtualenv "$PYTHON_BINARY" .venv - echo "Creating virtual environment... done." -fi -if [ -f $VENV_DIR/Scripts/activate ]; then - . $VENV_DIR/Scripts/activate -else - . $VENV_DIR/bin/activate -fi - -echo "Installing hatch..." -python -m pip install -U pip -python -m pip install hatch || { - # Install rust and try again. - CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} - # Handle paths on Windows. - if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - CARGO_HOME=$(cygpath -m $CARGO_HOME) - fi - export RUSTUP_HOME="${CARGO_HOME}/.rustup" - ${DRIVERS_TOOLS}/.evergreen/install-rust.sh - source "${CARGO_HOME}/env" - python -m pip install hatch -} -# Ensure hatch does not write to user or global locations. -touch hatch_config.toml -HATCH_CONFIG=$(pwd)/hatch_config.toml -if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin - HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG") -fi -export HATCH_CONFIG -hatch config restore -hatch config set dirs.data "$(pwd)/.hatch/data" -hatch config set dirs.cache "$(pwd)/.hatch/cache" - -echo "Installing hatch... done." diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh new file mode 100755 index 0000000000..305a66622c --- /dev/null +++ b/.evergreen/scripts/setup-dev-env.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -eu + +HERE=$(dirname ${BASH_SOURCE:-$0}) +pushd "$(dirname "$(dirname $HERE)")" > /dev/null + +# Source the env file to pick up common variables. +if [ -f $HERE/scripts/env.sh ]; then + source $HERE/scripts/env.sh +fi + +# Set the location of the python bin dir. +if [ "Windows_NT" = "${OS:-}" ]; then + BIN_DIR=.venv/Scripts +else + BIN_DIR=.venv/bin +fi + +# Ensure there is a python venv. +if [ ! -d $BIN_DIR ]; then + . .evergreen/utils.sh + + if [ -z "${PYTHON_BINARY:-}" ]; then + PYTHON_BINARY=$(find_python3) + fi + + echo "Creating virtual environment..." + createvirtualenv "$PYTHON_BINARY" .venv + echo "Creating virtual environment... done." +fi + +# Activate the virtual env. +. $BIN_DIR/activate + +# Ensure there is a local hatch. +if [ ! -f $BIN_DIR/hatch ]; then + echo "Installing hatch..." + python -m pip install hatch || { + # CARGO_HOME is defined in configure-env.sh + export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + source "${CARGO_HOME}/env" + python -m pip install hatch + } + echo "Installing hatch... done." +fi + +# Ensure hatch does not write to user or global locations. +HATCH_CONFIG=${HATCH_CONFIG:-hatch_config.toml} +if [ ! -f ${HATCH_CONFIG} ]; then + touch hatch_config.toml + hatch config restore + hatch config set dirs.data "$(pwd)/.hatch/data" + hatch config set dirs.cache "$(pwd)/.hatch/cache" +fi + +# Ensure there is a local pre-commit. +if [ ! -f $BIN_DIR/pre-commit ]; then + python -m pip install pre-commit +fi + +# Ensure the pre-commit hook is installed. +if [ ! -f .git/hooks/pre-commit ]; then + pre-commit install +fi + +# Install pymongo and its test deps. +python -m pip install ".[test]" diff --git a/.evergreen/setup-spawn-host.sh b/.evergreen/setup-spawn-host.sh index 4109e59183..b0e7abddbd 100755 --- a/.evergreen/setup-spawn-host.sh +++ b/.evergreen/setup-spawn-host.sh @@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_ echo "Copying files to $target... done" ssh $target $remote_dir/.evergreen/scripts/setup-system.sh -ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/ensure-hatch.sh" +ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/setup-dev-env.sh" diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index d3af2dcc7a..e044b3d766 100755 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -1,6 +1,6 @@ -#!/bin/bash -ex +#!/bin/bash -set -o xtrace +set -eu find_python3() { PYTHON="" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 9a0c033d90..5c062f9ebf 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -27,9 +27,11 @@ jobs: python-version: "3.9" cache: 'pip' cache-dependency-path: 'pyproject.toml' + - name: Install just + uses: extractions/setup-just@v2 - name: Install Python dependencies run: | - python -m pip install -U pip hatch rust-just pre-commit + just install - name: Run linters run: | just lint-manual @@ -73,18 +75,16 @@ jobs: cache: 'pip' cache-dependency-path: 'pyproject.toml' allow-prereleases: true + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip rust-just - if [[ "${{ matrix.python-version }}" == "3.13" ]]; then - pip install --pre cffi setuptools - pip install --no-build-isolation hatch - elif [[ "${{ matrix.python-version }}" == "3.13t" ]]; then + if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then # Hatch can't be installed on 3.13t, use pytest directly. pip install . pip install -r requirements/test.txt else - pip install hatch + just install fi - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 @@ -111,9 +111,8 @@ jobs: python-version: "3.9" cache: 'pip' cache-dependency-path: 'pyproject.toml' - - name: Install dependencies - run: | - pip install -U hatch pip rust-just + - name: Install just + uses: extractions/setup-just@v2 - name: Start MongoDB uses: supercharge/mongodb-github-action@1.12.0 with: @@ -135,9 +134,11 @@ jobs: cache-dependency-path: 'pyproject.toml' # Build docs on lowest supported Python for furo python-version: '3.9' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch rust-just + just install - name: Build docs run: | just docs @@ -155,9 +156,11 @@ jobs: cache-dependency-path: 'pyproject.toml' # Build docs on lowest supported Python for furo python-version: '3.9' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch rust-just + just install - name: Build docs run: | just docs-linkcheck @@ -177,9 +180,11 @@ jobs: python-version: "${{matrix.python}}" cache: 'pip' cache-dependency-path: 'pyproject.toml' + - name: Install just + uses: extractions/setup-just@v2 - name: Install dependencies run: | - pip install -U pip hatch rust-just + just install - name: Run typecheck run: | just typing diff --git a/.gitignore b/.gitignore index e4587125e8..01f896d316 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ libmongocrypt/ libmongocrypt_git/ hatch_config.toml .venv +expansion.yml +.evergreen/scripts/env.sh # Lambda temp files test/lambda/.aws-sam diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11a04e8190..5a46151760 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,6 +158,7 @@ pre-commit run --all-files --hook-stage manual ruff Typically we use `just` to run the linters, e.g. ```bash +just install # this will install a venv with pre-commit installed, and install the pre-commit hook. just typing-mypy just run lint-manual ``` @@ -194,7 +195,8 @@ the pages will re-render and the browser will automatically refresh. - Ensure you have started the appropriate Mongo Server(s). - Run `just install` to set up `hatch` in a local virtual environment, or you can manually - create a virtual environment and run `pytest` directly. + create a virtual environment and run `pytest` directly. If you want to use a specific + version of Python, remove the `.venv` folder and set `PYTHON_BINARY` before running `just install`. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class diff --git a/justfile b/justfile index c679e12c58..23f0993c6b 100644 --- a/justfile +++ b/justfile @@ -3,57 +3,62 @@ set shell := ["bash", "-c"] set dotenv-load set dotenv-filename := "./.evergreen/scripts/env.sh" +# Handle cross-platform paths to local python cli tools. +python_bin_dir := if os_family() == "windows" { "./.venv/Scripts" } else { "./.venv/bin" } +hatch_bin := python_bin_dir + "/hatch" +pre_commit_bin := python_bin_dir + "/pre-commit" + # Make the default recipe private so it doesn't show up in the list. [private] default: @just --list install: - bash .evergreen/scripts/ensure-hatch.sh + bash .evergreen/scripts/setup-dev-env.sh [group('docs')] docs: - source .evergreen/scripts/ensure-hatch.sh && hatch run doc:build + {{hatch_bin}} run doc:build [group('docs')] docs-serve: - source .evergreen/scripts/ensure-hatch.sh && hatch run doc:serve + {{hatch_bin}} run doc:serve [group('docs')] docs-linkcheck: - hatch run doc:linkcheck + {{hatch_bin}} run doc:linkcheck [group('docs')] docs-test: - source .evergreen/scripts/ensure-hatch.sh && hatch run doctest:test + {{hatch_bin}} run doctest:test [group('typing')] typing: - source .evergreen/scripts/ensure-hatch.sh && hatch run typing:check + {{hatch_bin}} run typing:check [group('typing')] typing-mypy: - source .evergreen/scripts/ensure-hatch.sh && hatch run typing:mypy + {{hatch_bin}} run typing:mypy [group('lint')] lint: - pre-commit run --all-files + {{pre_commit_bin}} run --all-files [group('lint')] lint-manual: - pre-commit run --all-files --hook-stage manual + {{pre_commit_bin}} run --all-files --hook-stage manual [group('test')] test *args: - source .evergreen/scripts/ensure-hatch.sh && hatch run test:test {{args}} + {{hatch_bin}} run test:test {{args}} [group('test')] test-mockupdb: - source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-mockupdb + {{hatch_bin}} run test:test-mockupdb [group('test')] test-eg *args: - source .evergreen/scripts/ensure-hatch.sh && hatch run test:test-eg {{args}} + {{hatch_bin}} run test:test-eg {{args}} [group('encryption')] setup-encryption: From ce775f3804b6c55783d720caea6cefeea41a5d85 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 14:49:00 -0600 Subject: [PATCH 07/18] Use just on host --- .evergreen/install-dependencies.sh | 12 ++++++++++++ .evergreen/{hatch.sh => just.sh} | 2 +- .evergreen/run-azurekms-fail-test.sh | 2 +- .evergreen/run-azurekms-test.sh | 2 +- .evergreen/run-gcpkms-test.sh | 2 +- .evergreen/run-mongodb-aws-ecs-test.sh | 2 +- .evergreen/run-mongodb-oidc-test.sh | 2 +- .evergreen/run-perf-tests.sh | 2 +- .evergreen/scripts/run-atlas-tests.sh | 2 +- .evergreen/scripts/run-doctests.sh | 2 +- .evergreen/scripts/run-enterprise-auth-tests.sh | 2 +- .evergreen/scripts/run-gcpkms-fail-test.sh | 2 +- .evergreen/scripts/run-mockupdb-tests.sh | 2 +- .evergreen/scripts/run-mongodb-aws-test.sh | 2 +- .evergreen/scripts/run-ocsp-test.sh | 2 +- .evergreen/scripts/run-tests.sh | 2 +- .evergreen/scripts/setup-dev-env.sh | 16 +++++++++------- .evergreen/setup-spawn-host.sh | 2 +- 18 files changed, 37 insertions(+), 23 deletions(-) rename .evergreen/{hatch.sh => just.sh} (79%) diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index e6dceb33fc..194acde37b 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -13,6 +13,18 @@ else SUDO="sudo" fi +# Install just. +if [ ! -f $HOME/.local/bin/just ]; then + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$HOME/.local/bin" || { + # CARGO_HOME is defined in configure-env.sh + export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} + export RUSTUP_HOME="${CARGO_HOME}/.rustup" + ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + cargo install just + mv $CARGO_HOME/just $HOME/.local/bin + } +fi + # Add 'server' and 'hostname_not_in_cert' as a hostnames echo "127.0.0.1 server" | $SUDO tee -a /etc/hosts echo "127.0.0.1 hostname_not_in_cert" | $SUDO tee -a /etc/hosts diff --git a/.evergreen/hatch.sh b/.evergreen/just.sh similarity index 79% rename from .evergreen/hatch.sh rename to .evergreen/just.sh index 9bfc930382..bebbca8282 100755 --- a/.evergreen/hatch.sh +++ b/.evergreen/just.sh @@ -2,4 +2,4 @@ set -eu . .evergreen/scripts/setup-dev-env.sh -hatch run "$@" +just "$@" diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh index d99c178fb9..d1117dcb32 100755 --- a/.evergreen/run-azurekms-fail-test.sh +++ b/.evergreen/run-azurekms-fail-test.sh @@ -8,5 +8,5 @@ PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ KEY_NAME="${AZUREKMS_KEYNAME}" \ KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ - $HERE/hatch.sh test:test-eg + $HERE/just.sh test-eg bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh index bb515a9386..d5c332fa8d 100755 --- a/.evergreen/run-azurekms-test.sh +++ b/.evergreen/run-azurekms-test.sh @@ -18,7 +18,7 @@ AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/hatch.sh test:test-eg" \ +AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/just.sh test-eg" \ $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Running test ... end" bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh index 7ccc74b453..4c953584b2 100755 --- a/.evergreen/run-gcpkms-test.sh +++ b/.evergreen/run-gcpkms-test.sh @@ -17,6 +17,6 @@ echo "Untarring file ... begin" GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" echo "Running test ... begin" -GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Running test ... end" bash $HERE/teardown-encryption.sh diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index 3189a6cc6c..91777be226 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -31,4 +31,4 @@ export AUTH="auth" export SET_XTRACE_ON=1 cd src rm -rf .venv -bash .evergreen/hatch.sh test:test-eg +bash .evergreen/just.sh test-eg diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 22864528c0..46c4f24969 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -32,4 +32,4 @@ fi export TEST_AUTH_OIDC=1 export COVERAGE=1 export AUTH="auth" -bash ./.evergreen/hatch.sh test:test-eg -- "${@:1}" +bash ./.evergreen/just.sh test-eg "${@:1}" diff --git a/.evergreen/run-perf-tests.sh b/.evergreen/run-perf-tests.sh index ff8d81a837..e6a51b3297 100755 --- a/.evergreen/run-perf-tests.sh +++ b/.evergreen/run-perf-tests.sh @@ -16,4 +16,4 @@ export OUTPUT_FILE="${PROJECT_DIRECTORY}/results.json" export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export PERF_TEST=1 -bash ./.evergreen/hatch.sh test:test-eg +bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-atlas-tests.sh b/.evergreen/scripts/run-atlas-tests.sh index 98a19f047f..30b8d5a615 100755 --- a/.evergreen/scripts/run-atlas-tests.sh +++ b/.evergreen/scripts/run-atlas-tests.sh @@ -4,4 +4,4 @@ set +x set -o errexit bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect -TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-doctests.sh b/.evergreen/scripts/run-doctests.sh index f7215ad347..5950e2c107 100755 --- a/.evergreen/scripts/run-doctests.sh +++ b/.evergreen/scripts/run-doctests.sh @@ -1,4 +1,4 @@ #!/bin/bash set -o xtrace -PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test +PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh docs-test diff --git a/.evergreen/scripts/run-enterprise-auth-tests.sh b/.evergreen/scripts/run-enterprise-auth-tests.sh index 7f936b1955..e015a34ca4 100755 --- a/.evergreen/scripts/run-enterprise-auth-tests.sh +++ b/.evergreen/scripts/run-enterprise-auth-tests.sh @@ -5,4 +5,4 @@ set -eu set +x # Use the default python to bootstrap secrets. bash "${DRIVERS_TOOLS}"/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth -TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-gcpkms-fail-test.sh b/.evergreen/scripts/run-gcpkms-fail-test.sh index dd9d522c8a..594a2984fa 100755 --- a/.evergreen/scripts/run-gcpkms-fail-test.sh +++ b/.evergreen/scripts/run-gcpkms-fail-test.sh @@ -4,4 +4,4 @@ export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh -SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg +SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-mockupdb-tests.sh b/.evergreen/scripts/run-mockupdb-tests.sh index 8825a0237d..32594f05d3 100755 --- a/.evergreen/scripts/run-mockupdb-tests.sh +++ b/.evergreen/scripts/run-mockupdb-tests.sh @@ -2,4 +2,4 @@ set -o xtrace export PYTHON_BINARY=${PYTHON_BINARY} -bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-mockupdb diff --git a/.evergreen/scripts/run-mongodb-aws-test.sh b/.evergreen/scripts/run-mongodb-aws-test.sh index ec20bfd06b..88c3236b3f 100755 --- a/.evergreen/scripts/run-mongodb-aws-test.sh +++ b/.evergreen/scripts/run-mongodb-aws-test.sh @@ -30,4 +30,4 @@ set -x export TEST_AUTH_AWS=1 export AUTH="auth" export SET_XTRACE_ON=1 -bash ./.evergreen/hatch.sh test:test-eg +bash ./.evergreen/just.sh test-eg diff --git a/.evergreen/scripts/run-ocsp-test.sh b/.evergreen/scripts/run-ocsp-test.sh index 3c6d3b2b3b..328bd2f203 100755 --- a/.evergreen/scripts/run-ocsp-test.sh +++ b/.evergreen/scripts/run-ocsp-test.sh @@ -4,5 +4,5 @@ TEST_OCSP=1 \ PYTHON_BINARY="${PYTHON_BINARY}" \ CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ -bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg +bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 6986a0bbee..0542c4c5f1 100755 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -51,4 +51,4 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ TEST_SUITES=${TEST_SUITES:-} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg + bash "${PROJECT_DIRECTORY}"/.evergreen/test.sh test-eg diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 305a66622c..c53105bc6d 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -56,14 +56,16 @@ if [ ! -f ${HATCH_CONFIG} ]; then hatch config set dirs.cache "$(pwd)/.hatch/cache" fi -# Ensure there is a local pre-commit. -if [ ! -f $BIN_DIR/pre-commit ]; then - python -m pip install pre-commit -fi +# Ensure there is a local pre-commit if there is a git checkout. +if [ -d .git ]; then + if [ ! -f $BIN_DIR/pre-commit ]; then + python -m pip install pre-commit + fi -# Ensure the pre-commit hook is installed. -if [ ! -f .git/hooks/pre-commit ]; then - pre-commit install + # Ensure the pre-commit hook is installed. + if ! -f .git/hooks/pre-commit ]; then + pre-commit install + fi fi # Install pymongo and its test deps. diff --git a/.evergreen/setup-spawn-host.sh b/.evergreen/setup-spawn-host.sh index b0e7abddbd..c20e1c756e 100755 --- a/.evergreen/setup-spawn-host.sh +++ b/.evergreen/setup-spawn-host.sh @@ -16,4 +16,4 @@ rsync -az -e ssh --exclude '.git' --filter=':- .gitignore' -r . $target:$remote_ echo "Copying files to $target... done" ssh $target $remote_dir/.evergreen/scripts/setup-system.sh -ssh $target "PYTHON_BINARY=${PYTHON_BINARY:-} $remote_dir/.evergreen/scripts/setup-dev-env.sh" +ssh $target "cd $remote_dir && PYTHON_BINARY=${PYTHON_BINARY:-} just install" From cd4265ee7eb68f9732da29bec9c65076be7a7018 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 14:52:43 -0600 Subject: [PATCH 08/18] fix script call --- .evergreen/scripts/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/run-tests.sh b/.evergreen/scripts/run-tests.sh index 0542c4c5f1..ea923b3f5e 100755 --- a/.evergreen/scripts/run-tests.sh +++ b/.evergreen/scripts/run-tests.sh @@ -51,4 +51,4 @@ GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \ TEST_SUITES=${TEST_SUITES:-} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - bash "${PROJECT_DIRECTORY}"/.evergreen/test.sh test-eg + bash "${PROJECT_DIRECTORY}"/.evergreen/just.sh test-eg From 098ab708e0c30b3b271389d3c362c13ae5df3308 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 15:08:51 -0600 Subject: [PATCH 09/18] fix handling of just --- .evergreen/install-dependencies.sh | 11 ++++++++--- .evergreen/scripts/setup-dev-env.sh | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 194acde37b..b7d38c09b2 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -14,14 +14,19 @@ else fi # Install just. -if [ ! -f $HOME/.local/bin/just ]; then - curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$HOME/.local/bin" || { +if [ "${CI:-}" == "true" ]; then + BIN_DIR=$MONGODB_BINARIES +else + BIN_DIR=$HOME/.local/bin +fi +if [ ! -f $BIN_DIR/just ]; then + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$BIN_DIR" || { # CARGO_HOME is defined in configure-env.sh export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} export RUSTUP_HOME="${CARGO_HOME}/.rustup" ${DRIVERS_TOOLS}/.evergreen/install-rust.sh cargo install just - mv $CARGO_HOME/just $HOME/.local/bin + mv $CARGO_HOME/just $BIN_DIR } fi diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index c53105bc6d..7042871942 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -63,7 +63,7 @@ if [ -d .git ]; then fi # Ensure the pre-commit hook is installed. - if ! -f .git/hooks/pre-commit ]; then + if [ ! -f .git/hooks/pre-commit ]; then pre-commit install fi fi From e76b93115439af8ec17dd6b262b8c5c2287f5c4e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 15:13:00 -0600 Subject: [PATCH 10/18] use drivers_tools_binaries --- .evergreen/install-dependencies.sh | 2 +- .evergreen/scripts/configure-env.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index b7d38c09b2..f58186275f 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -15,7 +15,7 @@ fi # Install just. if [ "${CI:-}" == "true" ]; then - BIN_DIR=$MONGODB_BINARIES + BIN_DIR=$DRIVERS_TOOLS_BINARIES else BIN_DIR=$HOME/.local/bin fi diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 5c7dff82f7..cbe6d0e7e8 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -34,6 +34,7 @@ fi export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" +export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" cat < "$SCRIPT_DIR"/env.sh export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" @@ -61,7 +62,7 @@ export skip_ECS_auth_test="${skip_ECS_auth_test:-}" export CARGO_HOME="$CARGO_HOME" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export HATCH_CONFIG="$HATCH_CONFIG" -export PATH="$MONGODB_BINARIES:$PATH" +export PATH="$MONGODB_BINARIES:$DRIVERS_TOOLS_BINARIES:$PATH" # shellcheck disable=SC2154 export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 From 3fe6ed0b469a500fbc6b4514df0a3959c8c2476b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 15:15:23 -0600 Subject: [PATCH 11/18] use drivers_tools_binaries --- .evergreen/scripts/configure-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index cbe6d0e7e8..5617f3e981 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -43,6 +43,7 @@ export SKIP_LEGACY_SHELL=1 export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" +export DRIVERS_TOOLS_BINARIES=$DRIVERS_TOOLS_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" From 07d8483708d810a2130bd2b356f514bf568d2b57 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 15:17:34 -0600 Subject: [PATCH 12/18] syntax --- .evergreen/scripts/configure-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 5617f3e981..ebbffcf1db 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -43,7 +43,7 @@ export SKIP_LEGACY_SHELL=1 export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" -export DRIVERS_TOOLS_BINARIES=$DRIVERS_TOOLS_BINARIES" +export DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS_BINARIES" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}" export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}" From f41033d256fb8c70dd184b0db1ea95610393fb91 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 11 Jan 2025 18:00:58 -0600 Subject: [PATCH 13/18] fix handling of just on windows --- .evergreen/install-dependencies.sh | 15 ++++++++++++--- .evergreen/scripts/install-dependencies.sh | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index f58186275f..36ef134e50 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -20,13 +20,22 @@ else BIN_DIR=$HOME/.local/bin fi if [ ! -f $BIN_DIR/just ]; then - curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$BIN_DIR" || { + if [ "Windows_NT" = "${OS:-}" ]; then + TARGET="--target x86_64-pc-windows-msvc" + else + TARGET="" + fi + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- $TARGET --to "$BIN_DIR" || { # CARGO_HOME is defined in configure-env.sh export CARGO_HOME=${CARGO_HOME:-$HOME/.cargo/} export RUSTUP_HOME="${CARGO_HOME}/.rustup" - ${DRIVERS_TOOLS}/.evergreen/install-rust.sh + . ${DRIVERS_TOOLS}/.evergreen/install-rust.sh cargo install just - mv $CARGO_HOME/just $BIN_DIR + if [ "Windows_NT" = "${OS:-}" ]; then + mv $CARGO_HOME/just.exe $BIN_DIR/just + else + mv $CARGO_HOME/just $BIN_DIR + fi } fi diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index ebcc8f3069..bbbfc745ec 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -o xtrace +set -eu file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. [ -f "$file" ] && bash "$file" || echo "$file not available, skipping" From 7c91d952bfce6b63fa84cafa2d016c9fdb1f9b26 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 13 Jan 2025 06:50:08 -0600 Subject: [PATCH 14/18] cleanup --- .github/workflows/test-python.yml | 3 +++ pymongo/pyopenssl_context.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5c062f9ebf..e5fa8ae40d 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -117,6 +117,9 @@ jobs: uses: supercharge/mongodb-github-action@1.12.0 with: mongodb-version: '8.0.0-rc4' + - name: Install dependencies + run: | + just install - name: Run tests run: | just docs-test diff --git a/pymongo/pyopenssl_context.py b/pymongo/pyopenssl_context.py index a320e94929..430fc1f01a 100644 --- a/pymongo/pyopenssl_context.py +++ b/pymongo/pyopenssl_context.py @@ -277,7 +277,7 @@ def __get_options(self) -> None: # Calling set_options adds the option to the existing bitmask and # returns the new bitmask. # https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_options - return self._ctx.set_options(0) + self._ctx.set_options(0) def __set_options(self, value: int) -> None: # Explicitly convert to int, since newer CPython versions From f8116d667afd851483c38eb53eca866331ff946c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 13 Jan 2025 06:59:35 -0600 Subject: [PATCH 15/18] fix import time test --- .evergreen/run-import-time-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-import-time-test.sh b/.evergreen/run-import-time-test.sh index e9f6161bcc..95e3c93d25 100755 --- a/.evergreen/run-import-time-test.sh +++ b/.evergreen/run-import-time-test.sh @@ -5,7 +5,7 @@ set -x . .evergreen/utils.sh -if [ -z "$PYTHON_BINARY" ]; then +if [ -z "${PYTHON_BINARY:-}" ]; then PYTHON_BINARY=$(find_python3) fi From 6830c2acfa32a4ca19c7daf55089ff4b42f97d02 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 13 Jan 2025 08:56:50 -0600 Subject: [PATCH 16/18] fix coverage script --- .evergreen/combine-coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/combine-coverage.sh b/.evergreen/combine-coverage.sh index 92d2f1f1f8..c31f755bd9 100755 --- a/.evergreen/combine-coverage.sh +++ b/.evergreen/combine-coverage.sh @@ -8,7 +8,7 @@ set -o errexit # Exit the script with error if any of the commands fail . .evergreen/utils.sh -if [ -z "$PYTHON_BINARY" ]; then +if [ -z "${PYTHON_BINARY:-}" ]; then PYTHON_BINARY=$(find_python3) fi From 29fa6b5553d670ff5d201f05afcd290ae9414bd3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 17 Jan 2025 12:38:16 -0600 Subject: [PATCH 17/18] Update .github/workflows/test-python.yml Co-authored-by: Noah Stapp --- .github/workflows/test-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index e5fa8ae40d..a41daaabb1 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -80,7 +80,7 @@ jobs: - name: Install dependencies run: | if [[ "${{ matrix.python-version }}" == "3.13t" ]]; then - # Hatch can't be installed on 3.13t, use pytest directly. + # Just can't be installed on 3.13t, use pytest directly. pip install . pip install -r requirements/test.txt else From e604fb651f2d575c97d73c661cf907a8a940b920 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 17 Jan 2025 12:39:40 -0600 Subject: [PATCH 18/18] add comment --- .evergreen/install-dependencies.sh | 1 + pymongo/pyopenssl_context.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 36ef134e50..8773fa2c6d 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -14,6 +14,7 @@ else fi # Install just. +# On Evergreen jobs, "CI" will be set, and we don't want to write to $HOME. if [ "${CI:-}" == "true" ]; then BIN_DIR=$DRIVERS_TOOLS_BINARIES else diff --git a/pymongo/pyopenssl_context.py b/pymongo/pyopenssl_context.py index 1b891173fd..8c643394b2 100644 --- a/pymongo/pyopenssl_context.py +++ b/pymongo/pyopenssl_context.py @@ -277,7 +277,7 @@ def __get_options(self) -> int: # Calling set_options adds the option to the existing bitmask and # returns the new bitmask. # https://www.pyopenssl.org/en/stable/api/ssl.html#OpenSSL.SSL.Context.set_options - self._ctx.set_options(0) + return self._ctx.set_options(0) def __set_options(self, value: int) -> None: # Explicitly convert to int, since newer CPython versions