Merge pull request #415 from mdboom/coverage-from-subprocesses

Better coverage collection from subprocesses

Author: mdboom

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
           python -m pip install .[test]
       - name: run tests
         run: |
-          python -m pytest tests -n auto -m long_running
+          python -m pytest tests -m long_running
 
   long_test_windows:
     runs-on: windows-latest
@@ -67,5 +67,5 @@ jobs:
           python -m pip install .[test]
       - name: run tests
         run: |
-          python -m pytest tests -n auto -m long_running
+          python -m pytest tests -m long_running
 

bench_runner/scripts/run_benchmarks.py

@@ -459,4 +459,13 @@ def dummy(*args, **kwargs):
 
 
 if __name__ == "__main__":
+    # This lets pytest-cov collect coverage data in a subprocess
+    if "COV_CORE_SOURCE" in os.environ:
+        try:
+            from pytest_cov.embed import init
+
+            init()
+        except Exception:
+            sys.stderr.write("pytest-cov: Failed to setup subprocess coverage.")
+
     main()

bench_runner/scripts/workflow.py

@@ -391,4 +391,13 @@ def main():
 
 
 if __name__ == "__main__":
+    # This lets pytest-cov collect coverage data in a subprocess
+    if "COV_CORE_SOURCE" in os.environ:
+        try:
+            from pytest_cov.embed import init
+
+            init()
+        except Exception:
+            sys.stderr.write("pytest-cov: Failed to setup subprocess coverage.")
+
     main()

bench_runner/templates/workflow_bootstrap.src.py

@@ -52,6 +52,10 @@ def install_requirements(venv: Path) -> None:
     run_in_venv(venv, "pip", ["install", "--upgrade", "pip"])
     run_in_venv(venv, "pip", ["install", "-r", "requirements.txt"])
 
+    # To facilitate coverage testing
+    if "--_fast" in sys.argv:
+        run_in_venv(venv, "pip", ["install", "pytest-cov"])
+
 
 def main():
     venv = Path("venv")

pyproject.toml

@@ -32,6 +32,7 @@ test = [
     "flake8==7.2.0",
     "pyright==1.1.399",
     "pytest==8.3.5",
+    "pytest-cov==6.1.1",
     "pytest-xdist==3.6.1",
 ]
 
@@ -46,4 +47,11 @@ bench_runner = ["templates/*"]
 [tool.pytest.ini_options]
 markers = [
     "long_running: marks tests as long_running (deselect with '-m \"not running\"')",
+]
+
+[tool.coverage.paths]
+source = [
+    "bench_runner",
+    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/outer_venv/lib/python*/site-packages/bench_runner",
+    "/tmp/pytest-of-*/pytest-*/test_whole_workflow*/repo/venv/lib/python*/site-packages/bench_runner",
 ]

tests/test_workflow.py

@@ -226,13 +226,7 @@ def test_whole_workflow(tmpdir):
             ]
         )
         subprocess.check_call(
-            [
-                str(binary),
-                "-m",
-                "pip",
-                "install",
-                str(bench_runner_checkout),
-            ]
+            [str(binary), "-m", "pip", "install", f"{bench_runner_checkout}[test]"]
         )
         subprocess.check_call([str(binary), "-m", "bench_runner", "install"])
         # install --check should never fail immediately after install