From 4250035d76b029aae83db494385dd06ad6debc8c Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 23 Nov 2025 18:58:25 +0800 Subject: [PATCH 1/6] fix: profiling or tracing multiprocessing can cause error Signed-off-by: yihong0618 Co-Authored-By: YvesDup --- Lib/profile.py | 9 +++-- Lib/profiling/tracing/__init__.py | 2 +- Lib/test/test_profile.py | 35 +++++++++++++++++++ .../test_profiling/test_tracing_profiler.py | 35 +++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/Lib/profile.py b/Lib/profile.py index 20c500d28bc5b9..bde567a8eba282 100644 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -602,13 +602,16 @@ def main(): code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) - globs = { - '__spec__': spec, + module = importlib.util.module_from_spec(spec) + sys.modules['__main__'] = module + globs = module.__dict__ + globs.update({ + '__spec__': None, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, - } + }) try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py index a6b8edf721611f..143573ff747d1b 100644 --- a/Lib/profiling/tracing/__init__.py +++ b/Lib/profiling/tracing/__init__.py @@ -197,7 +197,7 @@ def main(): # in the module's namespace. globs = module.__dict__ globs.update({ - '__spec__': spec, + '__spec__': None, '__file__': spec.origin, '__name__': spec.name, '__package__': None, diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index c4b2d7ca05c0a6..6e7a77bbe5ff3b 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -4,10 +4,12 @@ import pstats import unittest import os +import subprocess import warnings from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd +from test.support import script_helper, os_helper, SHORT_TIMEOUT from contextlib import contextmanager, redirect_stdout # Suppress deprecation warning for profile module (PEP 799) @@ -134,6 +136,39 @@ def test_output_file_when_changing_directory(self): self.assertTrue(os.path.exists('out.pstats')) + def test_profile_multiprocessing(self): + test_script = ''' +import multiprocessing +def worker_proc(x): + return x * 42 + +def main_proc(): + p = multiprocessing.Process(target=worker_proc, args=(10,)) + p.start() + p.join() + +if __name__ == "__main__": + main_proc() +''' + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_profile_multiprocessing', test_script + ) + with script_helper.spawn_python( + "-m", self.profilermodule.__name__, + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=SHORT_TIMEOUT) + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn("main_proc", stdout) + self.assertNotIn("Can't pickle", stderr) + self.assertNotIn("ModuleSpec", stderr) + self.assertEqual(proc.returncode, 0) + def regenerate_expected_output(filename, cls): filename = filename.rstrip('co') diff --git a/Lib/test/test_profiling/test_tracing_profiler.py b/Lib/test/test_profiling/test_tracing_profiler.py index d09ca441d4ae46..388f3c5d0528f9 100644 --- a/Lib/test/test_profiling/test_tracing_profiler.py +++ b/Lib/test/test_profiling/test_tracing_profiler.py @@ -2,12 +2,14 @@ import sys import unittest +import subprocess # rip off all interesting stuff from test_profile import profiling.tracing as cProfile import tempfile import textwrap from test.test_profile import ProfileTest, regenerate_expected_output +from test.support import script_helper, os_helper, SHORT_TIMEOUT from test.support.script_helper import assert_python_failure, assert_python_ok from test import support @@ -170,6 +172,39 @@ class Foo: f.close() assert_python_ok('-m', "cProfile", f.name) + def test_profile_multiprocessing(self): + test_script = ''' +import multiprocessing + +def worker_proc(x): + return x * 42 + +def main_proc(): + p = multiprocessing.Process(target=worker_proc, args=(10,)) + p.start() + p.join() + print("SUCCESS") + +if __name__ == "__main__": + main_proc() +''' + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_cprofile_multiprocessing', test_script + ) + with script_helper.spawn_python( + "-m", "cProfile", + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=SHORT_TIMEOUT) + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn("SUCCESS", stdout) + self.assertNotIn("has no attribute \'worker_proc\'", stderr) + def main(): if '-r' not in sys.argv: From c2173031d619596d9e2c8410b1f8fd41d4863ce7 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 23 Nov 2025 19:06:55 +0800 Subject: [PATCH 2/6] fix: add news Signed-off-by: yihong0618 --- .../next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst new file mode 100644 index 00000000000000..5f58e12fec67b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst @@ -0,0 +1,2 @@ +Set ``__main__.__spec__`` to ``None`` when running a script with +:mod:`profile` or `cProfile` From d4f461533dfd2808debd7fced32fdf591d992e93 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 23 Nov 2025 19:11:05 +0800 Subject: [PATCH 3/6] fix: news format Signed-off-by: yihong0618 --- .../next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst index 5f58e12fec67b9..f307d8808b976a 100644 --- a/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst +++ b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst @@ -1,2 +1,2 @@ Set ``__main__.__spec__`` to ``None`` when running a script with -:mod:`profile` or `cProfile` +:mod:`profile` or :mod:`cProfile` From 50dec2f9eb8383b0a25baa49ca04f8a6f295df69 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 23 Nov 2025 19:46:37 +0800 Subject: [PATCH 4/6] fix: skip these tests on windows Signed-off-by: yihong0618 --- Lib/test/test_profile.py | 2 ++ Lib/test/test_profiling/test_tracing_profiler.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index 6e7a77bbe5ff3b..bf5698cf0c9b3e 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -136,6 +136,8 @@ def test_output_file_when_changing_directory(self): self.assertTrue(os.path.exists('out.pstats')) + @unittest.skipIf(sys.platform == 'win32', + 'Profiler with multiprocessing can not run on win32') def test_profile_multiprocessing(self): test_script = ''' import multiprocessing diff --git a/Lib/test/test_profiling/test_tracing_profiler.py b/Lib/test/test_profiling/test_tracing_profiler.py index 388f3c5d0528f9..d4bed67ffe8175 100644 --- a/Lib/test/test_profiling/test_tracing_profiler.py +++ b/Lib/test/test_profiling/test_tracing_profiler.py @@ -172,6 +172,8 @@ class Foo: f.close() assert_python_ok('-m', "cProfile", f.name) + @unittest.skipIf(sys.platform == 'win32', + 'Profiler with multiprocessing can not run on win32') def test_profile_multiprocessing(self): test_script = ''' import multiprocessing From a8a26da09b39c76fe25431785e344ab67b179055 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 3 Dec 2025 16:42:35 +0800 Subject: [PATCH 5/6] fix: only fix cProfile Signed-off-by: yihong0618 --- Lib/profile.py | 7 ++-- Lib/test/test_profile.py | 37 ------------------- ...-11-23-19-06-41.gh-issue-140729.bk_HDs.rst | 3 +- 3 files changed, 4 insertions(+), 43 deletions(-) diff --git a/Lib/profile.py b/Lib/profile.py index bde567a8eba282..7436b7537f744e 100644 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -604,14 +604,13 @@ def main(): origin=progname) module = importlib.util.module_from_spec(spec) sys.modules['__main__'] = module - globs = module.__dict__ - globs.update({ - '__spec__': None, + globs = { + '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, - }) + } try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index bf5698cf0c9b3e..c4b2d7ca05c0a6 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -4,12 +4,10 @@ import pstats import unittest import os -import subprocess import warnings from difflib import unified_diff from io import StringIO from test.support.os_helper import TESTFN, unlink, temp_dir, change_cwd -from test.support import script_helper, os_helper, SHORT_TIMEOUT from contextlib import contextmanager, redirect_stdout # Suppress deprecation warning for profile module (PEP 799) @@ -136,41 +134,6 @@ def test_output_file_when_changing_directory(self): self.assertTrue(os.path.exists('out.pstats')) - @unittest.skipIf(sys.platform == 'win32', - 'Profiler with multiprocessing can not run on win32') - def test_profile_multiprocessing(self): - test_script = ''' -import multiprocessing -def worker_proc(x): - return x * 42 - -def main_proc(): - p = multiprocessing.Process(target=worker_proc, args=(10,)) - p.start() - p.join() - -if __name__ == "__main__": - main_proc() -''' - with os_helper.temp_dir() as temp_dir: - script = script_helper.make_script( - temp_dir, 'test_profile_multiprocessing', test_script - ) - with script_helper.spawn_python( - "-m", self.profilermodule.__name__, - script, - stderr=subprocess.PIPE, - text=True - ) as proc: - proc.wait(timeout=SHORT_TIMEOUT) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - - self.assertIn("main_proc", stdout) - self.assertNotIn("Can't pickle", stderr) - self.assertNotIn("ModuleSpec", stderr) - self.assertEqual(proc.returncode, 0) - def regenerate_expected_output(filename, cls): filename = filename.rstrip('co') diff --git a/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst index f307d8808b976a..0341c43b68eab6 100644 --- a/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst +++ b/Misc/NEWS.d/next/Library/2025-11-23-19-06-41.gh-issue-140729.bk_HDs.rst @@ -1,2 +1 @@ -Set ``__main__.__spec__`` to ``None`` when running a script with -:mod:`profile` or :mod:`cProfile` +Set ``__main__.__spec__`` to ``None`` when running a script with :mod:`cProfile` From dbf6e55ab0f8b0c5be52fa2ebddcd65508d9ca82 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 3 Dec 2025 16:43:41 +0800 Subject: [PATCH 6/6] fix: forget to delete lines Signed-off-by: yihong0618 --- Lib/profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/profile.py b/Lib/profile.py index 7436b7537f744e..20c500d28bc5b9 100644 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -602,8 +602,6 @@ def main(): code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) - module = importlib.util.module_from_spec(spec) - sys.modules['__main__'] = module globs = { '__spec__': spec, '__file__': spec.origin,