From 8b7c3ee79d689dd6d3d52db491a50ac303140fae Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 01:32:41 +0000 Subject: [PATCH 01/17] Add raise_exceptions parameter to multiprocessing.set_forkserver_preload Add a keyword-only `raise_exceptions` parameter (default False) to `multiprocessing.set_forkserver_preload()`. When True, ImportError exceptions during module preloading cause the forkserver to exit, breaking all use of the forkserver multiprocessing context. This allows developers to catch import errors during development rather than having them silently ignored. Implementation adds the parameter to both the ForkServer class method and the BaseContext wrapper, passing it through to the forkserver main() function which conditionally raises ImportError instead of ignoring it. Tests are in new test_multiprocessing_forkserver/test_preload.py with proper resource cleanup using try/finally. Documentation describes the behavior, consequences (forkserver exit, EOFError/ConnectionError on subsequent use), and recommends use during development. Based on original work by Nick Neumann in GH-99515. Contributed by Nick Neumann. Co-authored-by: aggieNick02 Co-authored-by: Claude (Sonnet 4.5) Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 17 +++- Lib/multiprocessing/context.py | 9 +- Lib/multiprocessing/forkserver.py | 17 +++- .../test_preload.py | 96 +++++++++++++++++++ Misc/ACKS | 1 + ...-11-22-20-30-00.gh-issue-101100.frksvr.rst | 5 + 6 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 Lib/test/test_multiprocessing_forkserver/test_preload.py create mode 100644 Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 714207cb0aefcd..6e555b4fed60b6 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1211,22 +1211,31 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_forkserver_preload(module_names) +.. function:: set_forkserver_preload(module_names, *, raise_exceptions=False) Set a list of module names for the forkserver main process to attempt to import so that their already imported state is inherited by forked - processes. Any :exc:`ImportError` when doing so is silently ignored. - This can be used as a performance enhancement to avoid repeated work - in every process. + processes. This can be used as a performance enhancement to avoid repeated + work in every process. For this to work, it must be called before the forkserver process has been launched (before creating a :class:`Pool` or starting a :class:`Process`). + By default, any :exc:`ImportError` when importing modules is silently + ignored. If *raise_exceptions* is ``True``, :exc:`ImportError` exceptions + will be raised in the forkserver subprocess, causing it to exit. The + exception traceback will appear on stderr, and subsequent attempts to + create processes will fail with :exc:`EOFError` or :exc:`ConnectionError`. + Use *raise_exceptions* during development to catch import problems early. + Only meaningful when using the ``'forkserver'`` start method. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 + .. versionchanged:: next + Added the *raise_exceptions* parameter. + .. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 051d567d457928..a9d5aa5c7be877 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -177,12 +177,15 @@ def set_executable(self, executable): from .spawn import set_executable set_executable(executable) - def set_forkserver_preload(self, module_names): + def set_forkserver_preload(self, module_names, *, raise_exceptions=False): '''Set list of module names to try to load in forkserver process. - This is really just a hint. + + If raise_exceptions is True, ImportError exceptions during preload + will be raised instead of being silently ignored. Such errors will + break all use of the forkserver multiprocessing context. ''' from .forkserver import set_forkserver_preload - set_forkserver_preload(module_names) + set_forkserver_preload(module_names, raise_exceptions=raise_exceptions) def get_context(self, method=None): if method is None: diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 8a4e8d835b0c91..1f50a7842977a7 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -42,6 +42,7 @@ def __init__(self): self._inherited_fds = None self._lock = threading.Lock() self._preload_modules = ['__main__'] + self._raise_exceptions = False def _stop(self): # Method used by unit tests to stop the server @@ -64,11 +65,17 @@ def _stop_unlocked(self): self._forkserver_address = None self._forkserver_authkey = None - def set_forkserver_preload(self, modules_names): - '''Set list of module names to try to load in forkserver process.''' + def set_forkserver_preload(self, modules_names, *, raise_exceptions=False): + '''Set list of module names to try to load in forkserver process. + + If raise_exceptions is True, ImportError exceptions during preload + will be raised instead of being silently ignored. Such errors will + break all use of the forkserver multiprocessing context. + ''' if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names + self._raise_exceptions = raise_exceptions def get_inherited_fds(self): '''Return list of fds inherited from parent process. @@ -152,6 +159,8 @@ def ensure_running(self): main_kws['sys_path'] = data['sys_path'] if 'init_main_from_path' in data: main_kws['main_path'] = data['init_main_from_path'] + if self._raise_exceptions: + main_kws['raise_exceptions'] = True with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -197,7 +206,7 @@ def ensure_running(self): # def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, authkey_r=None): + *, authkey_r=None, raise_exceptions=False): """Run forkserver.""" if authkey_r is not None: try: @@ -221,6 +230,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, try: __import__(modname) except ImportError: + if raise_exceptions: + raise pass # gh-135335: flush stdout/stderr in case any of the preloaded modules diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py new file mode 100644 index 00000000000000..21468acf2cdc51 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -0,0 +1,96 @@ +"""Tests for forkserver preload functionality.""" + +import multiprocessing +import multiprocessing.forkserver +import sys +import unittest + + +@unittest.skipIf(sys.platform == "win32", "forkserver not available on Windows") +class TestForkserverPreload(unittest.TestCase): + """Tests for forkserver preload functionality.""" + + def setUp(self): + self.ctx = multiprocessing.get_context('forkserver') + # Ensure clean state for each test + multiprocessing.forkserver._forkserver._stop() + + def tearDown(self): + # Clean up after tests + multiprocessing.forkserver._forkserver._stop() + self.ctx.set_forkserver_preload([]) + + @staticmethod + def _send_value(conn, value): + """Helper to send a value through a connection.""" + conn.send(value) + + def test_preload_raise_exceptions_false_default(self): + """Test that invalid modules are silently ignored by default.""" + # With raise_exceptions=False (default), invalid module is ignored + self.ctx.set_forkserver_preload(['nonexistent_module_xyz']) + + # Should be able to start a process without errors + r, w = self.ctx.Pipe(duplex=False) + p = self.ctx.Process(target=self._send_value, args=(w, 42)) + p.start() + w.close() + result = r.recv() + r.close() + p.join() + + self.assertEqual(result, 42) + self.assertEqual(p.exitcode, 0) + + def test_preload_raise_exceptions_false_explicit(self): + """Test that invalid modules are silently ignored with raise_exceptions=False.""" + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=False) + + # Should be able to start a process without errors + r, w = self.ctx.Pipe(duplex=False) + p = self.ctx.Process(target=self._send_value, args=(w, 99)) + p.start() + w.close() + result = r.recv() + r.close() + p.join() + + self.assertEqual(result, 99) + self.assertEqual(p.exitcode, 0) + + def test_preload_raise_exceptions_true_breaks_context(self): + """Test that invalid modules with raise_exceptions=True breaks the forkserver.""" + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=True) + + # The forkserver should fail to start when it tries to import + # The exception is raised during p.start() when trying to communicate + # with the failed forkserver process + r, w = self.ctx.Pipe(duplex=False) + try: + p = self.ctx.Process(target=self._send_value, args=(w, 42)) + with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)): + p.start() # Exception raised here + finally: + # Ensure pipes are closed even if exception is raised + w.close() + r.close() + + def test_preload_valid_modules_with_raise_exceptions(self): + """Test that valid modules work fine with raise_exceptions=True.""" + # Valid modules should work even with raise_exceptions=True + self.ctx.set_forkserver_preload(['os', 'sys'], raise_exceptions=True) + + r, w = self.ctx.Pipe(duplex=False) + p = self.ctx.Process(target=self._send_value, args=(w, 'success')) + p.start() + w.close() + result = r.recv() + r.close() + p.join() + + self.assertEqual(result, 'success') + self.assertEqual(p.exitcode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index f5f15f2eb7ea24..fc9085696e3d03 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1337,6 +1337,7 @@ Trent Nelson Andrew Nester Osvaldo Santana Neto Chad Netzer +Nick Neumann Max Neunhöffer Anthon van der Neut George Neville-Neil diff --git a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst new file mode 100644 index 00000000000000..b7e6c1674cdc3b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst @@ -0,0 +1,5 @@ +Add a ``raise_exceptions`` keyword-only parameter to +:func:`multiprocessing.set_forkserver_preload`. When set to ``True``, +:exc:`ImportError` exceptions during module preloading will be raised instead +of being silently ignored. Defaults to ``False`` for backward compatibility. +Contributed by Nick Neumann. From e8836d5a9292d1e5f75d4c398927b7e0407862c7 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 01:36:46 +0000 Subject: [PATCH 02/17] Skip forkserver preload tests on platforms without fork support Add skip decorators to exclude test_preload.py on Android, iOS, WASI, and other platforms that don't support fork, using the existing has_fork_support check from test.support. Co-authored-by: Claude (Sonnet 4.5) Co-authored-by: Gregory P. Smith --- Lib/test/test_multiprocessing_forkserver/test_preload.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 21468acf2cdc51..b843c7cbb4e36c 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -5,8 +5,11 @@ import sys import unittest +from test.support import has_fork_support + @unittest.skipIf(sys.platform == "win32", "forkserver not available on Windows") +@unittest.skipUnless(has_fork_support, "requires working os.fork()") class TestForkserverPreload(unittest.TestCase): """Tests for forkserver preload functionality.""" From 5ce91baaee345ade18077e48522d4afce88818ef Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 02:52:46 +0000 Subject: [PATCH 03/17] Skip all forkserver tests on platforms without fork support Add has_fork_support check at the package level in __init__.py to skip the entire test_multiprocessing_forkserver package on Android, iOS, WASI, and other platforms that don't support fork. This prevents import errors before individual test modules are loaded. Remove the now-redundant skip decorators from test_preload.py since the package-level skip makes them unnecessary. Co-authored-by: Claude (Sonnet 4.5) Co-authored-by: Gregory P. Smith --- Lib/test/test_multiprocessing_forkserver/__init__.py | 3 +++ Lib/test/test_multiprocessing_forkserver/test_preload.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_multiprocessing_forkserver/__init__.py b/Lib/test/test_multiprocessing_forkserver/__init__.py index d91715a344dfa7..7b1b884ab297b5 100644 --- a/Lib/test/test_multiprocessing_forkserver/__init__.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -9,5 +9,8 @@ if sys.platform == "win32": raise unittest.SkipTest("forkserver is not available on Windows") +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index b843c7cbb4e36c..0a5137da7685f6 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -2,14 +2,9 @@ import multiprocessing import multiprocessing.forkserver -import sys import unittest -from test.support import has_fork_support - -@unittest.skipIf(sys.platform == "win32", "forkserver not available on Windows") -@unittest.skipUnless(has_fork_support, "requires working os.fork()") class TestForkserverPreload(unittest.TestCase): """Tests for forkserver preload functionality.""" From 75495cc9e25f7bdeda7b260321100c2ca7325a0c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:01:53 +0000 Subject: [PATCH 04/17] Refactor set_forkserver_preload to use on_error parameter Changed from a boolean raise_exceptions parameter to a more flexible on_error parameter that accepts 'ignore', 'warn', or 'fail'. - 'ignore' (default): silently ignores import failures - 'warn': emits ImportWarning from forkserver subprocess - 'fail': raises exception, causing forkserver to exit Also improved error messages by adding .add_note() to connection failures when on_error='fail' to guide users to check stderr. Updated both spawn.import_main_path() and module __import__() failure paths to support all three modes using match/case syntax. Co-authored-by: aggieNick02 Co-authored-by: Claude (Sonnet 4.5) Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 17 ++--- Lib/multiprocessing/context.py | 10 +-- Lib/multiprocessing/forkserver.py | 62 +++++++++++++++---- .../test_preload.py | 51 +++++++++++---- ...-11-22-20-30-00.gh-issue-101100.frksvr.rst | 5 -- ...-11-22-20-30-00.gh-issue-141860.frksvr.rst | 5 ++ 6 files changed, 106 insertions(+), 44 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst create mode 100644 Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6e555b4fed60b6..a6a312f10b75ef 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1211,7 +1211,7 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_forkserver_preload(module_names, *, raise_exceptions=False) +.. function:: set_forkserver_preload(module_names, *, on_error='ignore') Set a list of module names for the forkserver main process to attempt to import so that their already imported state is inherited by forked @@ -1221,12 +1221,13 @@ Miscellaneous For this to work, it must be called before the forkserver process has been launched (before creating a :class:`Pool` or starting a :class:`Process`). - By default, any :exc:`ImportError` when importing modules is silently - ignored. If *raise_exceptions* is ``True``, :exc:`ImportError` exceptions - will be raised in the forkserver subprocess, causing it to exit. The - exception traceback will appear on stderr, and subsequent attempts to - create processes will fail with :exc:`EOFError` or :exc:`ConnectionError`. - Use *raise_exceptions* during development to catch import problems early. + The *on_error* parameter controls how :exc:`ImportError` exceptions during + module preloading are handled: ``'ignore'`` (default) silently ignores + failures, ``'warn'`` causes the forkserver subprocess to emit an + :exc:`ImportWarning` to stderr, and ``'fail'`` causes the forkserver + subprocess to exit with the exception traceback on stderr, making + subsequent process creation fail with :exc:`EOFError` or + :exc:`ConnectionError`. Only meaningful when using the ``'forkserver'`` start method. See :ref:`multiprocessing-start-methods`. @@ -1234,7 +1235,7 @@ Miscellaneous .. versionadded:: 3.4 .. versionchanged:: next - Added the *raise_exceptions* parameter. + Added the *on_error* parameter. .. function:: set_start_method(method, force=False) diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index a9d5aa5c7be877..3d005aeab4ccf7 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -177,15 +177,15 @@ def set_executable(self, executable): from .spawn import set_executable set_executable(executable) - def set_forkserver_preload(self, module_names, *, raise_exceptions=False): + def set_forkserver_preload(self, module_names, *, on_error='ignore'): '''Set list of module names to try to load in forkserver process. - If raise_exceptions is True, ImportError exceptions during preload - will be raised instead of being silently ignored. Such errors will - break all use of the forkserver multiprocessing context. + The on_error parameter controls how import failures are handled: + 'ignore' (default) silently ignores failures, 'warn' emits warnings, + and 'fail' raises exceptions breaking the forkserver context. ''' from .forkserver import set_forkserver_preload - set_forkserver_preload(module_names, raise_exceptions=raise_exceptions) + set_forkserver_preload(module_names, on_error=on_error) def get_context(self, method=None): if method is None: diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 1f50a7842977a7..e3a2e7bb5c3e05 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -42,7 +42,7 @@ def __init__(self): self._inherited_fds = None self._lock = threading.Lock() self._preload_modules = ['__main__'] - self._raise_exceptions = False + self._preload_on_error = 'ignore' def _stop(self): # Method used by unit tests to stop the server @@ -65,17 +65,22 @@ def _stop_unlocked(self): self._forkserver_address = None self._forkserver_authkey = None - def set_forkserver_preload(self, modules_names, *, raise_exceptions=False): + def set_forkserver_preload(self, modules_names, *, on_error='ignore'): '''Set list of module names to try to load in forkserver process. - If raise_exceptions is True, ImportError exceptions during preload - will be raised instead of being silently ignored. Such errors will - break all use of the forkserver multiprocessing context. + The on_error parameter controls how import failures are handled: + 'ignore' (default) silently ignores failures, 'warn' emits warnings, + and 'fail' raises exceptions breaking the forkserver context. ''' if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') + if on_error not in ('ignore', 'warn', 'fail'): + raise ValueError( + f"on_error must be 'ignore', 'warn', or 'fail', " + f"not {on_error!r}" + ) self._preload_modules = modules_names - self._raise_exceptions = raise_exceptions + self._preload_on_error = on_error def get_inherited_fds(self): '''Return list of fds inherited from parent process. @@ -114,6 +119,15 @@ def connect_to_new_process(self, fds): wrapped_client, self._forkserver_authkey) connection.deliver_challenge( wrapped_client, self._forkserver_authkey) + except (EOFError, ConnectionError, BrokenPipeError) as exc: + # Add helpful context if forkserver likely crashed during preload + if (self._preload_modules and + self._preload_on_error == 'fail'): + exc.add_note( + "Forkserver process may have crashed during module " + "preloading. Check stderr for ImportError traceback." + ) + raise finally: wrapped_client._detach() del wrapped_client @@ -159,8 +173,8 @@ def ensure_running(self): main_kws['sys_path'] = data['sys_path'] if 'init_main_from_path' in data: main_kws['main_path'] = data['init_main_from_path'] - if self._raise_exceptions: - main_kws['raise_exceptions'] = True + if self._preload_on_error != 'ignore': + main_kws['on_error'] = self._preload_on_error with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -206,7 +220,7 @@ def ensure_running(self): # def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, authkey_r=None, raise_exceptions=False): + *, authkey_r=None, on_error='ignore'): """Run forkserver.""" if authkey_r is not None: try: @@ -224,15 +238,37 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, process.current_process()._inheriting = True try: spawn.import_main_path(main_path) + except Exception as e: + match on_error: + case 'fail': + raise + case 'warn': + import warnings + warnings.warn( + f"Failed to import __main__ from {main_path!r}: {e}", + ImportWarning, + stacklevel=2 + ) + case 'ignore': + pass finally: del process.current_process()._inheriting for modname in preload: try: __import__(modname) - except ImportError: - if raise_exceptions: - raise - pass + except ImportError as e: + match on_error: + case 'fail': + raise + case 'warn': + import warnings + warnings.warn( + f"Failed to preload module {modname!r}: {e}", + ImportWarning, + stacklevel=2 + ) + case 'ignore': + pass # gh-135335: flush stdout/stderr in case any of the preloaded modules # wrote to them, otherwise children might inherit buffered data diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 0a5137da7685f6..647d4ff89052e0 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -3,6 +3,7 @@ import multiprocessing import multiprocessing.forkserver import unittest +import warnings class TestForkserverPreload(unittest.TestCase): @@ -23,9 +24,9 @@ def _send_value(conn, value): """Helper to send a value through a connection.""" conn.send(value) - def test_preload_raise_exceptions_false_default(self): + def test_preload_on_error_ignore_default(self): """Test that invalid modules are silently ignored by default.""" - # With raise_exceptions=False (default), invalid module is ignored + # With on_error='ignore' (default), invalid module is ignored self.ctx.set_forkserver_preload(['nonexistent_module_xyz']) # Should be able to start a process without errors @@ -40,9 +41,9 @@ def test_preload_raise_exceptions_false_default(self): self.assertEqual(result, 42) self.assertEqual(p.exitcode, 0) - def test_preload_raise_exceptions_false_explicit(self): - """Test that invalid modules are silently ignored with raise_exceptions=False.""" - self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=False) + def test_preload_on_error_ignore_explicit(self): + """Test that invalid modules are silently ignored with on_error='ignore'.""" + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='ignore') # Should be able to start a process without errors r, w = self.ctx.Pipe(duplex=False) @@ -56,9 +57,25 @@ def test_preload_raise_exceptions_false_explicit(self): self.assertEqual(result, 99) self.assertEqual(p.exitcode, 0) - def test_preload_raise_exceptions_true_breaks_context(self): - """Test that invalid modules with raise_exceptions=True breaks the forkserver.""" - self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], raise_exceptions=True) + def test_preload_on_error_warn(self): + """Test that invalid modules emit warnings with on_error='warn'.""" + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='warn') + + # Should still be able to start a process, warnings are in subprocess + r, w = self.ctx.Pipe(duplex=False) + p = self.ctx.Process(target=self._send_value, args=(w, 123)) + p.start() + w.close() + result = r.recv() + r.close() + p.join() + + self.assertEqual(result, 123) + self.assertEqual(p.exitcode, 0) + + def test_preload_on_error_fail_breaks_context(self): + """Test that invalid modules with on_error='fail' breaks the forkserver.""" + self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='fail') # The forkserver should fail to start when it tries to import # The exception is raised during p.start() when trying to communicate @@ -66,17 +83,19 @@ def test_preload_raise_exceptions_true_breaks_context(self): r, w = self.ctx.Pipe(duplex=False) try: p = self.ctx.Process(target=self._send_value, args=(w, 42)) - with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)): + with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)) as cm: p.start() # Exception raised here + # Verify that the helpful note was added + self.assertIn('Forkserver process may have crashed', str(cm.exception.__notes__[0])) finally: # Ensure pipes are closed even if exception is raised w.close() r.close() - def test_preload_valid_modules_with_raise_exceptions(self): - """Test that valid modules work fine with raise_exceptions=True.""" - # Valid modules should work even with raise_exceptions=True - self.ctx.set_forkserver_preload(['os', 'sys'], raise_exceptions=True) + def test_preload_valid_modules_with_on_error_fail(self): + """Test that valid modules work fine with on_error='fail'.""" + # Valid modules should work even with on_error='fail' + self.ctx.set_forkserver_preload(['os', 'sys'], on_error='fail') r, w = self.ctx.Pipe(duplex=False) p = self.ctx.Process(target=self._send_value, args=(w, 'success')) @@ -89,6 +108,12 @@ def test_preload_valid_modules_with_raise_exceptions(self): self.assertEqual(result, 'success') self.assertEqual(p.exitcode, 0) + def test_preload_invalid_on_error_value(self): + """Test that invalid on_error values raise ValueError.""" + with self.assertRaises(ValueError) as cm: + self.ctx.set_forkserver_preload(['os'], on_error='invalid') + self.assertIn("on_error must be 'ignore', 'warn', or 'fail'", str(cm.exception)) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst deleted file mode 100644 index b7e6c1674cdc3b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-101100.frksvr.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add a ``raise_exceptions`` keyword-only parameter to -:func:`multiprocessing.set_forkserver_preload`. When set to ``True``, -:exc:`ImportError` exceptions during module preloading will be raised instead -of being silently ignored. Defaults to ``False`` for backward compatibility. -Contributed by Nick Neumann. diff --git a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst new file mode 100644 index 00000000000000..316016a8c480a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst @@ -0,0 +1,5 @@ +Add an ``on_error`` keyword-only parameter to +:func:`multiprocessing.set_forkserver_preload` to control how import failures +during module preloading are handled. Accepts ``'ignore'`` (default, silent), +``'warn'`` (emit :exc:`ImportWarning`), or ``'fail'`` (raise exception). +Contributed by Nick Neumann. From 84c9e5b2fc81b23cdde90935522c8f3dd2c52de8 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:24:14 +0000 Subject: [PATCH 05/17] Fix unused import and make __notes__ test more robust - Remove unused warnings import - Use getattr() to safely check for __notes__ attribute - Add assertion for notes existence before checking content --- Lib/test/test_multiprocessing_forkserver/test_preload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 647d4ff89052e0..ea4d2f74cbc4c3 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -3,7 +3,6 @@ import multiprocessing import multiprocessing.forkserver import unittest -import warnings class TestForkserverPreload(unittest.TestCase): @@ -86,7 +85,9 @@ def test_preload_on_error_fail_breaks_context(self): with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)) as cm: p.start() # Exception raised here # Verify that the helpful note was added - self.assertIn('Forkserver process may have crashed', str(cm.exception.__notes__[0])) + notes = getattr(cm.exception, '__notes__', []) + self.assertTrue(notes, "Expected exception to have __notes__") + self.assertIn('Forkserver process may have crashed', notes[0]) finally: # Ensure pipes are closed even if exception is raised w.close() From a3992182592e7dd5f555251a62dff5213f28766d Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:27:18 +0000 Subject: [PATCH 06/17] Fix warn mode to work when warnings are configured as errors Use warnings.catch_warnings() context manager to ensure ImportWarning is always emitted and never converted to an error, even when the test environment has warnings configured with -W error. --- Lib/multiprocessing/forkserver.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index e3a2e7bb5c3e05..042feb65d6f8ec 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -244,11 +244,13 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, raise case 'warn': import warnings - warnings.warn( - f"Failed to import __main__ from {main_path!r}: {e}", - ImportWarning, - stacklevel=2 - ) + with warnings.catch_warnings(): + warnings.simplefilter('always', ImportWarning) + warnings.warn( + f"Failed to import __main__ from {main_path!r}: {e}", + ImportWarning, + stacklevel=2 + ) case 'ignore': pass finally: @@ -262,11 +264,13 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, raise case 'warn': import warnings - warnings.warn( - f"Failed to preload module {modname!r}: {e}", - ImportWarning, - stacklevel=2 - ) + with warnings.catch_warnings(): + warnings.simplefilter('always', ImportWarning) + warnings.warn( + f"Failed to preload module {modname!r}: {e}", + ImportWarning, + stacklevel=2 + ) case 'ignore': pass From 9c3ba8418aee231582b713e91d556784b198f27c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:35:07 +0000 Subject: [PATCH 07/17] Change __main__ warning message from 'import' to 'preload' For consistency with module preload warnings, use 'Failed to preload __main__' instead of 'Failed to import __main__'. --- Lib/multiprocessing/forkserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 042feb65d6f8ec..a5036ec92d4da9 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -247,7 +247,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, with warnings.catch_warnings(): warnings.simplefilter('always', ImportWarning) warnings.warn( - f"Failed to import __main__ from {main_path!r}: {e}", + f"Failed to preload __main__ from {main_path!r}: {e}", ImportWarning, stacklevel=2 ) From 70c05d87ab682d7a00818495c555a0ee078614f3 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:40:16 +0000 Subject: [PATCH 08/17] Refactor set_forkserver_preload to use _handle_preload helper Extract preload logic into a separate _handle_preload() function to enable targeted unit testing. Add comprehensive unit tests for both module and __main__ preload with all three on_error modes. New tests: - test_handle_preload_main_on_error_{fail,warn,ignore} - test_handle_preload_module_on_error_{fail,warn,ignore} - test_handle_preload_main_valid - test_handle_preload_combined Total test count increased from 6 to 14 tests. --- Lib/multiprocessing/forkserver.py | 108 ++++++++++-------- .../test_preload.py | 86 ++++++++++++++ 2 files changed, 148 insertions(+), 46 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index a5036ec92d4da9..aac1110824b3a3 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -219,6 +219,67 @@ def ensure_running(self): # # +def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): + """Handle module preloading with configurable error handling. + + Args: + preload: List of module names to preload. + main_path: Path to __main__ module if '__main__' is in preload. + sys_path: sys.path to use for imports (None means use current). + on_error: How to handle import errors ('ignore', 'warn', or 'fail'). + """ + if not preload: + return + + if sys_path is not None: + sys.path[:] = sys_path + + if '__main__' in preload and main_path is not None: + process.current_process()._inheriting = True + try: + spawn.import_main_path(main_path) + except Exception as e: + match on_error: + case 'fail': + raise + case 'warn': + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('always', ImportWarning) + warnings.warn( + f"Failed to preload __main__ from {main_path!r}: {e}", + ImportWarning, + stacklevel=2 + ) + case 'ignore': + pass + finally: + del process.current_process()._inheriting + + for modname in preload: + try: + __import__(modname) + except ImportError as e: + match on_error: + case 'fail': + raise + case 'warn': + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('always', ImportWarning) + warnings.warn( + f"Failed to preload module {modname!r}: {e}", + ImportWarning, + stacklevel=2 + ) + case 'ignore': + pass + + # gh-135335: flush stdout/stderr in case any of the preloaded modules + # wrote to them, otherwise children might inherit buffered data + util._flush_std_streams() + + def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, *, authkey_r=None, on_error='ignore'): """Run forkserver.""" @@ -231,52 +292,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, else: authkey = b'' - if preload: - if sys_path is not None: - sys.path[:] = sys_path - if '__main__' in preload and main_path is not None: - process.current_process()._inheriting = True - try: - spawn.import_main_path(main_path) - except Exception as e: - match on_error: - case 'fail': - raise - case 'warn': - import warnings - with warnings.catch_warnings(): - warnings.simplefilter('always', ImportWarning) - warnings.warn( - f"Failed to preload __main__ from {main_path!r}: {e}", - ImportWarning, - stacklevel=2 - ) - case 'ignore': - pass - finally: - del process.current_process()._inheriting - for modname in preload: - try: - __import__(modname) - except ImportError as e: - match on_error: - case 'fail': - raise - case 'warn': - import warnings - with warnings.catch_warnings(): - warnings.simplefilter('always', ImportWarning) - warnings.warn( - f"Failed to preload module {modname!r}: {e}", - ImportWarning, - stacklevel=2 - ) - case 'ignore': - pass - - # gh-135335: flush stdout/stderr in case any of the preloaded modules - # wrote to them, otherwise children might inherit buffered data - util._flush_std_streams() + _handle_preload(preload, main_path, sys_path, on_error) util._close_stdin() diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index ea4d2f74cbc4c3..194030c750d079 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -2,7 +2,10 @@ import multiprocessing import multiprocessing.forkserver +import os +import tempfile import unittest +from multiprocessing.forkserver import _handle_preload class TestForkserverPreload(unittest.TestCase): @@ -116,5 +119,88 @@ def test_preload_invalid_on_error_value(self): self.assertIn("on_error must be 'ignore', 'warn', or 'fail'", str(cm.exception)) +class TestHandlePreload(unittest.TestCase): + """Unit tests for _handle_preload() function.""" + + def test_handle_preload_main_on_error_fail(self): + """Test that __main__ import failures raise with on_error='fail'.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write('raise RuntimeError("test error in __main__")\n') + bad_main_path = f.name + + try: + with self.assertRaises(RuntimeError) as cm: + _handle_preload(['__main__'], main_path=bad_main_path, on_error='fail') + self.assertIn("test error in __main__", str(cm.exception)) + finally: + os.unlink(bad_main_path) + + def test_handle_preload_main_on_error_warn(self): + """Test that __main__ import failures warn with on_error='warn'.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write('raise ImportError("test import error")\n') + bad_main_path = f.name + + try: + with self.assertWarns(ImportWarning) as cm: + _handle_preload(['__main__'], main_path=bad_main_path, on_error='warn') + self.assertIn("Failed to preload __main__", str(cm.warning)) + self.assertIn("test import error", str(cm.warning)) + finally: + os.unlink(bad_main_path) + + def test_handle_preload_main_on_error_ignore(self): + """Test that __main__ import failures are ignored with on_error='ignore'.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write('raise ImportError("test import error")\n') + bad_main_path = f.name + + try: + # Should not raise + _handle_preload(['__main__'], main_path=bad_main_path, on_error='ignore') + finally: + os.unlink(bad_main_path) + + def test_handle_preload_main_valid(self): + """Test that valid __main__ preload works.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write('test_var = 42\n') + valid_main_path = f.name + + try: + _handle_preload(['__main__'], main_path=valid_main_path, on_error='fail') + # Should complete without raising + finally: + os.unlink(valid_main_path) + + def test_handle_preload_module_on_error_fail(self): + """Test that module import failures raise with on_error='fail'.""" + with self.assertRaises(ModuleNotFoundError): + _handle_preload(['nonexistent_test_module_xyz'], on_error='fail') + + def test_handle_preload_module_on_error_warn(self): + """Test that module import failures warn with on_error='warn'.""" + with self.assertWarns(ImportWarning) as cm: + _handle_preload(['nonexistent_test_module_xyz'], on_error='warn') + self.assertIn("Failed to preload module", str(cm.warning)) + + def test_handle_preload_module_on_error_ignore(self): + """Test that module import failures are ignored with on_error='ignore'.""" + # Should not raise + _handle_preload(['nonexistent_test_module_xyz'], on_error='ignore') + + def test_handle_preload_combined(self): + """Test preloading both __main__ and modules.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + f.write('import sys\n') + valid_main_path = f.name + + try: + _handle_preload(['__main__', 'os', 'sys'], main_path=valid_main_path, on_error='fail') + # Should complete without raising + finally: + os.unlink(valid_main_path) + + if __name__ == '__main__': unittest.main() From 045be9204362f723971714236991bfa9ac36b415 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:47:11 +0000 Subject: [PATCH 09/17] Simplify temporary file handling in tests Use delete=True (default) for NamedTemporaryFile instead of manually managing cleanup with try/finally blocks. Keep file open during test and let context manager handle automatic deletion. Also remove now- unused os import. --- .../test_preload.py | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 194030c750d079..08c8717b53e6d1 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -2,7 +2,6 @@ import multiprocessing import multiprocessing.forkserver -import os import tempfile import unittest from multiprocessing.forkserver import _handle_preload @@ -124,54 +123,38 @@ class TestHandlePreload(unittest.TestCase): def test_handle_preload_main_on_error_fail(self): """Test that __main__ import failures raise with on_error='fail'.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('raise RuntimeError("test error in __main__")\n') - bad_main_path = f.name - - try: + f.flush() with self.assertRaises(RuntimeError) as cm: - _handle_preload(['__main__'], main_path=bad_main_path, on_error='fail') + _handle_preload(['__main__'], main_path=f.name, on_error='fail') self.assertIn("test error in __main__", str(cm.exception)) - finally: - os.unlink(bad_main_path) def test_handle_preload_main_on_error_warn(self): """Test that __main__ import failures warn with on_error='warn'.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('raise ImportError("test import error")\n') - bad_main_path = f.name - - try: + f.flush() with self.assertWarns(ImportWarning) as cm: - _handle_preload(['__main__'], main_path=bad_main_path, on_error='warn') + _handle_preload(['__main__'], main_path=f.name, on_error='warn') self.assertIn("Failed to preload __main__", str(cm.warning)) self.assertIn("test import error", str(cm.warning)) - finally: - os.unlink(bad_main_path) def test_handle_preload_main_on_error_ignore(self): """Test that __main__ import failures are ignored with on_error='ignore'.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('raise ImportError("test import error")\n') - bad_main_path = f.name - - try: + f.flush() # Should not raise - _handle_preload(['__main__'], main_path=bad_main_path, on_error='ignore') - finally: - os.unlink(bad_main_path) + _handle_preload(['__main__'], main_path=f.name, on_error='ignore') def test_handle_preload_main_valid(self): """Test that valid __main__ preload works.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('test_var = 42\n') - valid_main_path = f.name - - try: - _handle_preload(['__main__'], main_path=valid_main_path, on_error='fail') + f.flush() + _handle_preload(['__main__'], main_path=f.name, on_error='fail') # Should complete without raising - finally: - os.unlink(valid_main_path) def test_handle_preload_module_on_error_fail(self): """Test that module import failures raise with on_error='fail'.""" @@ -191,15 +174,11 @@ def test_handle_preload_module_on_error_ignore(self): def test_handle_preload_combined(self): """Test preloading both __main__ and modules.""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('import sys\n') - valid_main_path = f.name - - try: - _handle_preload(['__main__', 'os', 'sys'], main_path=valid_main_path, on_error='fail') + f.flush() + _handle_preload(['__main__', 'os', 'sys'], main_path=f.name, on_error='fail') # Should complete without raising - finally: - os.unlink(valid_main_path) if __name__ == '__main__': From 6d4c5216eb6fb7c6b0463b123dda9fe60bc7adc9 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 08:51:35 +0000 Subject: [PATCH 10/17] Remove obvious comments and improve import style in tests - Remove comments that restate what the code obviously does - Change from 'from multiprocessing.forkserver import _handle_preload' to 'from multiprocessing import forkserver' and use qualified calls - Makes code cleaner and follows better import conventions --- .../test_preload.py | 41 ++++++------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 08c8717b53e6d1..7f56f60c43240a 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -1,10 +1,9 @@ """Tests for forkserver preload functionality.""" import multiprocessing -import multiprocessing.forkserver import tempfile import unittest -from multiprocessing.forkserver import _handle_preload +from multiprocessing import forkserver class TestForkserverPreload(unittest.TestCase): @@ -12,12 +11,10 @@ class TestForkserverPreload(unittest.TestCase): def setUp(self): self.ctx = multiprocessing.get_context('forkserver') - # Ensure clean state for each test - multiprocessing.forkserver._forkserver._stop() + forkserver._forkserver._stop() def tearDown(self): - # Clean up after tests - multiprocessing.forkserver._forkserver._stop() + forkserver._forkserver._stop() self.ctx.set_forkserver_preload([]) @staticmethod @@ -27,10 +24,8 @@ def _send_value(conn, value): def test_preload_on_error_ignore_default(self): """Test that invalid modules are silently ignored by default.""" - # With on_error='ignore' (default), invalid module is ignored self.ctx.set_forkserver_preload(['nonexistent_module_xyz']) - # Should be able to start a process without errors r, w = self.ctx.Pipe(duplex=False) p = self.ctx.Process(target=self._send_value, args=(w, 42)) p.start() @@ -46,7 +41,6 @@ def test_preload_on_error_ignore_explicit(self): """Test that invalid modules are silently ignored with on_error='ignore'.""" self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='ignore') - # Should be able to start a process without errors r, w = self.ctx.Pipe(duplex=False) p = self.ctx.Process(target=self._send_value, args=(w, 99)) p.start() @@ -62,7 +56,6 @@ def test_preload_on_error_warn(self): """Test that invalid modules emit warnings with on_error='warn'.""" self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='warn') - # Should still be able to start a process, warnings are in subprocess r, w = self.ctx.Pipe(duplex=False) p = self.ctx.Process(target=self._send_value, args=(w, 123)) p.start() @@ -78,26 +71,20 @@ def test_preload_on_error_fail_breaks_context(self): """Test that invalid modules with on_error='fail' breaks the forkserver.""" self.ctx.set_forkserver_preload(['nonexistent_module_xyz'], on_error='fail') - # The forkserver should fail to start when it tries to import - # The exception is raised during p.start() when trying to communicate - # with the failed forkserver process r, w = self.ctx.Pipe(duplex=False) try: p = self.ctx.Process(target=self._send_value, args=(w, 42)) with self.assertRaises((EOFError, ConnectionError, BrokenPipeError)) as cm: - p.start() # Exception raised here - # Verify that the helpful note was added + p.start() notes = getattr(cm.exception, '__notes__', []) self.assertTrue(notes, "Expected exception to have __notes__") self.assertIn('Forkserver process may have crashed', notes[0]) finally: - # Ensure pipes are closed even if exception is raised w.close() r.close() def test_preload_valid_modules_with_on_error_fail(self): """Test that valid modules work fine with on_error='fail'.""" - # Valid modules should work even with on_error='fail' self.ctx.set_forkserver_preload(['os', 'sys'], on_error='fail') r, w = self.ctx.Pipe(duplex=False) @@ -127,7 +114,7 @@ def test_handle_preload_main_on_error_fail(self): f.write('raise RuntimeError("test error in __main__")\n') f.flush() with self.assertRaises(RuntimeError) as cm: - _handle_preload(['__main__'], main_path=f.name, on_error='fail') + forkserver._handle_preload(['__main__'], main_path=f.name, on_error='fail') self.assertIn("test error in __main__", str(cm.exception)) def test_handle_preload_main_on_error_warn(self): @@ -136,7 +123,7 @@ def test_handle_preload_main_on_error_warn(self): f.write('raise ImportError("test import error")\n') f.flush() with self.assertWarns(ImportWarning) as cm: - _handle_preload(['__main__'], main_path=f.name, on_error='warn') + forkserver._handle_preload(['__main__'], main_path=f.name, on_error='warn') self.assertIn("Failed to preload __main__", str(cm.warning)) self.assertIn("test import error", str(cm.warning)) @@ -145,40 +132,36 @@ def test_handle_preload_main_on_error_ignore(self): with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('raise ImportError("test import error")\n') f.flush() - # Should not raise - _handle_preload(['__main__'], main_path=f.name, on_error='ignore') + forkserver._handle_preload(['__main__'], main_path=f.name, on_error='ignore') def test_handle_preload_main_valid(self): """Test that valid __main__ preload works.""" with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('test_var = 42\n') f.flush() - _handle_preload(['__main__'], main_path=f.name, on_error='fail') - # Should complete without raising + forkserver._handle_preload(['__main__'], main_path=f.name, on_error='fail') def test_handle_preload_module_on_error_fail(self): """Test that module import failures raise with on_error='fail'.""" with self.assertRaises(ModuleNotFoundError): - _handle_preload(['nonexistent_test_module_xyz'], on_error='fail') + forkserver._handle_preload(['nonexistent_test_module_xyz'], on_error='fail') def test_handle_preload_module_on_error_warn(self): """Test that module import failures warn with on_error='warn'.""" with self.assertWarns(ImportWarning) as cm: - _handle_preload(['nonexistent_test_module_xyz'], on_error='warn') + forkserver._handle_preload(['nonexistent_test_module_xyz'], on_error='warn') self.assertIn("Failed to preload module", str(cm.warning)) def test_handle_preload_module_on_error_ignore(self): """Test that module import failures are ignored with on_error='ignore'.""" - # Should not raise - _handle_preload(['nonexistent_test_module_xyz'], on_error='ignore') + forkserver._handle_preload(['nonexistent_test_module_xyz'], on_error='ignore') def test_handle_preload_combined(self): """Test preloading both __main__ and modules.""" with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as f: f.write('import sys\n') f.flush() - _handle_preload(['__main__', 'os', 'sys'], main_path=f.name, on_error='fail') - # Should complete without raising + forkserver._handle_preload(['__main__', 'os', 'sys'], main_path=f.name, on_error='fail') if __name__ == '__main__': From 30c2cf890103b5ff4f9c90ffb346b2d474c44c71 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 09:04:43 +0000 Subject: [PATCH 11/17] Fix warn mode to work when warnings are configured as errors Remove catch_warnings() workaround from production code and instead fix the root cause: tests were allowing sys.warnoptions to leak into the forkserver subprocess via _args_from_interpreter_flags(). When CI runs with -W error, the forkserver subprocess was inheriting this flag and converting our ImportWarning calls into exceptions, causing it to crash. Solution: Save and clear sys.warnoptions in test setUp, restore in tearDown. This gives the forkserver subprocess a clean warning state where warnings.warn() works as intended. Also remove unnecessary set_forkserver_preload([]) call from tearDown. --- Lib/multiprocessing/forkserver.py | 24 ++++++++----------- .../test_preload.py | 5 +++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index aac1110824b3a3..385c1f4b335a3c 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -244,13 +244,11 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): raise case 'warn': import warnings - with warnings.catch_warnings(): - warnings.simplefilter('always', ImportWarning) - warnings.warn( - f"Failed to preload __main__ from {main_path!r}: {e}", - ImportWarning, - stacklevel=2 - ) + warnings.warn( + f"Failed to preload __main__ from {main_path!r}: {e}", + ImportWarning, + stacklevel=2 + ) case 'ignore': pass finally: @@ -265,13 +263,11 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): raise case 'warn': import warnings - with warnings.catch_warnings(): - warnings.simplefilter('always', ImportWarning) - warnings.warn( - f"Failed to preload module {modname!r}: {e}", - ImportWarning, - stacklevel=2 - ) + warnings.warn( + f"Failed to preload module {modname!r}: {e}", + ImportWarning, + stacklevel=2 + ) case 'ignore': pass diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 7f56f60c43240a..3808947163603a 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -1,6 +1,7 @@ """Tests for forkserver preload functionality.""" import multiprocessing +import sys import tempfile import unittest from multiprocessing import forkserver @@ -10,12 +11,14 @@ class TestForkserverPreload(unittest.TestCase): """Tests for forkserver preload functionality.""" def setUp(self): + self._saved_warnoptions = sys.warnoptions.copy() + sys.warnoptions.clear() self.ctx = multiprocessing.get_context('forkserver') forkserver._forkserver._stop() def tearDown(self): + sys.warnoptions[:] = self._saved_warnoptions forkserver._forkserver._stop() - self.ctx.set_forkserver_preload([]) @staticmethod def _send_value(conn, value): From bad96917b39f6464390934f06ddad866121b065a Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 09:09:40 +0000 Subject: [PATCH 12/17] Add comments explaining exception catching strategy Explain why we catch broad Exception for import_main_path() (it uses runpy.run_path() which can raise any exception) vs only ImportError for regular __import__() calls. --- Lib/multiprocessing/forkserver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 385c1f4b335a3c..1d16398302d74d 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -239,6 +239,9 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): try: spawn.import_main_path(main_path) except Exception as e: + # Catch broad Exception because import_main_path() uses + # runpy.run_path() which executes the script and can raise + # any exception, not just ImportError match on_error: case 'fail': raise @@ -258,6 +261,7 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): try: __import__(modname) except ImportError as e: + # Only catch ImportError for regular module imports match on_error: case 'fail': raise From 9d8125f38c2a8e6cecebf7a045d6644144d020aa Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 09:11:48 +0000 Subject: [PATCH 13/17] Use double quotes for string values in documentation Change from single quotes to double quotes when describing string values like "ignore", "warn", and "fail" in docstrings and documentation for consistency. --- Doc/library/multiprocessing.rst | 6 +++--- Lib/multiprocessing/context.py | 4 ++-- Lib/multiprocessing/forkserver.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index a6a312f10b75ef..16d5c75a4353f4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1222,9 +1222,9 @@ Miscellaneous launched (before creating a :class:`Pool` or starting a :class:`Process`). The *on_error* parameter controls how :exc:`ImportError` exceptions during - module preloading are handled: ``'ignore'`` (default) silently ignores - failures, ``'warn'`` causes the forkserver subprocess to emit an - :exc:`ImportWarning` to stderr, and ``'fail'`` causes the forkserver + module preloading are handled: ``"ignore"`` (default) silently ignores + failures, ``"warn"`` causes the forkserver subprocess to emit an + :exc:`ImportWarning` to stderr, and ``"fail"`` causes the forkserver subprocess to exit with the exception traceback on stderr, making subsequent process creation fail with :exc:`EOFError` or :exc:`ConnectionError`. diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 3d005aeab4ccf7..a73261cde856bb 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -181,8 +181,8 @@ def set_forkserver_preload(self, module_names, *, on_error='ignore'): '''Set list of module names to try to load in forkserver process. The on_error parameter controls how import failures are handled: - 'ignore' (default) silently ignores failures, 'warn' emits warnings, - and 'fail' raises exceptions breaking the forkserver context. + "ignore" (default) silently ignores failures, "warn" emits warnings, + and "fail" raises exceptions breaking the forkserver context. ''' from .forkserver import set_forkserver_preload set_forkserver_preload(module_names, on_error=on_error) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 1d16398302d74d..02b810d73c295a 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -69,8 +69,8 @@ def set_forkserver_preload(self, modules_names, *, on_error='ignore'): '''Set list of module names to try to load in forkserver process. The on_error parameter controls how import failures are handled: - 'ignore' (default) silently ignores failures, 'warn' emits warnings, - and 'fail' raises exceptions breaking the forkserver context. + "ignore" (default) silently ignores failures, "warn" emits warnings, + and "fail" raises exceptions breaking the forkserver context. ''' if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') @@ -226,7 +226,7 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): preload: List of module names to preload. main_path: Path to __main__ module if '__main__' is in preload. sys_path: sys.path to use for imports (None means use current). - on_error: How to handle import errors ('ignore', 'warn', or 'fail'). + on_error: How to handle import errors ("ignore", "warn", or "fail"). """ if not preload: return From 2f8edb89a18d69d518fbf43c192b0db2d5207768 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 09:49:49 +0000 Subject: [PATCH 14/17] Fix warn mode to work when warnings are configured as errors Previous approach of clearing sys.warnoptions broke when CI used -bb flag, because _args_from_interpreter_flags() expects certain warning options to exist based on sys.flags settings. Instead of clearing completely, filter out only the specific warning options that would convert ImportWarning to errors: - 'error' (converts all warnings) - 'error::ImportWarning' (converts ImportWarning specifically) This preserves options like 'error::BytesWarning' that subprocess's _args_from_interpreter_flags() needs to remove, preventing ValueError. --- .../test_multiprocessing_forkserver/test_preload.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 3808947163603a..3dc5926ae0ba89 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -12,7 +12,15 @@ class TestForkserverPreload(unittest.TestCase): def setUp(self): self._saved_warnoptions = sys.warnoptions.copy() - sys.warnoptions.clear() + # Remove warning options that would convert ImportWarning to errors: + # - 'error' converts all warnings to errors + # - 'error::ImportWarning' specifically converts ImportWarning + # Keep other specific options like 'error::BytesWarning' that + # subprocess's _args_from_interpreter_flags() expects to remove + sys.warnoptions[:] = [ + opt for opt in sys.warnoptions + if opt not in ('error', 'error::ImportWarning') + ] self.ctx = multiprocessing.get_context('forkserver') forkserver._forkserver._stop() From 622345d2518219c953c2065afa4332398642bcda Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 10:12:57 +0000 Subject: [PATCH 15/17] Add Gregory P. Smith to NEWS entry contributors --- .../next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst index 316016a8c480a5..b1efd9c014f1f4 100644 --- a/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst +++ b/Misc/NEWS.d/next/Library/2025-11-22-20-30-00.gh-issue-141860.frksvr.rst @@ -2,4 +2,4 @@ Add an ``on_error`` keyword-only parameter to :func:`multiprocessing.set_forkserver_preload` to control how import failures during module preloading are handled. Accepts ``'ignore'`` (default, silent), ``'warn'`` (emit :exc:`ImportWarning`), or ``'fail'`` (raise exception). -Contributed by Nick Neumann. +Contributed by Nick Neumann and Gregory P. Smith. From 42e8eb1a726cc0079dceb8a6782a43f202152d10 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 19:21:13 +0000 Subject: [PATCH 16/17] Simplify comments and exception note message Remove unnecessary comment and simplify the exception note to just say 'Check stderr.' instead of the more verbose message. --- Lib/multiprocessing/forkserver.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 02b810d73c295a..7d9033415c0863 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -120,12 +120,11 @@ def connect_to_new_process(self, fds): connection.deliver_challenge( wrapped_client, self._forkserver_authkey) except (EOFError, ConnectionError, BrokenPipeError) as exc: - # Add helpful context if forkserver likely crashed during preload if (self._preload_modules and self._preload_on_error == 'fail'): exc.add_note( "Forkserver process may have crashed during module " - "preloading. Check stderr for ImportError traceback." + "preloading. Check stderr." ) raise finally: @@ -261,7 +260,6 @@ def _handle_preload(preload, main_path=None, sys_path=None, on_error='ignore'): try: __import__(modname) except ImportError as e: - # Only catch ImportError for regular module imports match on_error: case 'fail': raise From 64ca5a0d9f008fa4ae31f0a4ccafbf79bbe2a559 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith via Claude Code for the web" Date: Sun, 23 Nov 2025 19:30:11 +0000 Subject: [PATCH 17/17] Update _send_value docstring to explain pickling requirement Clarify that _send_value is a static method specifically to be picklable as a Process target function. --- Lib/test/test_multiprocessing_forkserver/test_preload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_multiprocessing_forkserver/test_preload.py b/Lib/test/test_multiprocessing_forkserver/test_preload.py index 3dc5926ae0ba89..7bc9ec18a3d471 100644 --- a/Lib/test/test_multiprocessing_forkserver/test_preload.py +++ b/Lib/test/test_multiprocessing_forkserver/test_preload.py @@ -30,7 +30,7 @@ def tearDown(self): @staticmethod def _send_value(conn, value): - """Helper to send a value through a connection.""" + """Send value through connection. Static method to be picklable as Process target.""" conn.send(value) def test_preload_on_error_ignore_default(self):