From c6a3bea2686423250bdf772a3c86b15a591be523 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:09:02 +0100 Subject: [PATCH 1/5] gh-144503: Pass `sys.argv` as separate command line arguments. The maximum length of a single command line argument is more restricted than the total size of all command line arguments together. --- Lib/multiprocessing/forkserver.py | 11 ++++++++--- Lib/test/_test_multiprocessing.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index d89b24ac59bec0..bce7b37a17886f 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -172,8 +172,6 @@ def ensure_running(self): main_kws['sys_path'] = data['sys_path'] if 'init_main_from_path' in data: main_kws['main_path'] = data['init_main_from_path'] - if 'sys_argv' in data: - main_kws['sys_argv'] = data['sys_argv'] if self._preload_on_error != 'ignore': main_kws['on_error'] = self._preload_on_error @@ -197,6 +195,8 @@ def ensure_running(self): exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] + if self._preload_modules: + args += data["sys_argv"] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) @@ -282,7 +282,7 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, sys_argv=None, authkey_r=None, on_error='ignore'): + *, authkey_r=None, on_error='ignore'): """Run forkserver.""" if authkey_r is not None: try: @@ -293,6 +293,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, else: authkey = b'' + if preload: + sys_argv = sys.argv[1:] + else: + sys_argv = None + _handle_preload(preload, main_path, sys_path, sys_argv, on_error) util._close_stdin() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cc07062eee6f98..20d5948a46479b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -20,6 +20,7 @@ import collections.abc import socket import random +import resource import logging import shutil import subprocess @@ -7106,6 +7107,28 @@ def test_preload_main_sys_argv(self): '', ]) + def test_preload_main_sys_argv_limits(self): + # gh-144503: Check that sys.argv is set before __main__ is pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + max_str_arglen = 32 * resource.getpagesize() + argv = ["a" * (max_str_arglen - 1), "b"] + name = os.path.join(os.path.dirname(__file__), 'mp_preload_sysargv.py') + _, out, err = test.support.script_helper.assert_python_ok( + name, *argv) + self.assertEqual(err, b'') + + out = out.decode().split("\n") + expected_argv = str(argv) + self.assertEqual(out, [ + f"module:{expected_argv}", + f"fun:{expected_argv}", + f"module:{expected_argv}", + f"fun:{expected_argv}", + '', + ]) + # # Mixins # From 5b8ff7bf337c4cee39a9bdcb07ca1a54db07a100 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:29:58 +0100 Subject: [PATCH 2/5] gh-144503: Add news entry --- .../Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst diff --git a/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst new file mode 100644 index 00000000000000..684ee1ab716c8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst @@ -0,0 +1,3 @@ +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented starting of +the forkserver if the total length of command line arguments in ``sys.argv`` +exceeded the maximum length of a single command line argument. From 0507aa09326c0d541c2e10390d88215e799f29bf Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:33:34 +0100 Subject: [PATCH 3/5] gh-144503: Accept non-importable resource The `resource` module is not available in a WASI environment. --- Lib/test/_test_multiprocessing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 20d5948a46479b..34bed9ac1e26a6 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -20,7 +20,6 @@ import collections.abc import socket import random -import resource import logging import shutil import subprocess @@ -82,6 +81,11 @@ except ImportError: msvcrt = None +try: + import resource +except ImportError: + resource = None + if support.HAVE_ASAN_FORK_BUG: # gh-89363: Skip multiprocessing tests if Python is built with ASAN to From be2e1f050d65dec1185aa25113424cf9b3334ca8 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Mon, 9 Feb 2026 09:35:50 +0100 Subject: [PATCH 4/5] gh-144503: Send initialization data for preload over pipe instead of argv The allowed size for argv is limited, while the amount of data that can be sent over a pipe is virtually unlimited. --- Lib/multiprocessing/forkserver.py | 72 +++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index bce7b37a17886f..c7f2d0191b8cc0 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,5 +1,6 @@ import atexit import errno +import json import os import selectors import signal @@ -163,17 +164,19 @@ def ensure_running(self): self._forkserver_pid = None cmd = ('from multiprocessing.forkserver import main; ' + - 'main(%d, %d, %r, **%r)') + 'main(listener_fd=%d, alive_r=%d, init_r=%d)') - main_kws = {} if self._preload_modules: data = spawn.get_preparation_data('ignore') - if 'sys_path' in data: - main_kws['sys_path'] = data['sys_path'] - if 'init_main_from_path' in data: - main_kws['main_path'] = data['init_main_from_path'] - if self._preload_on_error != 'ignore': - main_kws['on_error'] = self._preload_on_error + preload_kwargs = { + "preload": self._preload_modules, + "sys_path": data["sys_path"], + "main_path": data.get("init_main_from_path", None), + "sys_argv": data["sys_argv"], + "on_error": self._preload_on_error, + } + else: + preload_kwargs = None with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -186,32 +189,31 @@ def ensure_running(self): # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() # A short lived pipe to initialize the forkserver authkey. - authkey_r, authkey_w = os.pipe() + init_r, init_w = os.pipe() try: - fds_to_pass = [listener.fileno(), alive_r, authkey_r] - main_kws['authkey_r'] = authkey_r - cmd %= (listener.fileno(), alive_r, self._preload_modules, - main_kws) + fds_to_pass = [listener.fileno(), alive_r, init_r] + cmd %= (listener.fileno(), alive_r, init_r) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] - if self._preload_modules: - args += data["sys_argv"] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) - os.close(authkey_w) + os.close(init_w) raise finally: os.close(alive_r) - os.close(authkey_r) + os.close(init_r) # Authenticate our control socket to prevent access from # processes we have not shared this key with. try: self._forkserver_authkey = os.urandom(_AUTHKEY_LEN) - os.write(authkey_w, self._forkserver_authkey) + os.write(init_w, self._forkserver_authkey) + preload_data = json.dumps(preload_kwargs).encode() + os.write(init_w, struct.pack("Q", len(preload_data))) + os.write(init_w, preload_data) finally: - os.close(authkey_w) + os.close(init_w) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid @@ -281,24 +283,22 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, util._flush_std_streams() -def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, authkey_r=None, on_error='ignore'): +def main(listener_fd, alive_r, init_r): """Run forkserver.""" - if authkey_r is not None: - try: - authkey = os.read(authkey_r, _AUTHKEY_LEN) - assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' - finally: - os.close(authkey_r) - else: - authkey = b'' - - if preload: - sys_argv = sys.argv[1:] - else: - sys_argv = None - - _handle_preload(preload, main_path, sys_path, sys_argv, on_error) + try: + authkey = os.read(init_r, _AUTHKEY_LEN) + assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' + + preload_data_len, = struct.unpack("Q", os.read(init_r, struct.calcsize("Q"))) + preload_data = b"" + while len(preload_data) < preload_data_len: + preload_data += os.read(init_r, preload_data_len - len(preload_data)) + preload_kwargs = json.loads(preload_data.decode()) + finally: + os.close(init_r) + + if preload_kwargs: + _handle_preload(**preload_kwargs) util._close_stdin() From 80e7a0eebb8070703a29e550ccc684c497bdb1c9 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Tue, 7 Apr 2026 01:01:33 +0000 Subject: [PATCH 5/5] gh-144503: Harden init pipe I/O in forkserver - Use buffered os.fdopen() for both writing and reading the init pipe so that preload payloads larger than PIPE_BUF are written and read fully. A bare os.write() may short-write large buffers. - Use data.get() for preparation-data keys rather than direct indexing to avoid coupling to spawn.get_preparation_data() internals. --- Lib/multiprocessing/forkserver.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index c7f2d0191b8cc0..1cdb4a3b6fbf29 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -170,9 +170,9 @@ def ensure_running(self): data = spawn.get_preparation_data('ignore') preload_kwargs = { "preload": self._preload_modules, - "sys_path": data["sys_path"], - "main_path": data.get("init_main_from_path", None), - "sys_argv": data["sys_argv"], + "sys_path": data.get("sys_path"), + "main_path": data.get("init_main_from_path"), + "sys_argv": data.get("sys_argv"), "on_error": self._preload_on_error, } else: @@ -208,10 +208,13 @@ def ensure_running(self): # processes we have not shared this key with. try: self._forkserver_authkey = os.urandom(_AUTHKEY_LEN) - os.write(init_w, self._forkserver_authkey) preload_data = json.dumps(preload_kwargs).encode() - os.write(init_w, struct.pack("Q", len(preload_data))) - os.write(init_w, preload_data) + # Use a buffered writer so that payloads larger than + # PIPE_BUF are written fully (os.write may short-write). + with os.fdopen(init_w, 'wb', closefd=False) as f: + f.write(self._forkserver_authkey) + f.write(struct.pack("Q", len(preload_data))) + f.write(preload_data) finally: os.close(init_w) self._forkserver_address = address @@ -286,14 +289,14 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, def main(listener_fd, alive_r, init_r): """Run forkserver.""" try: - authkey = os.read(init_r, _AUTHKEY_LEN) - assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' - - preload_data_len, = struct.unpack("Q", os.read(init_r, struct.calcsize("Q"))) - preload_data = b"" - while len(preload_data) < preload_data_len: - preload_data += os.read(init_r, preload_data_len - len(preload_data)) - preload_kwargs = json.loads(preload_data.decode()) + # Buffered reader handles short reads on the length prefix and body. + with os.fdopen(init_r, 'rb', closefd=False) as f: + authkey = f.read(_AUTHKEY_LEN) + assert len(authkey) == _AUTHKEY_LEN, ( + f'{len(authkey)} < {_AUTHKEY_LEN}') + preload_data_len, = struct.unpack("Q", + f.read(struct.calcsize("Q"))) + preload_kwargs = json.loads(f.read(preload_data_len)) finally: os.close(init_r)