From 6d86d82295fedffca57460469d381a93fc46ef5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 26 Mar 2025 09:07:01 +0000 Subject: [PATCH] GH-131556: calculate PYBUILDDIR in configure instead of sysconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Right now, this value is being calculated during the build process when calling `python -m sysconfig --generate-posix-vars`. This command, ammong other things, calculates the PYBUILDDIR value and writes it to a pybuilddir.txt, making it available to the Makefile this way. This is problematic because it makes it impossible to write Makefile rules with targets under PYBUILDDIR — the target and prerequisites of rule are always expanded immediatily as the Makefile is read, only the rule recipe is deferred [1], meaning that all targets must be known before any rule is executed. Since PYBUILDDIR is only known in the middle of the build process — after the target list has been built — we cannot write rules for files under it. We have had to worked around this limitation in a couple ways: - Extension modules, which need to be present in PYBUILDDIR to execute the interpreter in-tree, are built in the source tree, and afterwards symlinked to PYBUILDDIR once its value is known. - Instead of the sysconfigdata module and the sysconfig vars JSON file, instead of having theirn own target, are generated by the pybuilddir.txt target. Additionally, this limitation is also prone to cause issues such as GH-131556. That said, on top of the Makefile issues, PYBUILDDIR being calculated by sysconfig also creates its own additional complications, necessitating more workarounds and introducing unecessary complexity to the sysconfig data generation code — there's a chicken-and-egg problem in certain systems, where we need to know PYBUILDDIR to install the sysconfigdata module, but the sysconfigdata module is necessary to be avaible to calculate the value PYBUILDDIR [2]. We currently handle this by manually building a module object for sysconfigdata and inject it to sys.modules. By defining PYBUILDDIR directly in Makefile.pre.in, we solve all these problems. The current build system design is the result of a sucession small fixes adapting the original sysconfig code from over 15 years ago to work with the rest of codebase as it evolved. That is to say — I don't think there's any technical resoning behind this particular design, I believe it is simply the result of technical debt. Therefore, I don't see any reason why not to move the PYBUILDDIR definition over to Makefile.pre.in, which to me seems to make more sense [1] https://www.gnu.org/software/make/manual/html_node/Reading-Makefiles.html [2] https://github.com/python/cpython/blob/898e6b395e63ad7f8bbe421adf0af8947d429925/Lib/sysconfig/__main__.py#L206-L221 Signed-off-by: Filipe Laíns --- Lib/sysconfig/__main__.py | 30 ++---------- Makefile.pre.in | 76 +++++++++++++++++-------------- Modules/makesetup | 4 +- Tools/wasm/emscripten/__main__.py | 16 ++++--- Tools/wasm/wasi.py | 17 +++---- configure | 9 +++- configure.ac | 7 ++- 7 files changed, 79 insertions(+), 80 deletions(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index bc2197cfe79402..f8fff43713d594 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,13 +1,12 @@ import json import os import sys -import types from sysconfig import ( _ALWAYS_STR, + _PROJECT_BASE, _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, - get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,10 +160,8 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if get_config_var('Py_DEBUG') == '1': - pybuilddir += '-pydebug' - return pybuilddir + with open(os.path.join(_PROJECT_BASE, 'pybuilddir.txt')) as f: + return f.read() def _get_json_data_name(): @@ -203,23 +200,6 @@ def _generate_posix_vars(): name = _get_sysconfigdata_name() - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - # GH-127178: Since we started generating a .json file, we also need this to - # be able to run sysconfig.get_config_vars(). - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -243,10 +223,6 @@ def _generate_posix_vars(): print(f'Written {jsonfile}') - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - def _print_dict(title, data): for index, (key, value) in enumerate(sorted(data.items())): diff --git a/Makefile.pre.in b/Makefile.pre.in index 9658bfa44b98e4..a5780e896b2878 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -32,6 +32,7 @@ srcdir= @srcdir@ VPATH= @srcdir@ abs_srcdir= @abs_srcdir@ abs_builddir= @abs_builddir@ +PYBUILDDIR= build/lib.@MACHDEP@-@MULTIARCH@-@VERSION@-@ABIFLAGS@ CC= @CC@ @@ -135,6 +136,10 @@ MACHDEP= @MACHDEP@ MULTIARCH= @MULTIARCH@ MULTIARCH_CPPFLAGS = @MULTIARCH_CPPFLAGS@ +# Sysconfig data +SYSCONFIGDATA_NAME= @SYSCONFIGDATA_NAME@ +SYSCONFIGVARS_JSON_NAME= @SYSCONFIGVARS_JSON_NAME@ + # Install prefix for architecture-independent files prefix= @prefix@ @@ -730,11 +735,12 @@ list-targets: .PHONY: build_all build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil pybuilddir.txt \ + $(PYBUILDDIR)/build-details.json $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods + $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py python-config checksharedmods .PHONY: build_emscripten build_emscripten: build_wasm web_example @@ -917,27 +923,37 @@ clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) -platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt +platform: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform -# Create build directory and generate the sysconfig build-time data there. -# pybuilddir.txt contains the name of the build dir and is used for -# sys.path fixup -- see Modules/getpath.c. -# Since this step runs before shared modules are built, try to avoid bootstrap -# problems by creating a dummy pybuilddir.txt just to allow interpreter -# initialization to succeed. It will be overwritten by generate-posix-vars -# or removed in case of failure. -pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) - @echo "none" > ./pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\ - if test $$? -ne 0 ; then \ - echo "generate-posix-vars failed" ; \ - rm -f ./pybuilddir.txt ; \ - exit 1 ; \ - fi +# Create build directory and pybuilddir.txt. +# pybuilddir.txt which is used by Modules/getpath.c to detect the build directory. +pybuilddir.txt: + mkdir -p $(PYBUILDDIR) + printf $(PYBUILDDIR) >pybuilddir.txt + +SYSCONFIG_SRC= \ + $(srcdir)/Lib/sysconfig/__init__.py \ + $(srcdir)/Lib/sysconfig/__main__.py +SYSCONFIGVARS_DEPS= \ + $(SYSCONFIG_SRC) \ + Makefile \ + Python/sysmodule.o \ + Modules/posixmodule.o + +# Generate the sysconfig data module. +$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py $(PYBUILDDIR)/$(SYSCONFIGVARS_JSON_NAME): $(SYSCONFIGVARS_DEPS) $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars + @# Sanity check, to make sure sysconfig._get_sysconfigdata_name() returns the same value. + @for file in $@; do \ + if test ! -f $<; then \ + echo "generate-posix-vars didn't generate '$(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py'" \ + exit 1; \ + fi; \ + done -build-details.json: pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json +$(PYBUILDDIR)/build-details.json: $(PYTHON_FOR_BUILD_DEPS) $(PYBUILDDIR)/$(SYSCONFIGDATA_NAME).py Makefile pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py $@ # Build static library $(LIBRARY): $(LIBRARY_OBJS) @@ -1443,18 +1459,8 @@ $(LIBHACL_BLAKE2_A): $(LIBHACL_BLAKE2_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBHACL_BLAKE2_OBJS) -# create relative links from build/lib.platform/egg.so to Modules/egg.so -# pybuilddir.txt is created too late. We cannot use it in Makefile -# targets. ln --relative is not portable. .PHONY: sharedmods -sharedmods: $(SHAREDMODS) pybuilddir.txt - @target=`cat pybuilddir.txt`; \ - $(MKDIR_P) $$target; \ - for mod in X $(SHAREDMODS); do \ - if test $$mod != X; then \ - $(LN) -sf ../../$$mod $$target/`basename $$mod`; \ - fi; \ - done +sharedmods: $(SHAREDMODS) # dependency on BUILDPYTHON ensures that the target is run last .PHONY: checksharedmods @@ -2656,9 +2662,9 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ - $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) $(PYBUILDDIR)/build-details.json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against @@ -3063,7 +3069,7 @@ clean-retain-profile: pycremoval find build -name 'fficonfig.h' -exec rm -f {} ';' || true find build -name '*.py' -exec rm -f {} ';' || true find build -name '*.py[co]' -exec rm -f {} ';' || true - -rm -f pybuilddir.txt + -rm -rf pybuilddir.txt $(PYBUILDDIR) -rm -f _bootstrap_python -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module diff --git a/Modules/makesetup b/Modules/makesetup index 8bb971b152a522..68a2724a0fb33b 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -266,7 +266,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | esac for mod in $mods do - file="$srcdir/$mod\$(EXT_SUFFIX)" + file="\$(PYBUILDDIR)/$mod\$(EXT_SUFFIX)" case $doconfig in no) SHAREDMODS="$SHAREDMODS $file" @@ -274,7 +274,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" - rule="$rule; \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" + rule="$rule; @mkdir -p \$(PYBUILDDIR); \$(BLDSHARED) $objs $libs \$(LIBPYTHON) -o $file" echo "$rule" >>$rulesf done done diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 4a53e0bd1bee1b..e2a6ff2f23428b 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -195,11 +195,10 @@ def configure_emscripten_python(context, working_dir): assert ( len(lib_dirs) == 1 ), f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + _, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2) sysconfig_data = ( - f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + f"{emscripten_build_dir}/build/" + f"lib.emscripten-wasm32-emscripten-{python_version}-{abiflags}" ) if pydebug: sysconfig_data += "-pydebug" @@ -227,8 +226,13 @@ def configure_emscripten_python(context, working_dir): "--enable-wasm-dynamic-linking", f"--prefix={PREFIX_DIR}", ] - if pydebug: - configure.append("--with-pydebug") + for flag in abiflags: + if flag == 'd': + configure.append("--with-pydebug") + elif flag == 't': + configure.append("--disable-gil") + else: + raise ValueError(f"Unknown ABI flag: {flag}") if context.args: configure.extend(context.args) call( diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index da847c4ff86215..acb72e0ec7c1e3 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -212,12 +212,8 @@ def configure_wasi_python(context, working_dir): python_build_dir = BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - if pydebug: - sysconfig_data += "-pydebug" + _, python_version, abiflags = os.fsdecode(lib_dirs[0]).rsplit('-', maxsplit=2) + sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-wasi-{python_version}-{abiflags}" # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. @@ -244,8 +240,13 @@ def configure_wasi_python(context, working_dir): f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] - if pydebug: - configure.append("--with-pydebug") + for flag in abiflags: + if flag == 'd': + configure.append("--with-pydebug") + elif flag == 't': + configure.append("--disable-gil") + else: + raise ValueError(f"Unknown ABI flag: {flag}") if context.args: configure.extend(context.args) call(configure, diff --git a/configure b/configure index a058553480ca5a..0ced7b2e970d74 100755 --- a/configure +++ b/configure @@ -1014,6 +1014,8 @@ FREEZE_MODULE FREEZE_MODULE_BOOTSTRAP_DEPS FREEZE_MODULE_BOOTSTRAP PYTHON_FOR_FREEZE +SYSCONFIGVARS_JSON_NAME +SYSCONFIGDATA_NAME PYTHON_FOR_BUILD host_os host_vendor @@ -3748,7 +3750,7 @@ fi fi ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_SYSCONFIGDATA_PATH=$(abs_builddir)/$(PYBUILDDIR) '$with_build_python { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5 printf "%s\n" "$with_build_python" >&6; } @@ -3768,6 +3770,11 @@ fi +SYSCONFIGDATA_NAME='_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)' +SYSCONFIGVARS_JSON_NAME='_sysconfigvars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json' + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for Python interpreter freezing" >&5 printf %s "checking for Python interpreter freezing... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_FREEZE" >&5 diff --git a/configure.ac b/configure.ac index 23bd81ed4431b9..a87979977c4483 100644 --- a/configure.ac +++ b/configure.ac @@ -164,7 +164,7 @@ AC_ARG_WITH([build-python], dnl Build Python interpreter is used for regeneration and freezing. ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python PYTHON_FOR_FREEZE="$with_build_python" - PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) _PYTHON_SYSCONFIGDATA_PATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`) '$with_build_python + PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_SYSCONFIGDATA_PATH=$(abs_builddir)/$(PYBUILDDIR) '$with_build_python AC_MSG_RESULT([$with_build_python]) ], [ AS_VAR_IF([cross_compiling], [yes], @@ -176,6 +176,11 @@ AC_ARG_WITH([build-python], ) AC_SUBST([PYTHON_FOR_BUILD]) +SYSCONFIGDATA_NAME='_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH)' +SYSCONFIGVARS_JSON_NAME='_sysconfigvars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json' +AC_SUBST([SYSCONFIGDATA_NAME]) +AC_SUBST([SYSCONFIGVARS_JSON_NAME]) + AC_MSG_CHECKING([for Python interpreter freezing]) AC_MSG_RESULT([$PYTHON_FOR_FREEZE]) AC_SUBST([PYTHON_FOR_FREEZE])