From 3111c2d05d429205f9b897d4b9528bb0914c1e15 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 3 Sep 2025 16:47:32 +0200 Subject: [PATCH 01/11] Support custom LLVM installation path Add support for explicitly defined LLVM installation location. This is necessary as MSVC now comes with its own LLVM installation and activating MSVC via vcvars.bat will put LLVM tools on the `PATH` before the local ones. --- Tools/jit/_llvm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 0b9cb5192f1b75..c3c81c97de6063 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -72,6 +72,11 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: + # Explicitly defined LLVM installation location + if (llvm_root := os.getenv("LLVM_ROOT")) is not None: + path = os.path.join(llvm_root, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): From 832179061273d0069fa3edf798aa490006cb410e Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:56:00 +0000 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000000..cdbbfa70e3ae64 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Search for LLVM tools in ``LLVM_ROOT`` during the build. From e6082d4e07167242a96a2af8e66bf51e162a11dd Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 5 Sep 2025 11:35:51 +0200 Subject: [PATCH 03/11] Use `LLVM_TOOLS_INSTALL_DIR` to search for LLVM tools --- .../next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst | 2 +- Tools/jit/_llvm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst index cdbbfa70e3ae64..46cc6ffc05ea2d 100644 --- a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -1 +1 @@ -Search for LLVM tools in ``LLVM_ROOT`` during the build. +Search for LLVM tools in ``LLVM_TOOLS_INSTALL_DIR`` during the build. diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index c3c81c97de6063..86b018092932ca 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -73,8 +73,8 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: # Explicitly defined LLVM installation location - if (llvm_root := os.getenv("LLVM_ROOT")) is not None: - path = os.path.join(llvm_root, "bin", tool) + if (llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR")) is not None: + path = os.path.join(llvm_tools_dir, tool) if await _check_tool_version(path, llvm_version, echo=echo): return path # Unversioned executables: From 8bd8a980833df6b91c8461cdc842ad0b165b7b2f Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sun, 7 Sep 2025 16:24:30 +0200 Subject: [PATCH 04/11] Don't join on empty values. Co-authored-by: Steve Dower --- Tools/jit/_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 86b018092932ca..b7a5845ced8256 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -73,7 +73,7 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: # Explicitly defined LLVM installation location - if (llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR")) is not None: + if llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR")): path = os.path.join(llvm_tools_dir, tool) if await _check_tool_version(path, llvm_version, echo=echo): return path From 297fe1c8d954d2167344d975c2d774bccc420e89 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sun, 7 Sep 2025 18:13:56 +0200 Subject: [PATCH 05/11] Fix syntax error --- Tools/jit/_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index b7a5845ced8256..82df2e2ecad04b 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -73,7 +73,7 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: # Explicitly defined LLVM installation location - if llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR")): + if llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR"): path = os.path.join(llvm_tools_dir, tool) if await _check_tool_version(path, llvm_version, echo=echo): return path From e9c2cc39ae40e04dfbf1d5c07b07f0c2c006b43c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sun, 26 Oct 2025 11:20:40 +0100 Subject: [PATCH 06/11] Document LLVM_TOOLS_INSTALL_DIR --- Tools/jit/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index c70c0c47d94ad2..f9ce82e8ecf227 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,7 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. If you have multiple matching LLVM installations, you can use `LLVM_TOOLS_INSTALL_DIR` to point to the preferred installation prefix. It's easy to install all of the required tools: @@ -53,7 +53,7 @@ choco install llvm --version=21.1.0 ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building From a57427ccbc1537dbc12a7358535754b10918a26d Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 27 Oct 2025 09:22:49 +0100 Subject: [PATCH 07/11] Add LLVM_TOOLS_INSTALL_DIR to configure --- Tools/jit/_llvm.py | 24 +++++++++++++++++++----- Tools/jit/_targets.py | 25 +++++++++++++++++++++---- Tools/jit/build.py | 5 +++++ configure | 2 +- configure.ac | 2 +- 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 82df2e2ecad04b..a0c8243f74527d 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -71,10 +71,16 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: # Explicitly defined LLVM installation location - if llvm_tools_dir := os.getenv("LLVM_TOOLS_INSTALL_DIR"): - path = os.path.join(llvm_tools_dir, tool) + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, tool) if await _check_tool_version(path, llvm_version, echo=echo): return path # Unversioned executables: @@ -110,10 +116,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -122,10 +129,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index a76d8ff2792602..e11a654ed51649 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -51,6 +51,7 @@ class _Target(typing.Generic[_S, _R]): verbose: bool = False cflags: str = "" llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -83,7 +84,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -103,7 +108,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -177,14 +186,22 @@ async def _compile( *shlex.split(self.cflags), ] await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) self.optimizer( s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 127d93b317fb09..5e1b05a3d86cb4 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -43,6 +43,9 @@ "--cflags", help="additional flags to pass to the compiler", default="" ) parser.add_argument("--llvm-version", help="LLVM version to use") + parser.add_argument( + "--llvm-tools-install-dir", help="Installation location of LLVM tools" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug @@ -52,6 +55,8 @@ target.pyconfig_dir = args.pyconfig_dir if args.llvm_version: target.llvm_version = args.llvm_version + if args.llvm_tools_install_dir: + target.llvm_tools_install_dir = args.llvm_tools_install_dir target.build( comment=comment, force=args.force, diff --git a/configure b/configure index 620878bb181378..6dd77d755561d5 100755 --- a/configure +++ b/configure @@ -10908,7 +10908,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" diff --git a/configure.ac b/configure.ac index 8ef479fe32036c..c9eb699579ebb7 100644 --- a/configure.ac +++ b/configure.ac @@ -2804,7 +2804,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], From b90f60bf40dc06b7bdc1380ed670d77218234906 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 2 Dec 2025 14:40:39 +0100 Subject: [PATCH 08/11] Fix wording in README --- Tools/jit/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tools/jit/README.md b/Tools/jit/README.md index f9ce82e8ecf227..714941e317d48c 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. If you have multiple matching LLVM installations, you can use `LLVM_TOOLS_INSTALL_DIR` to point to the preferred installation prefix. +LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `/bin`) It's easy to install all of the required tools: From 06c9e51f2fbd4574066ceb01cc4c6ccc36cacfa7 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 2 Dec 2025 14:41:25 +0100 Subject: [PATCH 09/11] Expect tools in /bin --- Tools/jit/_llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index a0c8243f74527d..082e7ed42fa747 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -80,7 +80,7 @@ async def _find_tool( ) -> str | None: # Explicitly defined LLVM installation location if llvm_tools_install_dir: - path = os.path.join(llvm_tools_install_dir, tool) + path = os.path.join(llvm_tools_install_dir, "bin", tool) if await _check_tool_version(path, llvm_version, echo=echo): return path # Unversioned executables: From c801103b4bbdd18d7335983606959ed1e2ffb9c5 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 2 Dec 2025 14:45:52 +0100 Subject: [PATCH 10/11] Add environment variables to PCbuild --- PCbuild/regen.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 742597f5cb5ebd..864229157035f3 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -125,7 +125,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + From 4801b374e2b1fe2e47677c2d97a6b5ada7ab3136 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Tue, 2 Dec 2025 16:42:47 +0100 Subject: [PATCH 11/11] Be more specific in NEWS entry --- .../next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst index 46cc6ffc05ea2d..d83aee08025502 100644 --- a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -1 +1 @@ -Search for LLVM tools in ``LLVM_TOOLS_INSTALL_DIR`` during the build. +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build.