diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000000..d83aee08025502 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 742597f5cb5ebd..864229157035f3 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -125,7 +125,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Tools/jit/README.md b/Tools/jit/README.md index c70c0c47d94ad2..714941e317d48c 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `/bin`) It's easy to install all of the required tools: @@ -53,7 +58,7 @@ choco install llvm --version=21.1.0 ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 0b9cb5192f1b75..082e7ed42fa747 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -71,7 +71,18 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: + # Explicitly defined LLVM installation location + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): @@ -105,10 +116,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -117,10 +129,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index a76d8ff2792602..e11a654ed51649 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -51,6 +51,7 @@ class _Target(typing.Generic[_S, _R]): verbose: bool = False cflags: str = "" llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -83,7 +84,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -103,7 +108,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -177,14 +186,22 @@ async def _compile( *shlex.split(self.cflags), ] await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) self.optimizer( s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 127d93b317fb09..5e1b05a3d86cb4 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -43,6 +43,9 @@ "--cflags", help="additional flags to pass to the compiler", default="" ) parser.add_argument("--llvm-version", help="LLVM version to use") + parser.add_argument( + "--llvm-tools-install-dir", help="Installation location of LLVM tools" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug @@ -52,6 +55,8 @@ target.pyconfig_dir = args.pyconfig_dir if args.llvm_version: target.llvm_version = args.llvm_version + if args.llvm_tools_install_dir: + target.llvm_tools_install_dir = args.llvm_tools_install_dir target.build( comment=comment, force=args.force, diff --git a/configure b/configure index 620878bb181378..6dd77d755561d5 100755 --- a/configure +++ b/configure @@ -10908,7 +10908,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" diff --git a/configure.ac b/configure.ac index 8ef479fe32036c..c9eb699579ebb7 100644 --- a/configure.ac +++ b/configure.ac @@ -2804,7 +2804,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],