From efd5863ee09843074db0f6ea8ea2ef1017663adb Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Tue, 20 Feb 2024 21:17:19 +0100 Subject: [PATCH 01/17] First draft of supporting fat builds on macOS with the experimental JIT --- Tools/jit/_targets.py | 9 ++++-- Tools/jit/build.py | 33 ++++++++++++++++--- configure | 75 ++++++++++++++++++++++++------------------- configure.ac | 51 +++++++++++++++++------------ 4 files changed, 106 insertions(+), 62 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 51b091eb246413..37b3e596bd73a4 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,8 @@ PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" +ASYNCIO_RUNNER=asyncio.Runner() + _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -106,6 +108,7 @@ async def _compile( o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", + "-isysroot", "/Users/ronald/Applications/Xcode-beta.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", "-DPy_BUILD_CORE", "-D_DEBUG" if self.debug else "-DNDEBUG", f"-D_JIT_OPCODE={opname}", @@ -152,17 +155,17 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} - def build(self, out: pathlib.Path, *, comment: str = "") -> None: + def build(self, out: pathlib.Path, *, comment: str = "", stencils_h: str = "jit_stencils.h") -> None: """Build jit_stencils.h in the given directory.""" digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / "jit_stencils.h" + jit_stencils = out / stencils_h if ( not self.force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) + stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) with jit_stencils.open("w") as file: file.write(digest) if comment: diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d4ace14ebf26c..800d8e31b0f464 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -10,7 +10,7 @@ comment = f"$ {shlex.join([sys.executable] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + "target", nargs="+", type=_targets.get_target, help="a PEP 11 target triple to compile for" ) parser.add_argument( "-d", "--debug", action="store_true", help="compile for a debug build of Python" @@ -22,7 +22,30 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() - args.target.debug = args.debug - args.target.force = args.force - args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + + if len(args.target) == -1: + args.target.debug = args.debug + args.target.force = args.force + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment) + + else: + # Multiple triples specified, assume this is a macOS multiarchitecture build + # - Generate multiple stencil headers + # - Generate a helper header that include sthe stencils for the current + # architecture. + for target in args.target: + target.debug = args.debug + target.force = args.force + target.verbose = args.verbose + target.build(pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h") + + with open("jit_stencils.h", "w") as fp: + for idx, target in enumerate(args.target): + cpu, _, _ = target.triple.partition("-") + fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{cpu}__)\n") + fp.write(f'# include "jit_stencils-{target.triple}.h"\n') + + fp.write("#else\n") + fp.write('# error "unexpected cpu type"\n') + fp.write("#endif\n") diff --git a/configure b/configure index ba2d49df7c65fe..d47ae755643d12 100755 --- a/configure +++ b/configure @@ -893,6 +893,8 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL +JIT_STENCILS_H +REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB WASM_ASSETS_DIR @@ -920,8 +922,6 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE -JIT_STENCILS_H -REGEN_JIT_COMMAND ABIFLAGS LN MKDIR_P @@ -1079,11 +1079,11 @@ with_pydebug with_trace_refs enable_pystats with_assertions -enable_experimental_jit enable_optimizations with_lto enable_bolt with_strict_overflow +enable_experimental_jit with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1807,13 +1807,13 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit - build the experimental just-in-time compiler - (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -8036,33 +8036,6 @@ else printf "%s\n" "no" >&6; } fi -# Check for --enable-experimental-jit: -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 -printf %s "checking for --enable-experimental-jit... " >&6; } -# Check whether --enable-experimental-jit was given. -if test ${enable_experimental_jit+y} -then : - enableval=$enable_experimental_jit; -else $as_nop - enable_experimental_jit=no -fi - -if test "x$enable_experimental_jit" = xno -then : - -else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" - JIT_STENCILS_H="jit_stencils.h" - if test "x$Py_DEBUG" = xtrue -then : - as_fn_append REGEN_JIT_COMMAND " --debug" -fi -fi - - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } # Enable optimization flags @@ -9948,42 +9921,50 @@ printf "%s\n" "$CC" >&6; } UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 @@ -10098,6 +10079,34 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else $as_nop + enable_experimental_jit=no +fi + +if test "x$enable_experimental_jit" = xno +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " -D_Py_JIT" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 +printf "%s\n" "$enable_experimental_jit" >&6; } + case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST" diff --git a/configure.ac b/configure.ac index b39af7422c4c7c..97d8c830a126c6 100644 --- a/configure.ac +++ b/configure.ac @@ -1621,27 +1621,6 @@ else AC_MSG_RESULT([no]) fi -# Check for --enable-experimental-jit: -AC_MSG_CHECKING([for --enable-experimental-jit]) -AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], - [build the experimental just-in-time compiler (default is no)])], - [], - [enable_experimental_jit=no]) -AS_VAR_IF([enable_experimental_jit], - [no], - [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) - AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) - AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) - AS_VAR_IF([Py_DEBUG], - [true], - [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], - [])]) -AC_SUBST([REGEN_JIT_COMMAND]) -AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) @@ -2436,42 +2415,50 @@ yes) UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) @@ -2565,6 +2552,28 @@ yes) ;; esac +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +AS_VAR_IF([enable_experimental_jit], + [no], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPPLES:-$host}"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$enable_experimental_jit]) + case "$CC_BASENAME" in *mpicc*) CFLAGS_NODIST="$CFLAGS_NODIST" From a55268e5631345e1e524418c8bf820d37559d994 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 25 Sep 2024 15:13:24 -0700 Subject: [PATCH 02/17] fix up merge --- Tools/jit/_targets.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index c4187c60e430e2..1169df2bb0a7d6 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -26,15 +26,11 @@ PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" -ASYNCIO_RUNNER=asyncio.Runner() - - _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation ) - @dataclasses.dataclass class _Target(typing.Generic[_S, _R]): triple: str @@ -209,7 +205,7 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: return {task.get_name(): task.result() for task in tasks} def build( - self, out: pathlib.Path, *, comment: str = "", force: bool = False + self, out: pathlib.Path, *, comment: str = "", force: bool = False, stencils_h: str = "jit_stencils.h" ) -> None: """Build jit_stencils.h in the given directory.""" if not self.stable: @@ -218,7 +214,7 @@ def build( outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / "jit_stencils.h" + jit_stencils = out / stencils_h if ( not force and jit_stencils.exists() From bce7980a8ee512b106861d87808cfd03e0d81f59 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 25 Sep 2024 15:17:55 -0700 Subject: [PATCH 03/17] add fix for CI?; --- Tools/jit/_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 1169df2bb0a7d6..5570598fb2e591 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -34,6 +34,7 @@ @dataclasses.dataclass class _Target(typing.Generic[_S, _R]): triple: str + condition: str _: dataclasses.KW_ONLY alignment: int = 1 args: typing.Sequence[str] = () From b606b48422a8f503342b4b0c5126464476cd795b Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 25 Sep 2024 16:04:26 -0700 Subject: [PATCH 04/17] Fix up to make generic --- Tools/jit/_targets.py | 21 ++++++++++++++------- Tools/jit/build.py | 7 +++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 5570598fb2e591..4cd79e39e19f5e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -518,24 +518,31 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: # ghccc currently crashes Clang when combined with musttail on aarch64. :( target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - target = _MachO(host, alignment=8, prefix="_") + condition = "defined(__aarch64__)" and "defined(__APPLE__)" + target = _MachO(host, condition, alignment=8, prefix="_") elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - target = _COFF(host, alignment=8, args=args) + condition = "defined(_M_ARM64)" + target = _COFF(host, condition, alignment=8, args=args) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): args = ["-fpic"] - target = _ELF(host, alignment=8, args=args) + condition = "defined(__aarch64__)" and "defined(__linux__)" + target = _ELF(host, condition, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] - target = _COFF(host, args=args, ghccc=True, prefix="_") + condition = "defined(_M_IX86)" + target = _COFF(host, condition, args=args, ghccc=True, prefix="_") elif re.fullmatch(r"x86_64-apple-darwin.*", host): - target = _MachO(host, ghccc=True, prefix="_") + condition = "defined(__x86_64__)" and "defined(__APPLE__)" + target = _MachO(host, condition, ghccc=True, prefix="_") elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - target = _COFF(host, args=args, ghccc=True) + condition = "defined(_M_X64)" + target = _COFF(host, condition, args=args, ghccc=True) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fpic"] - target = _ELF(host, args=args, ghccc=True) + condition = "defined(__x86_64__)" and "defined(__linux__)" + target = _ELF(host, condition, args=args, ghccc=True) else: raise ValueError(host) return target diff --git a/Tools/jit/build.py b/Tools/jit/build.py index eea969e9544f38..d40012bb842779 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -31,9 +31,9 @@ args.target.build(pathlib.Path.cwd(), comment=comment) else: - # Multiple triples specified, assume this is a macOS multiarchitecture build + # Multiple triples specified, assume this is a macOS multi-architecture build # - Generate multiple stencil headers - # - Generate a helper header that include sthe stencils for the current + # - Generate a helper header that includes the stencils for the current # architecture. for target in args.target: target.debug = args.debug @@ -43,8 +43,7 @@ with open("jit_stencils.h", "w") as fp: for idx, target in enumerate(args.target): - cpu, _, _ = target.triple.partition("-") - fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{cpu}__)\n") + fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{target.condition}__)\n") fp.write(f'# include "jit_stencils-{target.triple}.h"\n') fp.write("#else\n") From a382a44c535f9dea730d53c04d0a520afafc32a4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 27 Sep 2024 10:36:15 -0700 Subject: [PATCH 05/17] Fix up conditions --- Tools/jit/_targets.py | 11 ++++++----- Tools/jit/build.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 4cd79e39e19f5e..28bf82ca5340a6 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,7 @@ CPYTHON = TOOLS.parent PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" +ASYNCIO_RUNNER=asyncio.Runner() _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -222,7 +223,7 @@ def build( and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) + stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) jit_stencils_new = out / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: @@ -518,7 +519,7 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: # ghccc currently crashes Clang when combined with musttail on aarch64. :( target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - condition = "defined(__aarch64__)" and "defined(__APPLE__)" + condition = "defined(__aarch64__) && defined(__APPLE__)" target = _MachO(host, condition, alignment=8, prefix="_") elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] @@ -526,14 +527,14 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: target = _COFF(host, condition, alignment=8, args=args) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): args = ["-fpic"] - condition = "defined(__aarch64__)" and "defined(__linux__)" + condition = "aarch64__) && defined(__linux__)" target = _ELF(host, condition, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] condition = "defined(_M_IX86)" target = _COFF(host, condition, args=args, ghccc=True, prefix="_") elif re.fullmatch(r"x86_64-apple-darwin.*", host): - condition = "defined(__x86_64__)" and "defined(__APPLE__)" + condition = "defined(__x86_64__) && defined(__APPLE__)" target = _MachO(host, condition, ghccc=True, prefix="_") elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] @@ -541,7 +542,7 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: target = _COFF(host, condition, args=args, ghccc=True) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fpic"] - condition = "defined(__x86_64__)" and "defined(__linux__)" + condition = "defined(__x86_64__) && defined(__linux__)" target = _ELF(host, condition, args=args, ghccc=True) else: raise ValueError(host) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index d40012bb842779..772802d5742457 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -43,7 +43,7 @@ with open("jit_stencils.h", "w") as fp: for idx, target in enumerate(args.target): - fp.write(f"#{'if' if idx == 0 else 'elif'} defined(__{target.condition}__)\n") + fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n") fp.write(f'# include "jit_stencils-{target.triple}.h"\n') fp.write("#else\n") From ff3d3635f26b77bd95c0316f8b36c75b9a7613a7 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 25 Mar 2025 20:14:18 -0700 Subject: [PATCH 06/17] Save state before merge --- Tools/jit/build.py | 24 +++++++++++++++++------- configure | 22 +++++++--------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 772802d5742457..0ec64e428ecc1b 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -11,7 +11,10 @@ comment = f"$ {shlex.join([sys.executable] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "target", nargs="+", type=_targets.get_target, help="a PEP 11 target triple to compile for" + "target", + nargs="+", + type=_targets.get_target, + help="a PEP 11 target triple to compile for", ) parser.add_argument( "-d", "--debug", action="store_true", help="compile for a debug build of Python" @@ -23,12 +26,15 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() + print(args.target) - if len(args.target) == -1: - args.target.debug = args.debug - args.target.force = args.force - args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + if len(args.target) == 1: + # Single triple specified, assume this is a normal build + target = args.target[0] + target.debug = args.debug + target.force = args.force + target.verbose = args.verbose + target.build(pathlib.Path.cwd(), comment=comment) else: # Multiple triples specified, assume this is a macOS multi-architecture build @@ -39,7 +45,11 @@ target.debug = args.debug target.force = args.force target.verbose = args.verbose - target.build(pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h") + target.build( + pathlib.Path.cwd(), + comment=comment, + stencils_h=f"jit_stencils-{target.triple}.h", + ) with open("jit_stencils.h", "w") as fp: for idx, target in enumerate(args.target): diff --git a/configure b/configure index a525ad83f846ef..bbef278a3f7175 100755 --- a/configure +++ b/configure @@ -902,8 +902,6 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL -JIT_STENCILS_H -REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB WASM_ASSETS_DIR @@ -1094,16 +1092,13 @@ with_pydebug with_trace_refs enable_pystats with_assertions +enable_experimental_jit enable_optimizations with_lto enable_bolt with_strict_overflow -<<<<<<< HEAD -enable_experimental_jit -======= enable_safety enable_slower_safety ->>>>>>> main with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1826,26 +1821,20 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) -<<<<<<< HEAD -======= --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler (default is no) ->>>>>>> main --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) -<<<<<<< HEAD - --enable-experimental-jit - build the experimental just-in-time compiler - (default is no) -======= --enable-safety enable usage of the security compiler options with no performance overhead --enable-slower-safety enable usage of the security compiler options with performance overhead ->>>>>>> main + --enable-experimental-jit + build the experimental just-in-time compiler + (default is no) --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -8304,6 +8293,8 @@ printf "%s\n" "no" >&6; } fi # Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } # Check whether --enable-experimental-jit was given. if test ${enable_experimental_jit+y} then : @@ -8345,6 +8336,7 @@ printf "%s\n" "$tier2_flags $jit_flags" >&6; } # Enable optimization flags + Py_OPT='false' { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-optimizations" >&5 printf %s "checking for --enable-optimizations... " >&6; } From 348f2b6d24186acb28aa8ab7bdc5098feac93a1e Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 25 Mar 2025 21:49:13 -0700 Subject: [PATCH 07/17] Fix event loop binding error for _CORES in FAT builds --- .gitignore | 2 +- Tools/jit/_llvm.py | 11 +++++++++-- Tools/jit/_targets.py | 10 +++------- Tools/jit/build.py | 18 ++++++------------ 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 8872e9d5508ff1..bf22c30a361090 100644 --- a/.gitignore +++ b/.gitignore @@ -137,7 +137,7 @@ Tools/unicode/data/ # hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ -/jit_stencils.h +/jit_stencils*.h /platform /profile-clean-stamp /profile-run-stamp diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 925b56ac669aa0..f8529e30daada1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -32,12 +32,19 @@ async def wrapper( return wrapper -_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) +_CORES: asyncio.BoundedSemaphore | None = None + + +def _get_cores() -> asyncio.BoundedSemaphore: + global _CORES + if _CORES is None: + _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + return _CORES async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: command = [tool, *args] - async with _CORES: + async with _get_cores(): if echo: print(shlex.join(command)) try: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 962687606c69fd..7645dbf21419e2 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,7 +25,6 @@ CPYTHON = TOOLS.parent PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" -ASYNCIO_RUNNER = asyncio.Runner() _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -45,7 +44,6 @@ class _Target(typing.Generic[_S, _R]): debug: bool = False verbose: bool = False known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) - basename: str = "" def _compute_digest(self, out: pathlib.Path) -> str: hasher = hashlib.sha256() @@ -201,7 +199,7 @@ def build( and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) + stencil_groups = asyncio.run(self._build_stencils()) jit_stencils_new = out / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: @@ -506,9 +504,7 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): condition = "defined(__aarch64__) && defined(__APPLE__)" - target = _MachO( - host, condition, alignment=8, prefix="_", basename="aarch64-apple-darwin" - ) + target = _MachO(host, condition, alignment=8, prefix="_") elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll", "-fplt"] condition = "defined(_M_ARM64)" @@ -532,7 +528,7 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: target = _COFF(host, condition, args=args, prefix="_") elif re.fullmatch(r"x86_64-apple-darwin.*", host): condition = "defined(__x86_64__) && defined(__APPLE__)" - target = _MachO(host, condition, prefix="_", basename="x86_64-apple-darwin") + target = _MachO(host, condition, prefix="_") elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] condition = "defined(_M_X64)" diff --git a/Tools/jit/build.py b/Tools/jit/build.py index cc619f11e551b8..68949b72aecc2a 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -26,21 +26,14 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() - print(args.target) if len(args.target) == 1: - # Single triple specified, assume this is a normal build - target = args.target[0] - target.debug = args.debug - target.force = args.force - target.verbose = args.verbose - target.build(pathlib.Path.cwd(), comment=comment) + args.target.debug = args.debug + args.target.verbose = args.verbose + args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) else: - # Multiple triples specified, assume this is a macOS multi-architecture build - # - Generate multiple stencil headers - # - Generate a helper header that includes the stencils for the current - # architecture. + # Build for multiple targets (e.g. universal2) for target in args.target: target.debug = args.debug target.force = args.force @@ -49,6 +42,7 @@ pathlib.Path.cwd(), comment=comment, stencils_h=f"jit_stencils-{target.triple}.h", + force=args.force, ) with open("jit_stencils.h", "w") as fp: @@ -57,5 +51,5 @@ fp.write(f'# include "jit_stencils-{target.triple}.h"\n') fp.write("#else\n") - fp.write('# error "unexpected cpu type"\n') + fp.write('# error "unexpected target"\n') fp.write("#endif\n") From 4b31a30702b3efaf010022dace40979e3ebfbeda Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 26 Mar 2025 04:55:29 +0000 Subject: [PATCH 08/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst new file mode 100644 index 00000000000000..19d92c33bc6d21 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst @@ -0,0 +1 @@ +Add support for macOS multi-arch builds with the JIT enabled From 4564bc4b4a30e8813cf625df26601cd5d87ec2d0 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 25 Mar 2025 22:27:17 -0700 Subject: [PATCH 09/17] Try using weakref? --- Tools/jit/_llvm.py | 13 ++++++++----- Tools/jit/build.py | 7 ++++--- configure | 18 +++++++++--------- configure.ac | 18 +++++++++--------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f8529e30daada1..d37c5f2d76faba 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -7,6 +7,7 @@ import shlex import subprocess import typing +import weakref _LLVM_VERSION = 19 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") @@ -32,14 +33,16 @@ async def wrapper( return wrapper -_CORES: asyncio.BoundedSemaphore | None = None +_CORES_BY_LOOP: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, asyncio.BoundedSemaphore +] = weakref.WeakKeyDictionary() def _get_cores() -> asyncio.BoundedSemaphore: - global _CORES - if _CORES is None: - _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) - return _CORES + loop = asyncio.get_running_loop() + if loop not in _CORES_BY_LOOP: + _CORES_BY_LOOP[loop] = asyncio.BoundedSemaphore(os.cpu_count()) + return _CORES_BY_LOOP[loop] async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 68949b72aecc2a..dbaf240a097e48 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -28,9 +28,10 @@ args = parser.parse_args() if len(args.target) == 1: - args.target.debug = args.debug - args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) + target = args.target[0] + target.debug = args.debug + target.verbose = args.verbose + target.build(pathlib.Path.cwd(), comment=comment, force=args.force) else: # Build for multiple targets (e.g. universal2) diff --git a/configure b/configure index 9b8a5dc224e688..3d7a73104a0478 100755 --- a/configure +++ b/configure @@ -10655,50 +10655,50 @@ fi UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" - ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` + ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" - ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" - ARCH_TRIPLES=i386-apple-darwin + ARCH_TRIPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=x86_64-apple-darwin + ARCH_TRIPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 @@ -10847,7 +10847,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-host}" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : diff --git a/configure.ac b/configure.ac index eda154b231c464..86e83f724000dc 100644 --- a/configure.ac +++ b/configure.ac @@ -2617,50 +2617,50 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" - ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` + ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" - ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" - ARCH_TRIPLES=i386-apple-darwin + ARCH_TRIPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" - ARCH_TRIPLES=x86_64-apple-darwin + ARCH_TRIPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" - ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` + ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) @@ -2776,7 +2776,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-host}"]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) AS_VAR_IF([Py_DEBUG], [true], From 49f86a112d95984a1e49e0c5a2372387e28437ae Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 26 Mar 2025 19:14:38 -0700 Subject: [PATCH 10/17] Remove weakref and lazily instantiate cores --- Tools/jit/_llvm.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index d37c5f2d76faba..f8529e30daada1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -7,7 +7,6 @@ import shlex import subprocess import typing -import weakref _LLVM_VERSION = 19 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") @@ -33,16 +32,14 @@ async def wrapper( return wrapper -_CORES_BY_LOOP: weakref.WeakKeyDictionary[ - asyncio.AbstractEventLoop, asyncio.BoundedSemaphore -] = weakref.WeakKeyDictionary() +_CORES: asyncio.BoundedSemaphore | None = None def _get_cores() -> asyncio.BoundedSemaphore: - loop = asyncio.get_running_loop() - if loop not in _CORES_BY_LOOP: - _CORES_BY_LOOP[loop] = asyncio.BoundedSemaphore(os.cpu_count()) - return _CORES_BY_LOOP[loop] + global _CORES + if _CORES is None: + _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) + return _CORES async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: From 467ffe17493fbe9e923f0a2068cbbe70ea2b2613 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 2 Apr 2025 19:25:09 -0700 Subject: [PATCH 11/17] use Runner --- Tools/jit/_llvm.py | 11 ++--------- Tools/jit/_targets.py | 3 ++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f8529e30daada1..925b56ac669aa0 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -32,19 +32,12 @@ async def wrapper( return wrapper -_CORES: asyncio.BoundedSemaphore | None = None - - -def _get_cores() -> asyncio.BoundedSemaphore: - global _CORES - if _CORES is None: - _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) - return _CORES +_CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: command = [tool, *args] - async with _get_cores(): + async with _CORES: if echo: print(shlex.join(command)) try: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 7645dbf21419e2..362cf3024de1e3 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,7 @@ CPYTHON = TOOLS.parent PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" +ASYNCIO_RUNNER = asyncio.Runner() _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -199,7 +200,7 @@ def build( and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) + stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) jit_stencils_new = out / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: From 58649ffdfa5797738857190ea802e671a3b3c81c Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 10 Apr 2025 20:32:54 -0700 Subject: [PATCH 12/17] Remove conditional and fix indentation in stencils --- Tools/jit/build.py | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index dbaf240a097e48..4d1e484b6838eb 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -26,31 +26,22 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() - - if len(args.target) == 1: - target = args.target[0] + for target in args.target: target.debug = args.debug + target.force = args.force target.verbose = args.verbose - target.build(pathlib.Path.cwd(), comment=comment, force=args.force) - - else: - # Build for multiple targets (e.g. universal2) - for target in args.target: - target.debug = args.debug - target.force = args.force - target.verbose = args.verbose - target.build( - pathlib.Path.cwd(), - comment=comment, - stencils_h=f"jit_stencils-{target.triple}.h", - force=args.force, - ) + target.build( + pathlib.Path.cwd(), + comment=comment, + stencils_h=f"jit_stencils-{target.triple}.h", + force=args.force, + ) - with open("jit_stencils.h", "w") as fp: - for idx, target in enumerate(args.target): - fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n") - fp.write(f'# include "jit_stencils-{target.triple}.h"\n') + with open("jit_stencils.h", "w") as fp: + for idx, target in enumerate(args.target): + fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n") + fp.write(f'#include "jit_stencils-{target.triple}.h"\n') - fp.write("#else\n") - fp.write('# error "unexpected target"\n') - fp.write("#endif\n") + fp.write("#else\n") + fp.write('#error "unexpected target"\n') + fp.write("#endif\n") From b172a4128d79867e22a72b850be6dca74cc90dbf Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 10 Apr 2025 20:44:43 -0700 Subject: [PATCH 13/17] CI updates --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index d9e2ffc3fd4da6..f582c1967002e5 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -120,7 +120,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --enable-universal-sdk --with-universal-archs=universal2 make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From de3f89677a4744ab1a5972fb8308c2d2a9e37a80 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 10 Apr 2025 20:46:30 -0700 Subject: [PATCH 14/17] Remove extra space --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 9573dda29e90a5..dcdc0e5d4a8cae 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -115,7 +115,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --enable-universal-sdk --with-universal-archs=universal2 + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --enable-universal-sdk --with-universal-archs=universal2 make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From 7c22761673f8aa98445a499ee7b96ae5b2833714 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 10 Apr 2025 21:01:57 -0700 Subject: [PATCH 15/17] Fix condition typo --- Tools/jit/_targets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index ee00edd3aaeaae..40abd0c771f346 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -532,7 +532,7 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: # was required to disable them. "-mno-outline-atomics", ] - condition = "aarch64__) && defined(__linux__)" + condition = "defined(__aarch64__) && defined(__linux__)" target = _ELF(host, condition, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): args = [ From c54250b65ea089f7425d98c17af050e768d8e68d Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 10 Apr 2025 21:53:16 -0700 Subject: [PATCH 16/17] undo --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index dcdc0e5d4a8cae..c8fdb7712ca069 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -115,7 +115,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --enable-universal-sdk --with-universal-archs=universal2 + ./configure --enable-experimental-jit --enable-universal-sdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 From c21573fb81076125e554c200e1cb37046c9e7153 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 11 Apr 2025 12:44:00 -0700 Subject: [PATCH 17/17] Update jit.yml Co-authored-by: Brandt Bucher --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c8fdb7712ca069..69985bd00b6862 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -115,7 +115,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit --enable-universal-sdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} + ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3