From 13c3be595e907b816b21b735c32a20bebb3ebd03 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 9 Aug 2022 18:27:27 +0200 Subject: [PATCH 01/17] Add script to automate WASM build Automate WASM build with a new Python script. The script provides several build profiles with configure flags for Emscripten flavors and WASI. The script can detect and use Emscripten SDK and WASI SDK from default locations or env vars. ``configure`` now detects Node arguments and creates HOSTRUNNER arguments for Node 16. It also sets correct arguments for ``wasm64-emscripten``. --- Python/sysmodule.c | 10 +- Tools/wasm/README.md | 33 ++- Tools/wasm/wasm_build.py | 478 +++++++++++++++++++++++++++++++++++++++ configure | 162 ++++++++++++- configure.ac | 42 +++- 5 files changed, 709 insertions(+), 16 deletions(-) create mode 100755 Tools/wasm/wasm_build.py diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e861d9cbce415c..b8009b2db45f7b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2789,14 +2789,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), { if (typeof navigator == 'object') { info = navigator.userAgent; } else if (typeof process == 'object') { - info = "Node.js ".concat(process.version) + info = "Node.js ".concat(process.version); } else { - info = "UNKNOWN" + info = "UNKNOWN"; } var len = lengthBytesUTF8(info) + 1; var res = _malloc(len); - stringToUTF8(info, res, len); + if (res) stringToUTF8(info, res, len); +#if __wasm64__ + return BigInt(res); +#else return res; +#endif }); static PyObject * diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 6496a29e6ff809..1b297818ae9ab8 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -35,7 +35,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay. ### Compile a build Python interpreter -From within the container, run the following commands: +From within the container, run the following command: + +```shell +./Tools/wasm/wasm_build.py build +``` + +The command is roughly equivalent to: ```shell mkdir -p builddir/build @@ -45,13 +51,13 @@ make -j$(nproc) popd ``` -### Fetch and build additional emscripten ports +### Cross compile to wasm32-emscripten for browser ```shell -embuilder build zlib bzip2 +./Tools/wasm/wasm_build.py emscripten-browser ``` -### Cross compile to wasm32-emscripten for browser +The command is roughly equivalent to: ```shell mkdir -p builddir/emscripten-browser @@ -85,14 +91,21 @@ and header files with debug builds. ### Cross compile to wasm32-emscripten for node ```shell -mkdir -p builddir/emscripten-node -pushd builddir/emscripten-node +./Tools/wasm/wasm_build.py emscripten-browser-dl +``` + +The command is roughly equivalent to: + +```shell +mkdir -p builddir/emscripten-node-dl +pushd builddir/emscripten-node-dl CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ emconfigure ../../configure -C \ --host=wasm32-unknown-emscripten \ --build=$(../../config.guess) \ --with-emscripten-target=node \ + --enable-wasm-dynamic-linking \ --with-build-python=$(pwd)/../build/python emmake make -j$(nproc) @@ -100,7 +113,7 @@ popd ``` ```shell -node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js +node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js ``` (``--experimental-wasm-bigint`` is not needed with recent NodeJS versions) @@ -234,6 +247,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in ``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. +```shell +./Tools/wasm/wasm_build.py wasi +``` + +The command is roughly equivalent to: + ```shell mkdir -p builddir/wasi pushd builddir/wasi diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py new file mode 100755 index 00000000000000..4c347a3f84cc53 --- /dev/null +++ b/Tools/wasm/wasm_build.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +"""Build script for Python on WebAssembly platforms + + $ ./Tools/wasm/wasm_builder.py emscripten-browser compile + $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test + $ ./Tools/wasm/wasm_builder.py wasi test + +Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking), +"emscripten-browser", and "wasi". + +Emscripten builds require Emscripten SDK. The tools looks for 'EMSCRIPTEN' +env var and falls back to EMSDK installed at /opt/emsdk. + +WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH' +and falls back to /opt/wasi-sdk. +""" +import argparse +import enum +import dataclasses +import os +import pathlib +import shlex +import shutil +import subprocess +import sysconfig +from typing import Callable, List, Union # for Python 3.8 + +SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +WASMTOOLS = SRCDIR / "Tools" / "wasm" +BUILDDIR = SRCDIR / "builddir" +CONFIGURE = SRCDIR / "configure" +SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local" + +HAS_CCACHE = shutil.which("ccache") is not None + +# path to WASI-SDK root +WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk")) +# path to Emscripten directory (upstream/emscripten subdirectory in the EMSDK) +EMSCRIPTEN_PATH = pathlib.Path(os.environ.get("EMSCRIPTEN", "/opt/emsdk/upstream/emscripten")) + +# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py" + +INSTALL_EMSDK = """ +wasm32-emscripten builds need Emscripten SDK. Please follow instructions at +https://emscripten.org/docs/getting_started/downloads.html how to install +Emscripten tp "/opt/emsdk". Alternatively you can install the SDK in a +different location and set the environment variable +EMSCRIPTEN=.../upstream/emscripten (the directory with emcc and +emconfigure scripts), e.g. with ". emsdk_env.sh". +""" + +INSTALL_WASI_SDK = """ +wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from +https://github.com/WebAssembly/wasi-sdk/releases and install it to +"/opt/wasi-sdk". Alternatively you can install the SDK in a different location +and point the environment variable WASI_SDK_PATH to the root directory +of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW. +""" + +INSTALL_WASMTIME = """ +wasm32-wasi tests require wasmtime on PATH. Please follow instructions at +https://wasmtime.dev/ to install wasmtime. +""" + + +class MissingDependency(ValueError): + def __init__(self, command: str, text: str): + self.command = command + self.text = text + + def __str__(self): + return f"{type(self).__name__}: '{self.command}'\n{self.text}" + + +@dataclasses.dataclass +class Platform: + """Platform-specific settings + + - CONFIG_SITE override + - configure wrapper (e.g. emconfigure) + - make wrapper (e.g. emmake) + - additional environment variables + - check function to verify SDK + """ + + name: str + pythonexe: str + config_site: pathlib.Path + configure_wrapper: pathlib.Path + make_wrapper: pathlib.Path + environ: dict + check: Callable[[], None] + + def getenv(self) -> dict: + return self.environ.copy() + + +NATIVE = Platform( + "native", + pythonexe=sysconfig.get_config_var("PYTHON"), # TODO: check macOS + config_site=None, + configure_wrapper=None, + make_wrapper=None, + environ={}, + check=lambda: None, +) + + +def _check_emscripten(): + emconfigure = EMSCRIPTEN_PATH / "emconfigure" + if not emconfigure.exists(): + raise MissingDependency(emconfigure, INSTALL_EMSDK) + + +EMSCRIPTEN = Platform( + "emscripten", + pythonexe="python.js", + config_site=WASMTOOLS / "config.site-wasm32-emscripten", + configure_wrapper=EMSCRIPTEN_PATH / "emconfigure", + make_wrapper=EMSCRIPTEN_PATH / "emmake", + environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else None, + check=_check_emscripten, +) + + +def _check_wasi(): + wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld" + if not wasm_ld.exists(): + raise MissingDependency(wasm_ld, INSTALL_WASI_SDK) + wasmtime = shutil.which("wasmtime") + if wasmtime is None: + raise MissingDependency("wasmtime", INSTALL_WASMTIME) + + +WASI = Platform( + "wasi", + pythonexe="python.wasm", + config_site=WASMTOOLS / "config.site-wasm32-wasi", + configure_wrapper=WASMTOOLS / "wasi-env", + make_wrapper=None, + environ={"WASI_SDK_PATH": WASI_SDK_PATH}, + check=_check_wasi, +) + + +class Host(enum.Enum): + """Target host triplet""" + + wasm32_emscripten = "wasm32-unknown-emscripten" + wasm64_emscripten = "wasm64-unknown-emscripten" + wasm32_wasi = "wasm32-unknown-wasi" + wasm64_wasi = "wasm64-unknown-wasi" + # current platform + build = sysconfig.get_config_var("BUILD_GNU_TYPE") + + @property + def platform(self) -> Platform: + if self.is_emscripten: + return EMSCRIPTEN + elif self.is_wasi: + return WASI + else: + return NATIVE + + @property + def is_emscripten(self) -> bool: + cls = type(self) + return self in {cls.wasm32_emscripten, cls.wasm64_emscripten} + + @property + def is_wasi(self) -> bool: + cls = type(self) + return self in {cls.wasm32_wasi, cls.wasm64_wasi} + + +class EmscriptenTarget(enum.Enum): + """Emscripten-specific targets (--with-emscripten-target)""" + + browser = "browser" + browser_debug = "browser-debug" + node = "node" + node_debug = "node-debug" + + @property + def can_execute(self) -> bool: + cls = type(self) + return self not in {cls.browser, cls.browser_debug} + + +@dataclasses.dataclass +class BuildProfile: + name: str + host: Host + target: Union[EmscriptenTarget, None] = None + dynamic_linking: Union[bool, None] = None + pthreads: Union[bool, None] = None + testopts: str = "-j2" + + @property + def can_execute(self) -> bool: + """Can target run pythoninfo and tests? + + Disabled for browser, enabled for all other targets + """ + return self.target is None or self.target.can_execute + + @property + def builddir(self) -> pathlib.Path: + """Path to build directory""" + return BUILDDIR / self.name + + @property + def python_cmd(self) -> pathlib.Path: + """Path to python executable""" + return self.builddir / self.host.platform.pythonexe + + @property + def makefile(self) -> pathlib.Path: + """Path to Makefile""" + return self.builddir / "Makefile" + + @property + def configure_cmd(self) -> List[str]: + """Generate configure command""" + # use relative path, so WASI tests can find lib prefix. + # pathlib.Path.relative_to() does not work here. + configure = os.path.relpath(CONFIGURE, self.builddir) + cmd = [configure, "-C"] + platform = self.host.platform + if platform.configure_wrapper: + cmd.insert(0, os.fspath(platform.configure_wrapper)) + + cmd.append(f"--host={self.host.value}") + cmd.append(f"--build={Host.build.value}") + + if self.target is not None: + assert self.host.is_emscripten + cmd.append(f"--with-emscripten-target={self.target.value}") + + if self.dynamic_linking is not None: + assert self.host.is_emscripten + opt = "enable" if self.dynamic_linking else "disable" + cmd.append(f"--{opt}-wasm-dynamic-linking") + + if self.pthreads is not None: + assert self.host.is_emscripten + opt = "enable" if self.pthreads else "disable" + cmd.append(f"--{opt}-wasm-pthreads") + + if self.host != Host.build: + cmd.append(f"--with-build-python={BUILD.python_cmd}") + + if platform.config_site is not None: + cmd.append(f"CONFIG_SITE={platform.config_site}") + + return cmd + + @property + def make_cmd(self) -> List[str]: + """Generate make command""" + cmd = ["make"] + platform = self.host.platform + if platform.make_wrapper: + cmd.insert(0, os.fspath(platform.make_wrapper)) + return cmd + + def getenv(self) -> dict: + """Generate environ dict for platform""" + env = os.environ.copy() + env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}") + env.update(self.host.platform.getenv()) + return env + + def _run_cmd(self, cmd: List[str], args: List[str]): + cmd = list(cmd) + cmd.extend(args) + return subprocess.check_call( + cmd, + cwd=os.fspath(self.builddir), + env=self.getenv(), + ) + + def _check_execute(self): + if not self.can_execute: + raise ValueError(f"Cannot execute on {self.target}") + + def run_build(self, force_configure: bool = False): + """Run configure (if necessary) and make""" + if force_configure or not self.makefile.exists(): + self.run_configure() + self.run_make() + + def run_configure(self, *args): + """Run configure script to generate Makefile""" + os.makedirs(self.builddir, exist_ok=True) + return self._run_cmd(self.configure_cmd, args) + + def run_make(self, *args): + """Run make (defaults to build all)""" + return self._run_cmd(self.make_cmd, args) + + def run_pythoninfo(self): + """Run 'make pythoninfo'""" + self._check_execute() + return self.run_make("pythoninfo") + + def run_test(self): + """Run buildbottests""" + self._check_execute() + return self.run_make("buildbottest", f"TESTOPTS={self.testopts}") + + def run_py(self, *args): + """Run Python with hostrunner""" + self._check_execute() + self.run_make( + "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" + ) + + def clean(self, all: bool = False): + """Clean build directory""" + if all: + if self.builddir.exists(): + shutil.rmtree(self.builddir) + elif self.makefile.exists(): + self.run_make("clean") + + +# native build (build Python) +BUILD = BuildProfile( + "build", + host=Host.build, +) + +_profiles = [ + BUILD, + # wasm32-emscripten + BuildProfile( + "emscripten-browser", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.browser, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-browser-debug", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.browser_debug, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-dl", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-dl-debug", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + dynamic_linking=True, + ), + BuildProfile( + "emscripten-node-pthreads", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node, + pthreads=True, + ), + BuildProfile( + "emscripten-node-pthreads-debug", + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + pthreads=True, + ), + # wasm64-emscripten (currently not working) + BuildProfile( + "wasm64-emscripten-node-debug", + host=Host.wasm64_emscripten, + target=EmscriptenTarget.node_debug, + # MEMORY64 is not compatible with dynamic linking + dynamic_linking=False, + pthreads=False, + ), + # wasm32-wasi + BuildProfile( + "wasi", + host=Host.wasm32_wasi, + # skip sysconfig test_srcdir + testopts="-i '*.test_srcdir' -j2", + ), + # no SDK available yet + # BuildProfile( + # "wasm64-wasi", + # host=Host.wasm64_wasi, + # ), +] + +PROFILES = {p.name: p for p in _profiles} + +parser = argparse.ArgumentParser( + "wasm_build.py", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, +) +parser.add_argument( + "--clean", "-c", help="Clean build directories first", action="store_true" +) + +platforms = list(PROFILES) + ["cleanall"] +parser.add_argument( + "platform", + metavar="PLATFORM", + help=f"Build platform: {', '.join(platforms)}", + choices=platforms, +) + +ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"] +parser.add_argument( + "op", + metavar="OP", + help=f"operation: {', '.join(ops)}", + choices=ops, + default="compile", + nargs="?", +) + + +def main(): + args = parser.parse_args() + if args.platform == "cleanall": + for builder in PROFILES.values(): + builder.clean(all=True) + parser.exit(0) + + builder = PROFILES[args.platform] + try: + builder.host.platform.check() + except MissingDependency as e: + parser.exit(2, str(e)) + + # hack for WASI + if builder.host.is_wasi and not SETUP_LOCAL.exists(): + SETUP_LOCAL.touch() + + if args.op in {"compile", "pythoninfo", "repl", "test"}: + # all targets need a build Python + if builder is not BUILD: + if args.clean: + BUILD.clean(all=False) + BUILD.run_build() + elif not BUILD.python_cmd.exists(): + BUILD.run_build() + + if args.clean: + builder.clean(all=False) + + if args.op == "compile": + builder.run_build(force_configure=True) + else: + if not builder.makefile.exists(): + builder.run_configure() + if args.op == "pythoninfo": + builder.run_pythoninfo() + elif args.op == "repl": + builder.run_py() + elif args.op == "test": + builder.run_test() + elif args.op == "clean": + builder.clean(all=False) + elif args.op == "cleanall": + builder.clean(all=True) + else: + raise ValueError(args.op) + + print(builder.builddir) + parser.exit(0) + + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 3f25d43dde6f0c..eaeedffe3fc564 100755 --- a/configure +++ b/configure @@ -906,6 +906,7 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS +NODE HOSTRUNNER STATIC_LIBPYTHON GNULD @@ -4079,6 +4080,16 @@ if test -z "$CFLAGS"; then CFLAGS= fi +case $host in #( + wasm64-*-emscripten) : + + as_fn_append CFLAGS " -sMEMORY64=1" + as_fn_append LDFLAGS " -sMEMORY64=1" + ;; #( + *) : + ;; +esac + if test "$ac_sys_system" = "Darwin" then # Extract the first word of "xcrun", so it can be a program name with args. @@ -6220,7 +6231,7 @@ cat > conftest.c <&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_NODE+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $NODE in + [\\/]* | ?:[\\/]*) + ac_cv_path_NODE="$NODE" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_NODE="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +NODE=$ac_cv_path_NODE +if test -n "$NODE"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NODE" >&5 +$as_echo "$NODE" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_NODE"; then + ac_pt_NODE=$NODE + # Extract the first word of "node", so it can be a program name with args. +set dummy node; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_NODE+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_NODE in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_NODE="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_NODE=$ac_cv_path_ac_pt_NODE +if test -n "$ac_pt_NODE"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_NODE" >&5 +$as_echo "$ac_pt_NODE" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_NODE" = x; then + NODE="node" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + NODE=$ac_pt_NODE + fi +else + NODE="$ac_cv_path_NODE" +fi + + HOSTRUNNER="$NODE" # bigint for ctypes c_longlong, c_longdouble - HOSTRUNNER="node --experimental-wasm-bigint" + # no longer available in Node 16 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5 +$as_echo_n "checking for node --experimental-wasm-bigint... " >&6; } +if ${ac_cv_tool_node_wasm_bigint+:} false; then : + $as_echo_n "(cached) " >&6 +else + + if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bigint=yes + else + ac_cv_tool_node_wasm_bigint=no + fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5 +$as_echo "$ac_cv_tool_node_wasm_bigint" >&6; } + if test "x$ac_cv_tool_node_wasm_bigint" = xyes; then : + + as_fn_append HOSTRUNNER " --experimental-wasm-threads" + +fi + if test "x$enable_wasm_pthreads" = xyes; then : - HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory" + as_fn_append HOSTRUNNER " --experimental-wasm-threads" + # no longer available in Node 16 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5 +$as_echo_n "checking for node --experimental-wasm-bulk-memory... " >&6; } +if ${ac_cv_tool_node_wasm_bulk_memory+:} false; then : + $as_echo_n "(cached) " >&6 +else + + if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bulk_memory=yes + else + ac_cv_tool_node_wasm_bulk_memory=no + fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5 +$as_echo "$ac_cv_tool_node_wasm_bulk_memory" >&6; } + if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes; then : + + as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory" + +fi + +fi + if test "x$host_cpu" = xwasm64; then : + as_fn_append HOSTRUNNER " --experimental-wasm-memory64" fi ;; #( WASI/*) : diff --git a/configure.ac b/configure.ac index 8decd9ebae84cf..90f9e9cb6843f7 100644 --- a/configure.ac +++ b/configure.ac @@ -753,6 +753,16 @@ if test -z "$CFLAGS"; then CFLAGS= fi +dnl Emscripten SDK and WASI SDK default to wasm32. +dnl On Emscripten use MEMORY64 setting to build target wasm64-emscripten. +dnl for wasm64. +AS_CASE([$host], + [wasm64-*-emscripten], [ + AS_VAR_APPEND([CFLAGS], [" -sMEMORY64=1"]) + AS_VAR_APPEND([LDFLAGS], [" -sMEMORY64=1"]) + ], +) + if test "$ac_sys_system" = "Darwin" then dnl look for SDKROOT @@ -1048,7 +1058,7 @@ cat > conftest.c < /dev/null 2>&1; then + ac_cv_tool_node_wasm_bigint=yes + else + ac_cv_tool_node_wasm_bigint=no + fi + ]) + AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [ + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) + ]) + AS_VAR_IF([enable_wasm_pthreads], [yes], [ - HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory" + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) + # no longer available in Node 16 + AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [ + if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then + ac_cv_tool_node_wasm_bulk_memory=yes + else + ac_cv_tool_node_wasm_bulk_memory=no + fi + ]) + AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [ + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"]) + ]) ]) + + AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])]) ], dnl TODO: support other WASI runtimes dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the From c0faba0ffdec0a4beec3452277032e2c86994fac Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 9 Aug 2022 20:18:21 +0200 Subject: [PATCH 02/17] Use correct arg for bigint --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index eaeedffe3fc564..c89074a02dfa5c 100755 --- a/configure +++ b/configure @@ -6976,7 +6976,7 @@ fi $as_echo "$ac_cv_tool_node_wasm_bigint" >&6; } if test "x$ac_cv_tool_node_wasm_bigint" = xyes; then : - as_fn_append HOSTRUNNER " --experimental-wasm-threads" + as_fn_append HOSTRUNNER " --experimental-wasm-bigint" fi diff --git a/configure.ac b/configure.ac index 90f9e9cb6843f7..1b2af128e8ae61 100644 --- a/configure.ac +++ b/configure.ac @@ -1542,7 +1542,7 @@ then fi ]) AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [ - AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"]) + AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"]) ]) AS_VAR_IF([enable_wasm_pthreads], [yes], [ From 0c06ac260413266c78b936d86ce5f8d858f0d4fd Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 10 Aug 2022 07:53:16 +0200 Subject: [PATCH 03/17] Parse Emscripten config and check version --- Tools/wasm/wasm_build.py | 63 ++++++++++++++++++++++++++++++++-------- configure | 4 +-- configure.ac | 2 +- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 4c347a3f84cc53..7767582aaa9462 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -8,8 +8,8 @@ Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking), "emscripten-browser", and "wasi". -Emscripten builds require Emscripten SDK. The tools looks for 'EMSCRIPTEN' -env var and falls back to EMSDK installed at /opt/emsdk. +Emscripten builds require a recent Emscripten SDK. The tools looks for an +activated EMSDK environment (". /path/to/emsdk_env.sh"). WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH' and falls back to /opt/wasi-sdk. @@ -35,18 +35,20 @@ # path to WASI-SDK root WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk")) -# path to Emscripten directory (upstream/emscripten subdirectory in the EMSDK) -EMSCRIPTEN_PATH = pathlib.Path(os.environ.get("EMSCRIPTEN", "/opt/emsdk/upstream/emscripten")) + +# path to Emscripten SDK config file. +# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". +EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) +# 3.1.16 has broken utime() +EMSDK_MIN_VERSION = (3, 1, 17) +_MISSING = pathlib.PurePath("MISSING") # WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py" INSTALL_EMSDK = """ wasm32-emscripten builds need Emscripten SDK. Please follow instructions at https://emscripten.org/docs/getting_started/downloads.html how to install -Emscripten tp "/opt/emsdk". Alternatively you can install the SDK in a -different location and set the environment variable -EMSCRIPTEN=.../upstream/emscripten (the directory with emcc and -emconfigure scripts), e.g. with ". emsdk_env.sh". +Emscripten and how to activate the SDK with ". /path/to/emsdk_env.sh". """ INSTALL_WASI_SDK = """ @@ -63,6 +65,26 @@ """ +def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.Path: + """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT + + The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" + environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" + subdirectory with tools like "emconfigure". + """ + if not emconfig.exists(): + return _MISSING + with open(emconfig, encoding="utf-8") as f: + code = f.read() + # EM_CONFIG file is a Python snippet + local = {} + exec(code, globals(), local) + return pathlib.Path(local["EMSCRIPTEN_ROOT"]) + + +EMSCRIPTEN_ROOT = get_emscripten_root() + + class MissingDependency(ValueError): def __init__(self, command: str, text: str): self.command = command @@ -107,17 +129,34 @@ def getenv(self) -> dict: def _check_emscripten(): - emconfigure = EMSCRIPTEN_PATH / "emconfigure" + if EMSCRIPTEN_ROOT is _MISSING: + raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK) + # sanity check + emconfigure = EMSCRIPTEN.configure_wrapper if not emconfigure.exists(): raise MissingDependency(emconfigure, INSTALL_EMSDK) + # version check + version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt" + if not version_txt.exists(): + raise MissingDependency(version_txt, INSTALL_EMSDK) + with open(version_txt) as f: + version = f.read().strip().strip('"') + version_tuple = tuple(int(v) for v in version.split(".")) + if version_tuple < EMSDK_MIN_VERSION: + raise MissingDependency( + version_txt, + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " + "minimum required version " + f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", + ) EMSCRIPTEN = Platform( "emscripten", pythonexe="python.js", config_site=WASMTOOLS / "config.site-wasm32-emscripten", - configure_wrapper=EMSCRIPTEN_PATH / "emconfigure", - make_wrapper=EMSCRIPTEN_PATH / "emmake", + configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", + make_wrapper=EMSCRIPTEN_ROOT / "emmake", environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else None, check=_check_emscripten, ) @@ -434,7 +473,7 @@ def main(): try: builder.host.platform.check() except MissingDependency as e: - parser.exit(2, str(e)) + parser.exit(2, str(e) + "\n") # hack for WASI if builder.host.is_wasi and not SETUP_LOCAL.exists(): diff --git a/configure b/configure index c89074a02dfa5c..82b55a3745d575 100755 --- a/configure +++ b/configure @@ -6851,8 +6851,6 @@ if test "$cross_compiling" = yes; then fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 -$as_echo_n "checking HOSTRUNNER... " >&6; } if test -z "$HOSTRUNNER" then case $ac_sys_system/$ac_sys_emscripten_target in #( @@ -7019,6 +7017,8 @@ fi esac fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 +$as_echo_n "checking HOSTRUNNER... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5 $as_echo "$HOSTRUNNER" >&6; } diff --git a/configure.ac b/configure.ac index 1b2af128e8ae61..85d9e8011835ed 100644 --- a/configure.ac +++ b/configure.ac @@ -1525,7 +1525,6 @@ if test "$cross_compiling" = yes; then fi AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) -AC_MSG_CHECKING([HOSTRUNNER]) if test -z "$HOSTRUNNER" then AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], @@ -1570,6 +1569,7 @@ then ) fi AC_SUBST([HOSTRUNNER]) +AC_MSG_CHECKING([HOSTRUNNER]) AC_MSG_RESULT([$HOSTRUNNER]) if test -n "$HOSTRUNNER"; then From 11dd4884b9169f56019137a4babe72761b241d1b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 10 Aug 2022 16:58:36 +0200 Subject: [PATCH 04/17] Add note that system packages are not supported --- Tools/wasm/wasm_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 7767582aaa9462..4d72f0c18f7cc6 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -9,7 +9,8 @@ "emscripten-browser", and "wasi". Emscripten builds require a recent Emscripten SDK. The tools looks for an -activated EMSDK environment (". /path/to/emsdk_env.sh"). +activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages +(Debian, Homebrew) are not supported. WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH' and falls back to /opt/wasi-sdk. From 95b08058e54915fdd9622691cb7e157adef28a32 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 10 Aug 2022 17:08:50 +0200 Subject: [PATCH 05/17] add blurb --- .../Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst new file mode 100644 index 00000000000000..7351c28501c803 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst @@ -0,0 +1,2 @@ +The new tool `Tools/wasm/wasm_builder.py` automates configure, compile, and +test steps for building CPython on WebAssembly platforms. From 96692dad759358581e952ca4a0c0dfb68b6798c9 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 10 Aug 2022 17:15:12 +0200 Subject: [PATCH 06/17] Double quotes --- .../Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst index 7351c28501c803..c38db3af425e51 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst @@ -1,2 +1,2 @@ -The new tool `Tools/wasm/wasm_builder.py` automates configure, compile, and +The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and test steps for building CPython on WebAssembly platforms. From 4e3d2c261cd6cf83c5dee52c2e1bb6622249a461 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 11 Aug 2022 09:57:14 +0200 Subject: [PATCH 07/17] WASM64 fixes --- Lib/test/test_warnings/__init__.py | 9 ++++++++- Tools/wasm/README.md | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index b00ddd5df2f256..bcd50cca869ce9 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -487,7 +487,14 @@ def test_warn_explicit_non_ascii_filename(self): module=self.module) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"): + filenames = ["nonascii\xe9\u20ac"] + if not support.is_emscripten: + # JavaScript does not like surrogates + # Invalid UTF-8 leading byte 0x80 encountered when + # deserializing a UTF-8 string in wasm memory to a JS + # string! + filenames.append("surrogate\udc80") + for filename in filenames: try: os.fsencode(filename) except UnicodeEncodeError: diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 1b297818ae9ab8..74a25d54ffe31f 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -212,6 +212,15 @@ Node builds use ``NODERAWFS``. - Node RawFS allows direct access to the host file system without need to perform ``FS.mount()`` call. +## wasm64-emscripten + +- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``. +- ``EM_JS`` functions must return ``BigInt()``. +- ``Py_BuildValue()`` format strings must match size of types. Confusing 32 + and 64 bits types leads to memory corruption, see + [gh-95876](https://github.com/python/cpython/issues/95876) and + [gh-95878](https://github.com/python/cpython/issues/95878). + # Hosting Python WASM builds The simple REPL terminal uses SharedArrayBuffer. For security reasons From 9395e29c0cc3204fde96337787188102f1da0b8d Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 11 Aug 2022 14:10:14 +0200 Subject: [PATCH 08/17] Handle missing ccache, improve instructions --- Tools/wasm/wasm_build.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 4d72f0c18f7cc6..e617a18f421cc4 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -49,7 +49,13 @@ INSTALL_EMSDK = """ wasm32-emscripten builds need Emscripten SDK. Please follow instructions at https://emscripten.org/docs/getting_started/downloads.html how to install -Emscripten and how to activate the SDK with ". /path/to/emsdk_env.sh". +Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh". + + git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk + cd /path/to/emsdk + ./emsdk install latest + ./emsdk activate latest + source /path/to/emsdk_env.sh """ INSTALL_WASI_SDK = """ @@ -158,7 +164,7 @@ def _check_emscripten(): config_site=WASMTOOLS / "config.site-wasm32-emscripten", configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", make_wrapper=EMSCRIPTEN_ROOT / "emmake", - environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else None, + environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else {}, check=_check_emscripten, ) @@ -474,7 +480,7 @@ def main(): try: builder.host.platform.check() except MissingDependency as e: - parser.exit(2, str(e) + "\n") + parser.error(2, str(e)) # hack for WASI if builder.host.is_wasi and not SETUP_LOCAL.exists(): From ca9be4effd18852d55efc670949202fe998f7525 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 11 Aug 2022 14:12:03 +0200 Subject: [PATCH 09/17] error takes one arg --- Tools/wasm/wasm_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index e617a18f421cc4..0aabfc0837b6a1 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -480,7 +480,7 @@ def main(): try: builder.host.platform.check() except MissingDependency as e: - parser.error(2, str(e)) + parser.error(str(e)) # hack for WASI if builder.host.is_wasi and not SETUP_LOCAL.exists(): From d038dfd9a7d5ae31033340e97e81656163955f04 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 11 Aug 2022 17:11:28 +0200 Subject: [PATCH 10/17] Check for clean src dir --- Tools/wasm/wasm_build.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 0aabfc0837b6a1..060b78d489f3fa 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -46,6 +46,11 @@ # WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py" +CLEAN_SRCDIR = f""" +Builds require a clean source directory. Please use a clean checkout or +run "make clean -C '{SRCDIR}'". +""" + INSTALL_EMSDK = """ wasm32-emscripten builds need Emscripten SDK. Please follow instructions at https://emscripten.org/docs/getting_started/downloads.html how to install @@ -92,13 +97,21 @@ def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.Path: EMSCRIPTEN_ROOT = get_emscripten_root() -class MissingDependency(ValueError): - def __init__(self, command: str, text: str): - self.command = command +class ConditionError(ValueError): + def __init__(self, info: str, text: str): + self.info = info self.text = text def __str__(self): - return f"{type(self).__name__}: '{self.command}'\n{self.text}" + return f"{type(self).__name__}: '{self.info}'\n{self.text}" + + +class MissingDependency(ConditionError): + pass + + +class DirtySourceDirectory(ConditionError): + pass @dataclasses.dataclass @@ -124,6 +137,16 @@ def getenv(self) -> dict: return self.environ.copy() +def _check_clean_src(): + candidates = [ + SRCDIR / "Programs" / "python.o", + SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h", + ] + for candidate in candidates: + if candidate.exists(): + raise DirtySourceDirectory(candidate, CLEAN_SRCDIR) + + NATIVE = Platform( "native", pythonexe=sysconfig.get_config_var("PYTHON"), # TODO: check macOS @@ -131,7 +154,7 @@ def getenv(self) -> dict: configure_wrapper=None, make_wrapper=None, environ={}, - check=lambda: None, + check=_check_clean_src, ) @@ -156,6 +179,7 @@ def _check_emscripten(): "minimum required version " f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", ) + _check_clean_src() EMSCRIPTEN = Platform( @@ -176,6 +200,7 @@ def _check_wasi(): wasmtime = shutil.which("wasmtime") if wasmtime is None: raise MissingDependency("wasmtime", INSTALL_WASMTIME) + _check_clean_src() WASI = Platform( @@ -479,7 +504,7 @@ def main(): builder = PROFILES[args.platform] try: builder.host.platform.check() - except MissingDependency as e: + except ConditionError as e: parser.error(str(e)) # hack for WASI From 7558df525165533485a4744ee0df0ce6995edbb0 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 12 Aug 2022 08:31:34 +0200 Subject: [PATCH 11/17] Use python.exe on macOS --- Tools/wasm/wasm_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 060b78d489f3fa..c73e7c975af8e2 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -149,7 +149,7 @@ def _check_clean_src(): NATIVE = Platform( "native", - pythonexe=sysconfig.get_config_var("PYTHON"), # TODO: check macOS + pythonexe=sysconfig.get_config_var("BUILDPYTHON"), # macOS has python.exe config_site=None, configure_wrapper=None, make_wrapper=None, From 66f6fd8897eeb4fe03c3e3ee862aa52f0959fb6f Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 12 Aug 2022 11:59:49 +0200 Subject: [PATCH 12/17] Skip normalize test, fails on macOS host platform --- Lib/test/test_unicode_file_functions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 54916dec4eafa3..47619c8807bafe 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -6,6 +6,7 @@ import warnings from unicodedata import normalize from test.support import os_helper +from test import support filenames = [ @@ -123,6 +124,10 @@ def test_open(self): # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "test fails on Emscripten/WASI when host platform is macOS." + ) def test_normalize(self): files = set(self.files) others = set() From 52f41cd937b8914312460dc2bfc5702722747f57 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 13 Aug 2022 07:26:47 +0200 Subject: [PATCH 13/17] Apply suggestions from code review Co-authored-by: Brett Cannon --- Tools/wasm/wasm_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index c73e7c975af8e2..8f0c22a25e7843 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Build script for Python on WebAssembly platforms +"""Build script for Python on WebAssembly platforms. $ ./Tools/wasm/wasm_builder.py emscripten-browser compile $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test @@ -342,7 +342,7 @@ def getenv(self) -> dict: env.update(self.host.platform.getenv()) return env - def _run_cmd(self, cmd: List[str], args: List[str]): + def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]): cmd = list(cmd) cmd.extend(args) return subprocess.check_call( From 72223ceb341d3e55a2dedc8b95b4edf4978aa3f9 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 13 Aug 2022 07:37:17 +0200 Subject: [PATCH 14/17] Make mypy happy --- Tools/wasm/wasm_build.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 8f0c22a25e7843..a01845ed91e64a 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -24,7 +24,9 @@ import shutil import subprocess import sysconfig -from typing import Callable, List, Union # for Python 3.8 + +# for Python 3.8 +from typing import Any, Dict, Callable, Iterable, List, Optional, Union SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() WASMTOOLS = SRCDIR / "Tools" / "wasm" @@ -77,7 +79,7 @@ """ -def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.Path: +def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath: """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" @@ -89,7 +91,7 @@ def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.Path: with open(emconfig, encoding="utf-8") as f: code = f.read() # EM_CONFIG file is a Python snippet - local = {} + local: Dict[str, Any] = {} exec(code, globals(), local) return pathlib.Path(local["EMSCRIPTEN_ROOT"]) @@ -127,9 +129,9 @@ class Platform: name: str pythonexe: str - config_site: pathlib.Path - configure_wrapper: pathlib.Path - make_wrapper: pathlib.Path + config_site: Optional[pathlib.PurePath] + configure_wrapper: Optional[pathlib.PurePath] + make_wrapper: Optional[pathlib.PurePath] environ: dict check: Callable[[], None] @@ -144,12 +146,13 @@ def _check_clean_src(): ] for candidate in candidates: if candidate.exists(): - raise DirtySourceDirectory(candidate, CLEAN_SRCDIR) + raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR) NATIVE = Platform( "native", - pythonexe=sysconfig.get_config_var("BUILDPYTHON"), # macOS has python.exe + # macOS has python.exe + pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python", config_site=None, configure_wrapper=None, make_wrapper=None, @@ -164,17 +167,17 @@ def _check_emscripten(): # sanity check emconfigure = EMSCRIPTEN.configure_wrapper if not emconfigure.exists(): - raise MissingDependency(emconfigure, INSTALL_EMSDK) + raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK) # version check version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt" if not version_txt.exists(): - raise MissingDependency(version_txt, INSTALL_EMSDK) + raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK) with open(version_txt) as f: version = f.read().strip().strip('"') version_tuple = tuple(int(v) for v in version.split(".")) if version_tuple < EMSDK_MIN_VERSION: raise MissingDependency( - version_txt, + os.fspath(version_txt), f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " "minimum required version " f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", @@ -196,7 +199,7 @@ def _check_emscripten(): def _check_wasi(): wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld" if not wasm_ld.exists(): - raise MissingDependency(wasm_ld, INSTALL_WASI_SDK) + raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK) wasmtime = shutil.which("wasmtime") if wasmtime is None: raise MissingDependency("wasmtime", INSTALL_WASMTIME) From 77a957b9ca99e71328853f0b42d71e5ac6b0dba8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 13 Aug 2022 07:46:20 +0200 Subject: [PATCH 15/17] Apply suggestions from code review Co-authored-by: Brett Cannon --- Tools/wasm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 74a25d54ffe31f..c4c21b4f09dd2a 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -51,7 +51,7 @@ make -j$(nproc) popd ``` -### Cross compile to wasm32-emscripten for browser +### Cross-compile to wasm32-emscripten for browser ```shell ./Tools/wasm/wasm_build.py emscripten-browser From 29a74a5b3cb8c93455b725f5be3d488ae5f87ea8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 13 Aug 2022 10:58:39 +0200 Subject: [PATCH 16/17] Update Lib/test/test_warnings/__init__.py Co-authored-by: Brett Cannon --- Lib/test/test_warnings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index bcd50cca869ce9..9e473e923cad03 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -489,7 +489,7 @@ def test_warn_explicit_non_ascii_filename(self): self.module.filterwarnings("always", category=UserWarning) filenames = ["nonascii\xe9\u20ac"] if not support.is_emscripten: - # JavaScript does not like surrogates + # JavaScript does not like surrogates. # Invalid UTF-8 leading byte 0x80 encountered when # deserializing a UTF-8 string in wasm memory to a JS # string! From 5e0c3ee8899433a7227ff8d298d32c1454719720 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 13 Aug 2022 12:19:15 +0200 Subject: [PATCH 17/17] Add workaround for macOS shell --- Tools/wasm/wasi-env | 3 ++- Tools/wasm/wasm_build.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 6c2d56e0e5e32b..48908b02e60b96 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -72,4 +72,5 @@ export CFLAGS LDFLAGS export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR export PATH -exec "$@" +# no exec, it makes arvg[0] path absolute. +"$@" diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index a01845ed91e64a..e7a1f4a6007182 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -135,7 +135,7 @@ class Platform: environ: dict check: Callable[[], None] - def getenv(self) -> dict: + def getenv(self, profile: "BuildProfile") -> dict: return self.environ.copy() @@ -212,7 +212,15 @@ def _check_wasi(): config_site=WASMTOOLS / "config.site-wasm32-wasi", configure_wrapper=WASMTOOLS / "wasi-env", make_wrapper=None, - environ={"WASI_SDK_PATH": WASI_SDK_PATH}, + environ={ + "WASI_SDK_PATH": WASI_SDK_PATH, + # workaround for https://github.com/python/cpython/issues/95952 + "HOSTRUNNER": ( + "wasmtime run " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-$(VERSION):/Lib " + "--mapdir /::{srcdir} --" + ), + }, check=_check_wasi, ) @@ -342,7 +350,14 @@ def getenv(self) -> dict: """Generate environ dict for platform""" env = os.environ.copy() env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}") - env.update(self.host.platform.getenv()) + platenv = self.host.platform.getenv(self) + for key, value in platenv.items(): + if isinstance(value, str): + value = value.format( + relbuilddir=self.builddir.relative_to(SRCDIR), + srcdir=SRCDIR, + ) + env[key] = value return env def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]):