Skip to content

gh-145219: Cache Emscripten libffi and mpdec builds, add install-emscripten cmd #145664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hoodmane wants to merge 1 commit into
base: main
Choose a base branch
from
+154 −34
Open

gh-145219: Cache Emscripten libffi and mpdec builds, add install-emscripten cmd #145664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 140 additions & 33 deletions Tools/wasm/emscripten/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import contextlib
import functools
import hashlib
import json
import os
import shutil
import subprocess
Expand All @@ -14,6 +15,8 @@
from textwrap import dedent
from urllib.request import urlopen

import tomllib

try:
from os import process_cpu_count as cpu_count
except ImportError:
Expand All @@ -22,48 +25,62 @@

EMSCRIPTEN_DIR = Path(__file__).parent
CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt"
CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml"

DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build"
HOST_TRIPLE = "wasm32-emscripten"


def get_build_paths(cross_build_dir=None):
@functools.cache
def load_config_toml():
with CONFIG_FILE.open("rb") as file:
return tomllib.load(file)


@functools.cache
def required_emscripten_version():
return load_config_toml()["emscripten-version"]


@functools.cache
def emsdk_cache_root(emsdk_cache):
required_version = required_emscripten_version()
return Path(emsdk_cache) / required_version


@functools.cache
def emsdk_activate_path(emsdk_cache):
return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh"


def get_build_paths(cross_build_dir=None, emsdk_cache=None):
"""Compute all build paths from the given cross-build directory."""
if cross_build_dir is None:
cross_build_dir = DEFAULT_CROSS_BUILD_DIR
cross_build_dir = Path(cross_build_dir).absolute()
host_triple_dir = cross_build_dir / HOST_TRIPLE
prefix_dir = host_triple_dir / "prefix"
if emsdk_cache:
prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix"

return {
"cross_build_dir": cross_build_dir,
"native_build_dir": cross_build_dir / "build",
"host_triple_dir": host_triple_dir,
"host_build_dir": host_triple_dir / "build",
"host_dir": host_triple_dir / "build" / "python",
"prefix_dir": host_triple_dir / "prefix",
"prefix_dir": prefix_dir,
}


LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n"


@functools.cache
def get_required_emscripten_version():
"""Read the required emscripten version from emscripten_version.txt."""
return EMSCRIPTEN_VERSION_FILE.read_text().strip()


@functools.cache
def get_emsdk_activate_path(emsdk_cache):
required_version = get_required_emscripten_version()
return Path(emsdk_cache) / required_version / "emsdk_env.sh"


def validate_emsdk_version(emsdk_cache):
"""Validate that the emsdk cache contains the required emscripten version."""
required_version = get_required_emscripten_version()
emsdk_env = get_emsdk_activate_path(emsdk_cache)
required_version = required_emscripten_version()
emsdk_env = emsdk_activate_path(emsdk_cache)
if not emsdk_env.is_file():
print(
f"Required emscripten version {required_version} not found in {emsdk_cache}",
Expand All @@ -90,7 +107,7 @@ def get_emsdk_environ(emsdk_cache):
[
"bash",
"-c",
f"EMSDK_QUIET=1 source {get_emsdk_activate_path(emsdk_cache)} && env",
f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env",
],
text=True,
)
Expand Down Expand Up @@ -207,6 +224,35 @@ def build_python_path(context):
return binary


def install_emscripten(context):
emsdk_cache = context.emsdk_cache
if emsdk_cache is None:
print("install-emscripten requires --emsdk-cache", file=sys.stderr)
sys.exit(1)
version = required_emscripten_version()
emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk"
if emsdk_target.exists():
if not context.quiet:
print(f"Emscripten version {version} already installed")
return
if not context.quiet:
print(f"Installing emscripten version {version}")
emsdk_target.mkdir(parents=True)
call(
[
"git",
"clone",
"https://github.com/emscripten-core/emsdk.git",
emsdk_target,
],
quiet=context.quiet,
)
call([emsdk_target / "emsdk", "install", version], quiet=context.quiet)
call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet)
if not context.quiet:
print(f"Installed emscripten version {version}")


@subdir("native_build_dir", clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
Expand Down Expand Up @@ -258,43 +304,95 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
shutil.unpack_archive(tmp_file.name, working_dir)


def should_build_library(prefix, name, config, quiet):
cached_config = prefix / (name + ".json")
if not cached_config.exists():
if not quiet:
print(
f"No cached build of {name} version {config['version']} found, building"
)
return True

try:
with cached_config.open("rb") as f:
cached_config = json.load(f)
except json.JSONDecodeError:
if not quiet:
print(f"Cached data for {name} invalid, rebuilding")
return True
if config == cached_config:
if not quiet:
print(
f"Found cached build of {name} version {config['version']}, not rebuilding"
)
return False

if not quiet:
print(
f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding"
)
return True


def write_library_config(prefix, name, config, quiet):
cached_config = prefix / (name + ".json")
with cached_config.open("w") as f:
json.dump(config, f)
if not quiet:
print(f"Succeded building {name}, wrote config to {cached_config}")


@subdir("host_build_dir", clean_ok=True)
def make_emscripten_libffi(context, working_dir):
ver = "3.4.6"
libffi_dir = working_dir / f"libffi-{ver}"
prefix = context.build_paths["prefix_dir"]
libffi_config = load_config_toml()["libffi"]
if not should_build_library(
prefix, "libffi", libffi_config, context.quiet
):
return
url = libffi_config["url"]
version = libffi_config["version"]
shasum = libffi_config["shasum"]
libffi_dir = working_dir / f"libffi-{version}"
shutil.rmtree(libffi_dir, ignore_errors=True)
download_and_unpack(
working_dir,
f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz",
"b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e",
url.format(version=version),
shasum,
)
call(
[EMSCRIPTEN_DIR / "make_libffi.sh"],
env=updated_env(
{"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache
),
env=updated_env({"PREFIX": prefix}, context.emsdk_cache),
cwd=libffi_dir,
quiet=context.quiet,
)
write_library_config(prefix, "libffi", libffi_config, context.quiet)


@subdir("host_build_dir", clean_ok=True)
def make_mpdec(context, working_dir):
ver = "4.0.1"
mpdec_dir = working_dir / f"mpdecimal-{ver}"
prefix = context.build_paths["prefix_dir"]
mpdec_config = load_config_toml()["mpdec"]
if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet):
return

url = mpdec_config["url"]
version = mpdec_config["version"]
shasum = mpdec_config["shasum"]
mpdec_dir = working_dir / f"mpdecimal-{version}"
shutil.rmtree(mpdec_dir, ignore_errors=True)
download_and_unpack(
working_dir,
f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz",
"96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
url.format(version=version),
shasum,
)
call(
[
"emconfigure",
mpdec_dir / "configure",
"CFLAGS=-fPIC",
"--prefix",
context.build_paths["prefix_dir"],
prefix,
"--disable-shared",
],
cwd=mpdec_dir,
Expand All @@ -306,6 +404,7 @@ def make_mpdec(context, working_dir):
cwd=mpdec_dir,
quiet=context.quiet,
)
write_library_config(prefix, "mpdec", mpdec_config, context.quiet)


@subdir("host_dir", clean_ok=True)
Expand Down Expand Up @@ -475,6 +574,10 @@ def main():

parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand")
install_emscripten_cmd = subcommands.add_parser(
"install-emscripten",
help="Install the appropriate version of Emscripten",
)
build = subcommands.add_parser("build", help="Build everything")
configure_build = subcommands.add_parser(
"configure-build-python", help="Run `configure` for the build Python"
Expand Down Expand Up @@ -512,6 +615,7 @@ def main():
)

for subcommand in (
install_emscripten_cmd,
build,
configure_build,
make_libffi_cmd,
Expand Down Expand Up @@ -568,15 +672,18 @@ def main():

context = parser.parse_args()

context.build_paths = get_build_paths(context.cross_build_dir)

if context.emsdk_cache:
if context.emsdk_cache and context.subcommand != "install-emscripten":
validate_emsdk_version(context.emsdk_cache)
context.emsdk_cache = Path(context.emsdk_cache).absolute()
else:
print("Build will use EMSDK from current environment.")

context.build_paths = get_build_paths(
context.cross_build_dir, context.emsdk_cache
)

dispatch = {
"install-emscripten": install_emscripten,
"make-libffi": make_emscripten_libffi,
"make-mpdec": make_mpdec,
"configure-build-python": configure_build_python,
Expand Down
14 changes: 14 additions & 0 deletions Tools/wasm/emscripten/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Any data that can vary between Python versions is to be kept in this file.
# This allows for blanket copying of the Emscripten build code between supported
# Python versions.
emscripten-version = "4.0.12"

[libffi]
url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz"
version = "3.4.6"
shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e"

[mpdec]
url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz"
version = "4.0.1"
shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8"
1 change: 0 additions & 1 deletion Tools/wasm/emscripten/emscripten_version.txt
View file
Open in desktop

This file was deleted.

Loading