From 9da3ed3cd75fa3514ed74618358d77f18b89ffbf Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 10:30:52 +0100 Subject: [PATCH 01/10] gh-146197: Add Emscripten to CI --- .github/workflows/build.yml | 8 +++ .github/workflows/reusable-context.yml | 4 ++ .github/workflows/reusable-emscripten.yml | 67 +++++++++++++++++++++++ Tools/build/compute-changes.py | 13 +++++ 4 files changed, 92 insertions(+) create mode 100644 .github/workflows/reusable-emscripten.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fa2ab768dc48b..f58272f1cc3222 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -371,6 +371,12 @@ jobs: - name: Build and test run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml + build-wasi: name: 'WASI' needs: build-context @@ -650,6 +656,7 @@ jobs: - build-ubuntu - build-ubuntu-ssltests - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan @@ -706,5 +713,6 @@ jobs: }} ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d958d729168e23..fc80e6671b571c 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -41,6 +41,9 @@ on: # yamllint disable-line rule:truthy run-ubuntu: description: Whether to run the Ubuntu tests value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool run-wasi: description: Whether to run the WASI tests value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool @@ -65,6 +68,7 @@ jobs: run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 00000000000000..3ff659f4bfcc9e --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,67 @@ +name: Reusable Emscripten + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + EMSDK_CACHE: ${{ github.workspace }}/../emsdk-cache + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["libffi"], sort_keys=True).encode()) + h.update(json.dumps(config["mpdec"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + - name: "Install Node.js" + uses: actions/setup-node@v6 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + uses: actions/cache@v5 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + - name: "Install Python" + uses: actions/setup-python@v6 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Make dependencies" + run: python3 Platforms/emscripten make-dependencies + - name: "Configure host Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host Python" + run: python3 Platforms/emscripten make-host + - name: "Test" + run: python3 Platforms/emscripten run --test diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 4d92b083026b27..af4afb4c503457 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -50,6 +50,7 @@ ANDROID_DIRS = frozenset({"Android"}) IOS_DIRS = frozenset({"Apple", "iOS"}) MACOS_DIRS = frozenset({"Mac"}) +EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")}) WASI_DIRS = frozenset({Path("Platforms", "WASI")}) LIBRARY_FUZZER_PATHS = frozenset({ @@ -107,6 +108,7 @@ class Outputs: run_ci_fuzz: bool = False run_ci_fuzz_stdlib: bool = False run_docs: bool = False + run_emscripten: bool = False run_ios: bool = False run_macos: bool = False run_tests: bool = False @@ -126,6 +128,7 @@ def compute_changes() -> None: # Otherwise, just run the tests outputs = Outputs( run_android=True, + run_emscripten=True, run_ios=True, run_macos=True, run_tests=True, @@ -196,6 +199,8 @@ def get_file_platform(file: Path) -> str | None: return "ios" if first_part in ANDROID_DIRS: return "android" + if len(file.parts) >= 2 and Path(*file.parts[:2]) in EMSCRIPTEN_DIRS: + return "emscripten" if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: return "wasi" return None @@ -244,6 +249,10 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = True platforms_changed.add("macos") continue + if file.name == "reusable-emscripten.yml": + run_tests = True + platforms_changed.add("emscripten") + continue if file.name == "reusable-wasi.yml": run_tests = True platforms_changed.add("wasi") @@ -284,18 +293,21 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if run_tests: if not has_platform_specific_change or not platforms_changed: run_android = True + run_emscripten = True run_ios = True run_macos = True run_ubuntu = True run_wasi = True else: run_android = "android" in platforms_changed + run_emscripten = "emscripten" in platforms_changed run_ios = "ios" in platforms_changed run_macos = "macos" in platforms_changed run_ubuntu = False run_wasi = "wasi" in platforms_changed else: run_android = False + run_emscripten = False run_ios = False run_macos = False run_ubuntu = False @@ -306,6 +318,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_ci_fuzz=run_ci_fuzz, run_ci_fuzz_stdlib=run_ci_fuzz_stdlib, run_docs=run_docs, + run_emscripten=run_emscripten, run_ios=run_ios, run_macos=run_macos, run_tests=run_tests, From d77a3399175791e644f6309e27de53d92178824d Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 10:55:11 +0100 Subject: [PATCH 02/10] Try to fix cache --- .github/workflows/reusable-emscripten.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index 3ff659f4bfcc9e..a80de054f4fe9f 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -11,8 +11,6 @@ jobs: name: 'build and test' runs-on: ubuntu-24.04 timeout-minutes: 60 - env: - EMSDK_CACHE: ${{ github.workspace }}/../emsdk-cache steps: - uses: actions/checkout@v6 with: @@ -32,10 +30,13 @@ jobs: h.update(json.dumps(config["libffi"], sort_keys=True).encode()) h.update(json.dumps(config["mpdec"], sort_keys=True).encode()) h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"emscripten-version={config['emscripten-version']}\n") f.write(f"node-version={config['node-version']}\n") f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") - name: "Install Node.js" uses: actions/setup-node@v6 with: From 4276a5117df9c633e5624b41a5b142ac1d389f17 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 11:07:50 +0100 Subject: [PATCH 03/10] Update Tools/build/compute-changes.py Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Tools/build/compute-changes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index af4afb4c503457..c15dc599f993f3 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -48,9 +48,9 @@ SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) ANDROID_DIRS = frozenset({"Android"}) +EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")}) IOS_DIRS = frozenset({"Apple", "iOS"}) MACOS_DIRS = frozenset({"Mac"}) -EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")}) WASI_DIRS = frozenset({Path("Platforms", "WASI")}) LIBRARY_FUZZER_PATHS = frozenset({ From 830188cc36f5f79f10a535caa0b60a633fff6eb6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 11:30:06 +0100 Subject: [PATCH 04/10] Check that cache works From 1520ee81113f4ea962e9988387a5c6fff9aef957 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 12:44:38 +0100 Subject: [PATCH 05/10] Move mpdec and libffi into a dependencies subtable --- .github/workflows/reusable-emscripten.yml | 3 +-- Platforms/emscripten/__main__.py | 8 ++++---- Platforms/emscripten/config.toml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index a80de054f4fe9f..cbf8f65c3e1d5e 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -27,8 +27,7 @@ jobs: config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) h = hashlib.sha256() - h.update(json.dumps(config["libffi"], sort_keys=True).encode()) - h.update(json.dumps(config["mpdec"], sort_keys=True).encode()) + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" with open(os.environ["GITHUB_OUTPUT"], "a") as f: diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py index 6a7963413da31a..938ae3cd200f6c 100644 --- a/Platforms/emscripten/__main__.py +++ b/Platforms/emscripten/__main__.py @@ -350,7 +350,7 @@ def write_library_config(prefix, name, config, quiet): def make_emscripten_libffi(context, working_dir): validate_emsdk_version(context.emsdk_cache) prefix = context.build_paths["prefix_dir"] - libffi_config = load_config_toml()["libffi"] + libffi_config = load_config_toml()["dependencies"]["libffi"] if not should_build_library( prefix, "libffi", libffi_config, context.quiet ): @@ -378,7 +378,7 @@ def make_emscripten_libffi(context, working_dir): def make_mpdec(context, working_dir): validate_emsdk_version(context.emsdk_cache) prefix = context.build_paths["prefix_dir"] - mpdec_config = load_config_toml()["mpdec"] + mpdec_config = load_config_toml()["dependencies"]["mpdec"] if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): return @@ -705,7 +705,7 @@ def main(): help=( "If passed, will add the default test arguments to the beginning of the command. " "Default arguments loaded from Platforms/emscripten/config.toml" - ) + ), ) run.add_argument( "args", @@ -713,7 +713,7 @@ def main(): help=( "Arguments to pass to the emscripten Python " "(use '--' to separate from run options)", - ) + ), ) add_cross_build_dir_option(run) diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index c474078fb48ba3..99a7b73884559d 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -12,12 +12,12 @@ test-args = [ "-W", ] -[libffi] +[dependencies.libffi] url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz" version = "3.4.6" shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" -[mpdec] +[dependencies.mpdec] url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz" version = "4.0.1" shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8" From f9c238618d8d72c7139de1e3c7699b3b2c64beb9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 13:07:26 +0100 Subject: [PATCH 06/10] Add build-emscripten to allowed-failures --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f58272f1cc3222..a3898aad4e1911 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -671,6 +671,7 @@ jobs: with: allowed-failures: >- build-android, + build-emscripten, build-windows-msi, build-ubuntu-ssltests, test-hypothesis, From 4eff826fda2cae3bc2ad7dcbe8b9309e1382457f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 14:03:16 +0100 Subject: [PATCH 07/10] Update .github/workflows/reusable-emscripten.yml Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-emscripten.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index cbf8f65c3e1d5e..d2f3162a854398 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -45,6 +45,7 @@ jobs: with: path: ${{ env.EMSDK_CACHE }} key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} - name: "Install Python" uses: actions/setup-python@v6 with: From b40fbf901e18bed9f473f070fdb2c7c25c5033c5 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 14:08:14 +0100 Subject: [PATCH 08/10] restore-key*s* --- .github/workflows/reusable-emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index d2f3162a854398..e5e74d2ed7a37e 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -45,7 +45,7 @@ jobs: with: path: ${{ env.EMSDK_CACHE }} key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} - restore-key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} - name: "Install Python" uses: actions/setup-python@v6 with: From 54b1b12c43b482f743a5506c4864388ba506c012 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 15:09:01 +0100 Subject: [PATCH 09/10] Make libffi build also key on mpdec --- .github/workflows/reusable-emscripten.yml | 5 ++++- Platforms/emscripten/__main__.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index e5e74d2ed7a37e..eac1d78233d470 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -41,6 +41,7 @@ jobs: with: node-version: ${{ steps.emscripten-config.outputs.node-version }} - name: "Cache Emscripten SDK" + id: emsdk-cache uses: actions/cache@v5 with: path: ${{ env.EMSDK_CACHE }} @@ -59,7 +60,9 @@ jobs: - name: "Make build Python" run: python3 Platforms/emscripten make-build-python - name: "Make dependencies" - run: python3 Platforms/emscripten make-dependencies + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} - name: "Configure host Python" run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache - name: "Make host Python" diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py index 938ae3cd200f6c..3db998a048e7e2 100644 --- a/Platforms/emscripten/__main__.py +++ b/Platforms/emscripten/__main__.py @@ -351,10 +351,17 @@ def make_emscripten_libffi(context, working_dir): validate_emsdk_version(context.emsdk_cache) prefix = context.build_paths["prefix_dir"] libffi_config = load_config_toml()["dependencies"]["libffi"] + with open(EMSCRIPTEN_DIR / "make_libffi.sh", "rb") as f: + libffi_config["make_libffi_shasum"] = hashlib.file_digest(f, "sha256").hexdigest() if not should_build_library( prefix, "libffi", libffi_config, context.quiet ): return + + if context.check_up_to_date: + print("libffi out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + url = libffi_config["url"] version = libffi_config["version"] shasum = libffi_config["shasum"] @@ -382,6 +389,10 @@ def make_mpdec(context, working_dir): if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): return + if context.check_up_to_date: + print("libmpdec out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + url = mpdec_config["url"] version = mpdec_config["version"] shasum = mpdec_config["shasum"] @@ -678,6 +689,14 @@ def main(): help="Build all static library dependencies", ) + for cmd in [make_mpdec_cmd, make_libffi_cmd, make_dependencies_cmd]: + cmd.add_argument( + "--check-up-to-date", + action="store_true", + default=False, + help=("If passed, will fail if dependency is out of date"), + ) + make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) From 9d545d1bb00db7b8967448bf640ced5fb0403b0f Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Fri, 20 Mar 2026 15:15:00 +0100 Subject: [PATCH 10/10] Force deps-hash update --- .github/workflows/reusable-emscripten.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index eac1d78233d470..549ff671a68e9c 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -29,6 +29,7 @@ jobs: h = hashlib.sha256() h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"emscripten-version={config['emscripten-version']}\n")