Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade emsdk version to v4.0.3 #23633

Merged
merged 1 commit into from
Feb 12, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
[submodule "cmake/external/emsdk"]
path = cmake/external/emsdk
url = https://github.com/emscripten-core/emsdk.git
branch = 3.1.59
branch = 4.0.3
2 changes: 1 addition & 1 deletion cmake/external/emsdk
Submodule emsdk updated 43 files
+34 −38 .circleci/config.yml
+17 −17 .flake8
+39 −0 .github/workflows/create-release.yml
+78 −0 .github/workflows/tag-release.yml
+4 −3 README.md
+35 −9 bazel/README.md
+24 −6 bazel/deps.bzl
+20 −18 bazel/emscripten_deps.bzl
+3 −3 bazel/emscripten_toolchain/BUILD.bazel
+0 −0 bazel/emscripten_toolchain/default_config
+7 −6 bazel/emscripten_toolchain/link_wrapper.py
+13 −3 bazel/emscripten_toolchain/toolchain.bzl
+22 −13 bazel/emscripten_toolchain/wasm_cc_binary.bzl
+133 −0 bazel/revisions.bzl
+1 −0 bazel/test_secondary_lto_cache/.bazelrc
+4 −0 bazel/test_secondary_lto_cache/.gitignore
+19 −0 bazel/test_secondary_lto_cache/BUILD
+1 −0 bazel/test_secondary_lto_cache/MODULE.bazel
+31 −0 bazel/test_secondary_lto_cache/WORKSPACE
+6 −0 bazel/test_secondary_lto_cache/hello-world.cc
+95 −1 bazel/toolchains.bzl
+2 −2 docker/Dockerfile
+39 −1 emscripten-releases-tags.json
+1 −11 emsdk
+0 −21 emsdk.bat
+1 −9 emsdk.ps1
+67 −35 emsdk.py
+2 −0 emsdk_env.fish
+30 −324 emsdk_manifest.json
+33 −17 scripts/create_release.py
+44 −0 scripts/get_emscripten_revision_info.py
+25 −0 scripts/get_release_info.py
+1 −1 scripts/update_bazel_workspace.py
+15 −11 scripts/update_node.py
+12 −22 scripts/update_python.py
+19 −0 scripts/zip.py
+1 −1 test/test.bat
+7 −7 test/test.py
+0 −7 test/test_activation.ps1
+6 −0 test/test_bazel.ps1
+14 −12 test/test_bazel.sh
+7 −6 test/test_bazel_mac.sh
+0 −3 test/test_path_preservation.ps1
28 changes: 8 additions & 20 deletions cmake/external/onnxruntime_external_deps.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@ else()
message("Setting pybind11_dep")
set(pybind11_dep pybind11::pybind11)
endif()

endif()
onnxruntime_fetchcontent_declare(
onnx
Expand Down Expand Up @@ -559,7 +559,7 @@ if (onnxruntime_USE_XNNPACK)
find_library(xnnpack_LIBRARY NAMES XNNPACK)
find_library(microkernels_prod_LIBRARY NAMES microkernels-prod)
find_package(unofficial-pthreadpool CONFIG REQUIRED)

target_include_directories(xnnpack INTERFACE "${XNNPACK_HDR}")
set(XNNPACK_INCLUDE_DIR ${XNNPACK_DIR}/include)
set(onnxruntime_EXTERNAL_LIBRARIES_XNNPACK ${xnnpack_LIBRARY} ${microkernels_prod_LIBRARY} unofficial::pthreadpool unofficial::pthreadpool_interface)
Expand Down Expand Up @@ -643,10 +643,12 @@ if (onnxruntime_USE_WEBGPU)
dawn
URL ${DEP_URL_dawn}
URL_HASH SHA1=${DEP_SHA1_dawn}
# All previous patches are merged into the upstream dawn project. We don't need to apply any patches right now.
# if we need to apply patches in the future, we can uncomment the following line.

# PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/dawn/dawn.patch
# # All previous patches are merged into the upstream dawn project. We don't need to apply any patches right now.
# # if we need to apply patches in the future, we can uncomment the following line.
#
# The dawn.patch contains the following changes:
# - https://dawn-review.googlesource.com/c/dawn/+/225514
PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/dawn/dawn.patch
EXCLUDE_FROM_ALL
)
endif()
Expand All @@ -657,20 +659,6 @@ if (onnxruntime_USE_WEBGPU)

if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
set(DAWN_EMSCRIPTEN_TOOLCHAIN "${REPO_ROOT}/cmake/external/emsdk/upstream/emscripten" CACHE STRING "" FORCE)

# Add the missing files from the emsdk installation
#
# For a "standard" emscripten build, the folder "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/" is not used. This is the
# reason why EMSDK installation does not include it.
# However, currently the WebGPU support in Emscripten is still being developed and the Dawn project is maintaining
# a fork of the Emscripten toolchain. As an extra build step, Dawn needs to generate some files using the file
# "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/gen_struct_info.py" from emscripten, which is missing in the emscripten
# installed by emsdk.
#
# We keep a copy of the missing file(s) in ${PROJECT_SOURCE_DIR}/patches/emscripten/, and now we extract them to the
# emscripten toolchain folder.
execute_process(COMMAND ${CMAKE_COMMAND} -E tar x "${PROJECT_SOURCE_DIR}/patches/emscripten/patch_3.1.74.tgz"
WORKING_DIRECTORY ${DAWN_EMSCRIPTEN_TOOLCHAIN})
else()
if (onnxruntime_BUILD_DAWN_MONOLITHIC_LIBRARY)
set(DAWN_BUILD_MONOLITHIC_LIBRARY ON CACHE BOOL "" FORCE)
Expand Down
55 changes: 55 additions & 0 deletions cmake/onnxruntime_webassembly.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -472,4 +472,59 @@ jsepDownload:_pp_")
endif()

set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME ${target_name} SUFFIX ".mjs")

#
# The following POST_BUILD script is a workaround for enabling:
# - using onnxruntime-web with Multi-threading enabled when import from CDN
# - using onnxruntime-web when consumed in some frameworks like Vite
#
# In the use case mentioned above, the file name of the script may be changed. So we need to replace the line:
# `new Worker(new URL("ort-wasm-*.mjs", import.meta.url),`
# with
# `new Worker(new URL(import.meta.url),`
#
# This behavior is introduced in https://github.com/emscripten-core/emscripten/pull/22165. Since it's unlikely to be
# reverted, and there is no config to disable this behavior, we have to use a post-build script to workaround it.
#

# Generate a script to do the post-build work
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/wasm_post_build.js "
const fs = require('fs');
const path = require('path');
// node wasm_post_build.js <mjsFilePath>
const mjsFilePath = process.argv[2];
let contents = fs.readFileSync(mjsFilePath).toString();
const regex = 'new Worker\\\\(new URL\\\\(\".+?\", ?import\\\\.meta\\\\.url\\\\),';
const matches = [...contents.matchAll(new RegExp(regex, 'g'))];
if (matches.length !== 1) {
throw new Error(
`Unexpected number of matches for \"${regex}\" in \"${filepath}\": ${matches.length}.`,
);
}
// Replace the only occurrence.
contents = contents.replace(
new RegExp(regex),
`new Worker(new URL(import.meta.url),`,
);
fs.writeFileSync(mjsFilePath, contents);
"
)

find_program(NODE_EXECUTABLE node required)
if (NOT NODE_EXECUTABLE)
message(FATAL_ERROR "Node is required to run the post-build script")
endif()

add_custom_command(
TARGET onnxruntime_webassembly
POST_BUILD
# Backup file at $<TARGET_FILE_NAME:onnxruntime_webassembly>.bak
COMMAND ${CMAKE_COMMAND} -E copy_if_different "$<TARGET_FILE_NAME:onnxruntime_webassembly>" "$<TARGET_FILE_NAME:onnxruntime_webassembly>.bak"
snnn marked this conversation as resolved.
Show resolved Hide resolved
COMMAND ${CMAKE_COMMAND} -E echo "Performing post-process for $<TARGET_FILE_NAME:onnxruntime_webassembly>"
COMMAND ${NODE_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/wasm_post_build.js" "$<TARGET_FILE_NAME:onnxruntime_webassembly>"
)
endif()
23 changes: 23 additions & 0 deletions cmake/patches/dawn/dawn.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
diff --git a/src/emdawnwebgpu/CMakeLists.txt b/src/emdawnwebgpu/CMakeLists.txt
index 6e8ae37593..633af91eef 100644
--- a/src/emdawnwebgpu/CMakeLists.txt
+++ b/src/emdawnwebgpu/CMakeLists.txt
@@ -77,9 +77,17 @@ if (${DAWN_ENABLE_EMSCRIPTEN})
"${arg_UNPARSED_ARGUMENTS}")
endif()

+ # since Emscripten 4.0.3, file gen_struct_info.py is moved to outside of directory maint.
+ if (EXISTS "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/gen_struct_info.py")
+ set(EM_GEN_STRUCT_INFO_SCRIPT "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/gen_struct_info.py")
+ elseif (EXISTS "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/gen_struct_info.py")
+ set(EM_GEN_STRUCT_INFO_SCRIPT "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/gen_struct_info.py")
+ else()
+ message(FATAL_ERROR "Dawn: Failed to locate file gen_struct_info.py from Emscripten.")
+ endif()
set(ARGS
${Python3_EXECUTABLE}
- "${DAWN_EMSCRIPTEN_TOOLCHAIN}/tools/maint/gen_struct_info.py"
+ "${EM_GEN_STRUCT_INFO_SCRIPT}"
-q
"${EM_BUILD_GEN_DIR}/struct_info_webgpu.json"
"-I=${EM_BUILD_GEN_DIR}/include"
Binary file removed cmake/patches/emscripten/patch_3.1.74.tgz
Binary file not shown.
2 changes: 1 addition & 1 deletion tools/ci_build/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ def convert_arg_line_to_args(self, arg_line):
# WebAssembly build
parser.add_argument("--build_wasm", action="store_true", help="Build for WebAssembly")
parser.add_argument("--build_wasm_static_lib", action="store_true", help="Build for WebAssembly static library")
parser.add_argument("--emsdk_version", default="3.1.59", help="Specify version of emsdk")
parser.add_argument("--emsdk_version", default="4.0.3", help="Specify version of emsdk")

parser.add_argument("--enable_wasm_simd", action="store_true", help="Enable WebAssembly SIMD")
parser.add_argument("--enable_wasm_threads", action="store_true", help="Enable WebAssembly multi-threads support")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
name: ${{ parameters.PoolName }}
variables:
buildArch: x64
CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --build_wasm --enable_wasm_simd --enable_wasm_threads --use_extensions --cmake_extra_defines onnxruntime_WEBASSEMBLY_DEFAULT_EXTENSION_FLAGS=ON ${{ parameters.ExtraBuildArgs }}'
CommonBuildArgs: '--parallel --config ${{ parameters.BuildConfig }} --skip_submodule_sync --build_wasm --enable_wasm_simd --enable_wasm_threads ${{ parameters.ExtraBuildArgs }}'
runCodesignValidationInjection: false
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
Expand Down Expand Up @@ -82,15 +82,15 @@ jobs:
- script: |
set -ex
cd '$(Build.SourcesDirectory)/cmake/external/emsdk'
./emsdk install 3.1.59 ccache-git-emscripten-64bit
./emsdk activate 3.1.59 ccache-git-emscripten-64bit
./emsdk install 4.0.3 ccache-git-emscripten-64bit
./emsdk activate 4.0.3 ccache-git-emscripten-64bit
displayName: 'emsdk install and activate ccache for emscripten'
- ${{if eq(parameters.WithCache, false)}}:
- script: |
set -ex
cd '$(Build.SourcesDirectory)/cmake/external/emsdk'
./emsdk install 3.1.59
./emsdk activate 3.1.59
./emsdk install 4.0.3
./emsdk activate 4.0.3
displayName: 'emsdk install and activate ccache for emscripten'

- template: build-linux-wasm-step.yml
Expand Down
Loading