From 32f702944c758c4a63aeda539671bf040605f31d Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 14 Sep 2022 09:06:55 -0700 Subject: [PATCH] Allow JS library dependencies to be added in source code. This change introduces a new EM_JS_DEPS macros that can be used to specific the JS library dependencies of user code in EM_JS and/or EM_ASM blocks. This is especially important for library authors who don't want to have their users maintain link-time list of required symbols. See #14729 --- ChangeLog.md | 6 ++++ emcc.py | 2 +- emscripten.py | 3 ++ src/settings_internal.js | 1 + system/include/emscripten/em_macros.h | 27 ++++++++++++++ test/core/dyncall_specific.c | 2 ++ test/core/test_asan_js_stack_op.c | 2 ++ test/core/test_convertI32PairToI53Checked.cpp | 2 ++ test/core/test_int53.c | 2 ++ test/interop/test_add_function.cpp | 2 ++ test/other/test_offset_converter.c | 2 ++ test/other/test_runtime_keepalive.cpp | 2 ++ test/stack_overflow.cpp | 13 ++++--- test/test_c_preprocessor.c | 4 ++- test/test_core.py | 12 ++----- test/test_other.py | 35 +++++++++++++++---- test/unistd/io.c | 2 ++ tools/extract_metadata.py | 27 ++++++++++---- tools/ports/sdl2.py | 1 + tools/webidl_binder.py | 2 ++ 20 files changed, 119 insertions(+), 30 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index a80a6ec0ee331..d1118eb70b83b 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -22,6 +22,12 @@ See docs/process.md for more on how version tagging works. ----------------------- - In Wasm exception mode (`-fwasm-exceptions`), when `ASSERTIONS` is enabled, uncaught exceptions will display stack traces. (#17979) +- It is now possible to specify indirect dependencies on JS library functions + directly in C/C++ source code. For example, in the case of a EM_JS or EM_ASM + JavaScript function that depends on a JS library function. See the + `EM_JS_DEPS` macro in the `em_macros.h` header. Adding dependencies in this + way avoids the need to specify them on the command line with + `-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE`. (#17854) 3.1.23 - 09/23/22 ----------------- diff --git a/emcc.py b/emcc.py index a81f8bfa826c8..00581eda57756 100755 --- a/emcc.py +++ b/emcc.py @@ -1907,7 +1907,7 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.USE_PTHREADS: settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ - '$registerTLSInit', + '$registerTLSInit', ] if settings.RELOCATABLE: diff --git a/emscripten.py b/emscripten.py index dda3e2de07789..b8e30a9dfe7f9 100644 --- a/emscripten.py +++ b/emscripten.py @@ -165,6 +165,9 @@ def update_settings_glue(wasm_file, metadata): # exported. In theory it should always be present since its defined in compiler-rt. assert 'emscripten_stack_get_end' in metadata.exports + for deps in metadata.jsDeps: + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.extend(deps.split(',')) + def apply_static_code_hooks(forwarded_json, code): code = shared.do_replace(code, '<<< ATINITS >>>', str(forwarded_json['ATINITS'])) diff --git a/src/settings_internal.js b/src/settings_internal.js index 9fd59376bc31e..0732a2ace2a09 100644 --- a/src/settings_internal.js +++ b/src/settings_internal.js @@ -38,6 +38,7 @@ var SIDE_MODULE_IMPORTS = []; // programs contains EM_JS or EM_ASM data section, in which case these symbols // won't exist. var EXPORT_IF_DEFINED = ['__start_em_asm', '__stop_em_asm', + '__start_em_lib_deps', '__stop_em_lib_deps', '__start_em_js', '__stop_em_js']; // Like EXPORTED_FUNCTIONS, but symbol is required to exist in native code. diff --git a/system/include/emscripten/em_macros.h b/system/include/emscripten/em_macros.h index 3f5aecf171e64..2206e58591806 100644 --- a/system/include/emscripten/em_macros.h +++ b/system/include/emscripten/em_macros.h @@ -14,3 +14,30 @@ #else #define EM_IMPORT(NAME) #endif + +/* + * EM_JS_DEPS: Use this macro to declare indirect dependencies on JS symbols. + * The first argument is just unique name for the set of dependencies. The + * second argument is a C string that lists JS library symbols in the same way + * they would be specified in the DEFAULT_LIBRARY_FUNCS_TO_INCLUDE command line + * setting. + * + * For example, if your code contains an EM_ASM or EM_JS block that make use of + * the allocate and stackSave JS library functions then you might write this in + * your library source code: + * + * EM_JS_DEPS(mylib_dep, "$allocate,$stackSave"); + * + * The emscripten linker will then pick this up and make sure those symbols get + * included in the JS support library. + * + * Dependencies declared in this way will be included if-and-only-if the object + * file (translation unit) in which they exist is included by the linker, so + * it makes sense co-locate them with the EM_JS or EM_ASM code they correspond + * to. + */ +#define EM_JS_DEPS(tag, deps) \ + EMSCRIPTEN_KEEPALIVE \ + __attribute__((section("em_lib_deps"))) \ + __attribute__((aligned(1))) \ + char __em_lib_deps_##tag[] = deps; diff --git a/test/core/dyncall_specific.c b/test/core/dyncall_specific.c index fddd15cf57a7d..52c0ed1eef58d 100644 --- a/test/core/dyncall_specific.c +++ b/test/core/dyncall_specific.c @@ -19,6 +19,8 @@ int waka(int w, long long xy, int z) { return 42; } +EM_JS_DEPS(main, "$dynCall"); + int main() { EM_ASM({ // Note that these would need to use BigInts if the file were built with diff --git a/test/core/test_asan_js_stack_op.c b/test/core/test_asan_js_stack_op.c index 9ec1682894aa7..05895192e9633 100644 --- a/test/core/test_asan_js_stack_op.c +++ b/test/core/test_asan_js_stack_op.c @@ -5,6 +5,8 @@ EMSCRIPTEN_KEEPALIVE void c_func(char *str) { printf("%s\n", str); } +EM_JS_DEPS(js_func, "$allocateUTF8OnStack"); + EM_JS(void, js_func, (void), { _c_func(allocateUTF8OnStack('Hello, World!')); }); diff --git a/test/core/test_convertI32PairToI53Checked.cpp b/test/core/test_convertI32PairToI53Checked.cpp index f7a8f6646eed6..094cd86fc7bae 100644 --- a/test/core/test_convertI32PairToI53Checked.cpp +++ b/test/core/test_convertI32PairToI53Checked.cpp @@ -10,6 +10,8 @@ // Uncomment to compute the expected results without testing: //#define GENERATE_ANSWERS +EM_JS_DEPS(test, "$convertI32PairToI53Checked"); + double test(int64_t val) { int32_t lo = (uint32_t)val; int32_t hi = (uint64_t)val >> 32; diff --git a/test/core/test_int53.c b/test/core/test_int53.c index ab3c68bc3a78d..e72c04bc5f7a1 100644 --- a/test/core/test_int53.c +++ b/test/core/test_int53.c @@ -11,6 +11,8 @@ // Uncomment to compute the expected result: //#define GENERATE_ANSWERS +EM_JS_DEPS(main, "$convertI32PairToI53,$convertU32PairToI53,$readI53FromU64,$readI53FromI64,$writeI53ToI64,$writeI53ToI64Clamped,$writeI53ToU64Clamped,$writeI53ToI64Signaling,$writeI53ToU64Signaling"); + void writeI53ToI64_int64(int64_t *heapAddress, int64_t num) { #ifdef GENERATE_ANSWERS *heapAddress = num; diff --git a/test/interop/test_add_function.cpp b/test/interop/test_add_function.cpp index 03eccd37f1ac7..d20fc95bf3a51 100644 --- a/test/interop/test_add_function.cpp +++ b/test/interop/test_add_function.cpp @@ -20,6 +20,8 @@ extern "C" int baz() { return 3; } +EM_JS_DEPS(main, "$addFunction,$removeFunction"); + int main(int argc, char **argv) { #if defined(GROWTH) EM_ASM({ diff --git a/test/other/test_offset_converter.c b/test/other/test_offset_converter.c index bc866f501dba8..9076d607ca2d6 100644 --- a/test/other/test_offset_converter.c +++ b/test/other/test_offset_converter.c @@ -29,6 +29,8 @@ void magic_test_function(void) { puts("ok"); } +EM_JS_DEPS(test, "$ptrToString"); + int main(void) { magic_test_function(); return 0; diff --git a/test/other/test_runtime_keepalive.cpp b/test/other/test_runtime_keepalive.cpp index a6ef832effeda..005c309d59ac1 100644 --- a/test/other/test_runtime_keepalive.cpp +++ b/test/other/test_runtime_keepalive.cpp @@ -1,6 +1,8 @@ #include #include +EM_JS_DEPS(main, "$runtimeKeepalivePush,$runtimeKeepalivePop,$callUserCallback"); + int main() { EM_ASM({ Module["onExit"] = () => { out("onExit"); }; diff --git a/test/stack_overflow.cpp b/test/stack_overflow.cpp index 08f093c0ef782..506aa19447958 100644 --- a/test/stack_overflow.cpp +++ b/test/stack_overflow.cpp @@ -5,10 +5,13 @@ #include #include -#include -void __attribute__((noinline)) InteropString(char *staticBuffer) -{ +#include +#include + +EM_JS_DEPS(main, "$allocateUTF8OnStack"); + +void __attribute__((noinline)) InteropString(char *staticBuffer) { char *string = (char*)EM_ASM_PTR({ var str = "hello, this is a string! "; #if ONE_BIG_STRING @@ -27,12 +30,12 @@ void __attribute__((noinline)) InteropString(char *staticBuffer) }); } -int main() -{ +int main() { // Make C side consume a large portion of the stack, before bumping the rest with C++<->JS interop. char staticBuffer[512288] = {}; InteropString(staticBuffer); int stringLength = strlen(staticBuffer); printf("Got string: %s\n", staticBuffer); printf("Received a string of length %d.\n", stringLength); + return 0; } diff --git a/test/test_c_preprocessor.c b/test/test_c_preprocessor.c index cbdea92bcd888..9b13c8ad79b2e 100644 --- a/test/test_c_preprocessor.c +++ b/test/test_c_preprocessor.c @@ -33,6 +33,8 @@ EM_JS(void, test_remove_cpp_comments_in_shaders, (void), { if (numFailed) throw numFailed + ' tests failed!'; }); +EM_JS_DEPS(main, "$preprocess_c_code,$remove_cpp_comments_in_shaders"); + EM_JS(void, test_c_preprocessor, (void), { var numFailed = 0; function test(input, expected) { @@ -183,4 +185,4 @@ int main() test_remove_cpp_comments_in_shaders(); test_c_preprocessor(); -} \ No newline at end of file +} diff --git a/test/test_core.py b/test/test_core.py index 0b6c3e65c7368..7bb70c20f6e70 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -498,8 +498,6 @@ def test_sintvars(self): self.do_core_test('test_sintvars.c') def test_int53(self): - self.emcc_args += ['-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[$convertI32PairToI53,$convertU32PairToI53,$readI53FromU64,$readI53FromI64,$writeI53ToI64,$writeI53ToI64Clamped,$writeI53ToU64Clamped,$writeI53ToI64Signaling,$writeI53ToU64Signaling]'] - if common.EMTEST_REBASELINE: self.run_process([EMCC, test_file('core/test_int53.c'), '-o', 'a.js', '-DGENERATE_ANSWERS'] + self.emcc_args) ret = self.run_process(config.NODE_JS + ['a.js'], stdout=PIPE).stdout @@ -508,7 +506,6 @@ def test_int53(self): self.do_core_test('test_int53.c', interleaved_output=False) def test_int53_convertI32PairToI53Checked(self): - self.emcc_args += ['-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[$convertI32PairToI53Checked]'] if common.EMTEST_REBASELINE: self.run_process([EMCC, test_file('core/test_convertI32PairToI53Checked.cpp'), '-o', 'a.js', '-DGENERATE_ANSWERS'] + self.emcc_args) ret = self.run_process(config.NODE_JS + ['a.js'], stdout=PIPE).stdout @@ -6126,7 +6123,6 @@ def test_unistd_symlink_on_nodefs(self): @also_with_wasm_bigint def test_unistd_io(self): - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$ERRNO_CODES']) orig_compiler_opts = self.emcc_args.copy() for fs in ['MEMFS', 'NODEFS']: self.clear() @@ -7096,14 +7092,14 @@ def test_dyncall_specific(self, *args): self.skipTest('not compatible with WASM_BIGINT') cases = [ ('DIRECT', []), - ('DYNAMIC_SIG', ['-sDYNCALLS=1', '-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=$dynCall']), + ('DYNAMIC_SIG', ['-sDYNCALLS=1']), ] if '-sMINIMAL_RUNTIME=1' in args: self.emcc_args += ['--pre-js', test_file('minimal_runtime_exit_handling.js')] else: cases += [ ('EXPORTED', []), - ('EXPORTED_DYNAMIC_SIG', ['-sDYNCALLS=1', '-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=$dynCall', '-sEXPORTED_RUNTIME_METHODS=dynCall']), + ('EXPORTED_DYNAMIC_SIG', ['-sDYNCALLS=1', '-sEXPORTED_RUNTIME_METHODS=dynCall']), ('FROM_OUTSIDE', ['-sEXPORTED_RUNTIME_METHODS=dynCall_iiji']) ] @@ -7269,7 +7265,6 @@ def test_add_function(self): self.set_setting('WASM_ASYNC_COMPILATION', 0) self.set_setting('RESERVED_FUNCTION_POINTERS') self.set_setting('EXPORTED_RUNTIME_METHODS', ['callMain']) - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$addFunction', '$removeFunction']) src = test_file('interop/test_add_function.cpp') post_js = test_file('interop/test_add_function_post.js') self.emcc_args += ['--post-js', post_js] @@ -7668,7 +7663,6 @@ def test_webidl(self, mode, allow_memory_growth): # TODO(): Remove once we make webidl output closure-warning free. self.ldflags.append('-Wno-error=closure') self.set_setting('WASM_ASYNC_COMPILATION', 0) - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$intArrayFromString']) if self.maybe_closure(): # avoid closure minified names competing with our test code in the global name space self.set_setting('MODULARIZE') @@ -8575,7 +8569,6 @@ def test_fs_dict_none(self): def test_stack_overflow_check(self): self.set_setting('TOTAL_STACK', 1048576) self.set_setting('STACK_OVERFLOW_CHECK', 2) - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$allocateUTF8OnStack') self.do_runf(test_file('stack_overflow.cpp'), 'Aborted(stack overflow', assert_returncode=NON_ZERO) self.emcc_args += ['-DONE_BIG_STRING'] @@ -8983,7 +8976,6 @@ def test_asan_js_stack_op(self): self.emcc_args.append('-fsanitize=address') self.set_setting('ALLOW_MEMORY_GROWTH') self.set_setting('INITIAL_MEMORY', '300mb') - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$allocateUTF8OnStack']) self.do_runf(test_file('core/test_asan_js_stack_op.c'), expected_output='Hello, World!') diff --git a/test/test_other.py b/test/test_other.py index 200aa6357b855..118a04a38cd12 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -9684,7 +9684,7 @@ def get_file_gzipped_size(f): # Tests the library_c_preprocessor.js functionality. def test_c_preprocessor(self): - self.run_process([EMXX, test_file('test_c_preprocessor.c'), '--js-library', path_from_root('src/library_c_preprocessor.js'), '-sDEFAULT_LIBRARY_FUNCS_TO_INCLUDE=$remove_cpp_comments_in_shaders,$preprocess_c_code']) + self.run_process([EMXX, test_file('test_c_preprocessor.c'), '--js-library', path_from_root('src/library_c_preprocessor.js')]) normal = self.run_js('a.out.js') print(str(normal)) @@ -10023,7 +10023,6 @@ def test_proxy_to_pthread_stack(self): }) def test_offset_converter(self, *args): self.set_setting('USE_OFFSET_CONVERTER') - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$ptrToString']) self.emcc_args += ['--profiling-funcs'] self.do_runf(test_file('other/test_offset_converter.c'), 'ok', emcc_args=list(args)) @@ -10034,7 +10033,6 @@ def test_offset_converter(self, *args): def test_offset_converter_source_map(self, *args): self.set_setting('USE_OFFSET_CONVERTER') self.set_setting('LOAD_SOURCE_MAP') - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$ptrToString']) self.emcc_args += ['-gsource-map', '-DUSE_SOURCE_MAP'] self.do_runf(test_file('other/test_offset_converter.c'), 'ok', emcc_args=list(args)) @@ -11055,9 +11053,9 @@ def test_missing_malloc_export_indirect(self): # we used to include malloc by default. show a clear error in builds with # ASSERTIONS to help with any confusion when the user calls a JS API that # requires malloc - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$allocateUTF8') create_file('unincluded_malloc.c', r''' #include + EM_JS_DEPS(main, "$allocateUTF8"); int main() { EM_ASM({ try { @@ -11492,7 +11490,6 @@ def test_shell_Oz(self): def test_runtime_keepalive(self): self.uses_es6 = True - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$runtimeKeepalivePush', '$runtimeKeepalivePop', '$callUserCallback']) self.set_setting('EXIT_RUNTIME') self.do_other_test('test_runtime_keepalive.cpp') @@ -12495,11 +12492,12 @@ def test_bigint64array_polyfill(self): self.assertEqual(v1, v2, msg=m) def test_warn_once(self): - self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', ['$warnOnce']) create_file('main.c', r'''\ #include #include + EM_JS_DEPS(main, "$warnOnce"); + int main() { EM_ASM({ warnOnce("foo"); @@ -12540,3 +12538,28 @@ def test_fs_icase(self): def test_strict_js_closure(self): self.do_runf(test_file('hello_world.c'), emcc_args=['-sSTRICT_JS', '-Werror=closure', '--closure=1', '-O3']) + + def test_em_js_deps(self): + # Check that EM_JS_DEPS works. Specifically, multiple different instances in different + # object files. + create_file('f1.c', ''' + #include + + EM_JS_DEPS(other, "$allocateUTF8OnStack"); + ''') + create_file('f2.c', ''' + #include + + EM_JS_DEPS(main, "$getHeapMax"); + + int main() { + EM_ASM({ + err(getHeapMax()); + var x = stackSave(); + allocateUTF8OnStack("hello"); + stackRestore(x); + }); + return 0; + } + ''') + self.do_runf('f2.c', emcc_args=['f1.c']) diff --git a/test/unistd/io.c b/test/unistd/io.c index f09c562f6f275..86d8651058d49 100644 --- a/test/unistd/io.c +++ b/test/unistd/io.c @@ -13,6 +13,8 @@ #include #include +EM_JS_DEPS(main, "$ERRNO_CODES"); + int main() { EM_ASM( FS.mkdir('/working'); diff --git a/tools/extract_metadata.py b/tools/extract_metadata.py index c48c1d6367927..0dce6014fb4e0 100644 --- a/tools/extract_metadata.py +++ b/tools/extract_metadata.py @@ -3,11 +3,15 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. +import logging + from . import webassembly from .webassembly import OpCode, AtomicOpCode, MemoryOpCode from .shared import exit_with_error from .settings import settings +logger = logging.getLogger('extract_metadata') + def skip_function_header(module): num_local_decls = module.read_uleb() @@ -177,12 +181,15 @@ def data_to_string(data): return data -def get_asm_strings(module, export_map): - if '__start_em_asm' not in export_map or '__stop_em_asm' not in export_map: +def get_section_strings(module, export_map, section_name): + start_name = f'__start_{section_name}' + stop_name = f'__stop_{section_name}' + if start_name not in export_map or stop_name not in export_map: + logger.debug(f'no start/stop symbols found for section: {section_name}') return {} - start = export_map['__start_em_asm'] - end = export_map['__stop_em_asm'] + start = export_map[start_name] + end = export_map[stop_name] start_global = module.get_global(start.index) end_global = module.get_global(end.index) start_addr = get_global_value(start_global) @@ -190,7 +197,7 @@ def get_asm_strings(module, export_map): seg = find_segment_with_address(module, start_addr) if not seg: - exit_with_error('unable to find segment starting at __start_em_asm: %s' % start_addr) + exit_with_error(f'unable to find segment starting at __start_{section_name}: {start_addr}') seg, seg_offset = seg asm_strings = {} @@ -225,9 +232,13 @@ def get_main_reads_params(module, export_map): def get_named_globals(module, exports): named_globals = {} + internal_start_stop_symbols = set(['__start_em_asm', '__stop_em_asm', + '__start_em_lib_deps', '__stop_em_lib_deps', + '__em_lib_deps']) + internal_prefixes = ('__em_js__', '__em_lib_deps') for export in exports: if export.kind == webassembly.ExternType.GLOBAL: - if export.name in ('__start_em_asm', '__stop_em_asm') or export.name.startswith('__em_js__'): + if export.name in internal_start_stop_symbols or any(export.name.startswith(p) for p in internal_prefixes): continue g = module.get_global(export.index) named_globals[export.name] = str(get_global_value(g)) @@ -269,6 +280,7 @@ class Metadata: imports: [] export: [] asmConsts: [] + jsDeps: [] emJsFuncs: {} emJsFuncTypes: [] features: [] @@ -321,7 +333,8 @@ def extract_metadata(filename): metadata = Metadata() metadata.imports = import_names metadata.exports = get_export_names(module) - metadata.asmConsts = get_asm_strings(module, export_map) + metadata.asmConsts = get_section_strings(module, export_map, 'em_asm') + metadata.jsDeps = [d for d in get_section_strings(module, export_map, 'em_lib_deps').values() if d] metadata.emJsFuncs = em_js_funcs metadata.emJsFuncTypes = em_js_func_types metadata.features = features diff --git a/tools/ports/sdl2.py b/tools/ports/sdl2.py index f8907f2e1b72f..0e904bdd68d37 100644 --- a/tools/ports/sdl2.py +++ b/tools/ports/sdl2.py @@ -81,6 +81,7 @@ def clear(ports, settings, shared): def linker_setup(ports, settings): + # TODO(sbc): Move these into native code use EM_JS_DEPS macro. settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$autoResumeAudioContext', '$dynCall'] diff --git a/tools/webidl_binder.py b/tools/webidl_binder.py index 72f5ccc93aef9..cf826ec169c28 100644 --- a/tools/webidl_binder.py +++ b/tools/webidl_binder.py @@ -81,6 +81,8 @@ def getExtendedAttribute(self, _name): pre_c += [r''' #include + +EM_JS_DEPS(webidl_binder, "$intArrayFromString"); '''] mid_c += [r'''