From 4856f06aa77ea0e422cfa9b30b816b6aad6bba26 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Mon, 30 Nov 2020 08:10:58 -0800 Subject: [PATCH] Integrate posixtestsuite for pthread testing This change adds posixtestsuite as a git submodule (in tests/third_party). It also adds a new test suite that auto-populates based on the files in `posixtestsuite/conformance/interfaces/pthread_*` The submodule is optional in that the testsuite simply remains empty if it is not found. --- .circleci/config.yml | 15 +++- .gitmodules | 3 + src/preamble.js | 2 +- tests/runner.py | 4 +- tests/test_posixtest.py | 116 +++++++++++++++++++++++++++++++ tests/test_posixtest_browser.py | 18 +++++ tests/third_party/posixtestsuite | 1 + 7 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 100644 tests/test_posixtest.py create mode 100644 tests/test_posixtest_browser.py create mode 160000 tests/third_party/posixtestsuite diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c6f1b436a57f..46d2e20c1f495 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,6 +123,9 @@ commands: # Must be absolute path or relative path from working_directory at: ~/ - checkout + - run: + name: submodule update + command: git submodule update --init - emsdk-env - npm-install - run: @@ -160,6 +163,9 @@ commands: # Must be absolute path or relative path from working_directory at: ~/ - checkout + - run: + name: submodule update + command: git submodule update --init - emsdk-env - npm-install - run: @@ -250,7 +256,7 @@ commands: EXTRA_AUTOSTART=$TMPDIR/autostart startx /usr/bin/openbox-session -- $DISPLAY -config ~/.config/X11/xorg.conf -nolisten tcp & cat $TMPDIR/fifo > /dev/null # wait until $EXTRA_AUTOSTART is spawned, which indicates the end of Openbox initialization rm -r $TMPDIR - python3 tests/runner.py browser skip:browser.test_sdl2_mouse skip:browser.test_html5_webgl_create_context skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget + python3 tests/runner.py browser posixtest.test_pthread_create_1_1 skip:browser.test_sdl2_mouse skip:browser.test_html5_webgl_create_context skip:browser.test_webgl_offscreen_canvas_in_pthread skip:browser.test_webgl_offscreen_canvas_in_mainthread_after_pthread skip:browser.test_glut_glutget python3 tests/runner.py emrun openbox --exit wait || true # wait for startx to shutdown cleanly, or not @@ -261,6 +267,9 @@ commands: # Must be absolute path or relative path from working_directory at: ~/ - checkout + - run: + name: submodule update + command: git submodule update --init - emsdk-env - npm-install - run: @@ -282,7 +291,7 @@ commands: command: | export EMTEST_BROWSER="/usr/bin/google-chrome $CHROME_FLAGS_BASE $CHROME_FLAGS_HEADLESS $CHROME_FLAGS_WASM $CHROME_FLAGS_NOCACHE" # skip test_zzz_zzz_4GB_fail as it OOMs on the current bot - python3 tests/runner.py browser skip:browser.test_zzz_zzz_4GB_fail + python3 tests/runner.py browser posixtest.test_pthread_create_1_1 skip:browser.test_zzz_zzz_4GB_fail python3 tests/runner.py emrun test-sockets-chrome: description: "Runs emscripten sockets tests under chrome" @@ -384,7 +393,7 @@ jobs: steps: - run-tests: # also add a few asan tests - test_targets: "wasm2 asan.test_embind* asan.test_abort_on_exceptions asan.test_ubsan_full_left_shift_fsanitize_integer" + test_targets: "wasm2 asan.test_embind* asan.test_abort_on_exceptions asan.test_ubsan_full_left_shift_fsanitize_integer posixtest.test_pthread_create_1_1" test-wasm3: executor: bionic steps: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..40f9d1630049b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/third_party/posixtestsuite"] + path = tests/third_party/posixtestsuite + url = https://github.com/juj/posixtestsuite diff --git a/src/preamble.js b/src/preamble.js index cf04771f3e16f..9c1716156c6c8 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -60,7 +60,7 @@ var ABORT = false; // set by exit() and abort(). Passed to 'onExit' handler. // NOTE: This is also used as the process return code code in shell environments // but only when noExitRuntime is false. -var EXITSTATUS = 0; +var EXITSTATUS; /** @type {function(*, string=)} */ function assert(condition, text) { diff --git a/tests/runner.py b/tests/runner.py index 7aef46c99dfda..e91c36d6f2589 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -305,6 +305,8 @@ def make_executable(name): 'asan', 'lsan', 'wasm2ss', + 'posixtest', + 'posixtest_browser', ] @@ -1827,7 +1829,7 @@ def flattened_tests(loaded_tests): def suite_for_module(module, tests): - suite_supported = module.__name__ in ('test_core', 'test_other') + suite_supported = module.__name__ in ('test_core', 'test_other', 'test_posixtest') if not EMTEST_SAVE_DIR: has_multiple_tests = len(tests) > 1 has_multiple_cores = parallel_testsuite.num_cores() > 1 diff --git a/tests/test_posixtest.py b/tests/test_posixtest.py new file mode 100644 index 0000000000000..3c32098f08389 --- /dev/null +++ b/tests/test_posixtest.py @@ -0,0 +1,116 @@ +# Copyright 2020 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +"""Runs the pthreads test from the upstream posixtest suite in: + ./tests/third_party/posixtestsuite +See + https://github.com/juj/posixtestsuite +""" + +import glob +import os + +from runner import RunnerCore, path_from_root +from tools import config +from tools.shared import EMCC +import test_posixtest_browser + +testsuite_root = path_from_root('tests', 'third_party', 'posixtestsuite') + + +class posixtest(RunnerCore): + """Run the suite under node (and in parallel) + + This class get populated dynamically below. + """ + pass + + +def filter_tests(all_tests): + pthread_tests = [t for t in all_tests if t.startswith('pthread_')] + # filter out some tests we don't support + pthread_tests = [t for t in pthread_tests if not t.startswith('pthread_atfork')] + pthread_tests = [t for t in pthread_tests if not t.startswith('pthread_sigmask')] + return pthread_tests + + +def get_pthread_tests(): + # For now, we don't require the submodule to exist. In this case we just report + # no tests + pthread_test_root = os.path.join(testsuite_root, 'conformance', 'interfaces') + if not os.path.exists(pthread_test_root): + print('posixtestsuite not found (run git submodule update --init?)') + return [] + pthread_tests = filter_tests(os.listdir(pthread_test_root)) + pthread_tests = [os.path.join(pthread_test_root, t) for t in pthread_tests] + return pthread_tests + + +engine = config.NODE_JS + ['--experimental-wasm-threads', '--experimental-wasm-bulk-memory'] + +# Mark certain tests as not passing +disabled = { + 'test_pthread_create_11_1': 'never returns', + 'test_pthread_barrier_wait_2_1': 'never returns', + 'test_pthread_cond_timedwait_2_6': 'never returns', + 'test_pthread_cond_timedwait_4_3': 'never returns', + 'test_pthread_attr_setscope_5_1': 'internally skipped (PTS_UNTESTED)', + 'test_pthread_cond_wait_2_3': 'never returns', + 'test_pthread_create_5_1': 'never returns', + 'test_pthread_exit_1_2': 'never returns', + 'test_pthread_exit_2_2': 'never returns', + 'test_pthread_exit_3_2': 'never returns', + 'test_pthread_exit_4_1': 'never returns', + 'test_pthread_getcpuclockid_1_1': 'never returns', + 'test_pthread_key_create_1_2': 'never returns', + 'test_pthread_rwlock_rdlock_1_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedrdlock_1_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedrdlock_3_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedrdlock_5_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedwrlock_1_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedwrlock_3_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_timedwrlock_5_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_wrlock_1_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_rwlock_trywrlock_1_1': 'fails with "main: Unexpected thread state"', + 'test_pthread_spin_destroy_3_1': 'never returns', + 'test_pthread_spin_init_4_1': 'never returns', +} + + +def make_test(name, testfile, browser): + + def f(self): + if name in disabled: + self.skipTest(disabled[name]) + args = ['-I' + os.path.join(testsuite_root, 'include'), + '-Werror', + '-Wno-format-security', + '-Wno-int-conversion', + '-sUSE_PTHREADS', + '-sEXIT_RUNTIME', + '-sTOTAL_MEMORY=268435456', + '-sPTHREAD_POOL_SIZE=40'] + if browser: + # Only are only needed for browser tests of the was btest + # injects headers using `-include` flag. + args += ['-Wno-macro-redefined', '-D_GNU_SOURCE'] + self.btest(testfile, args=args, expected='exit:0') + else: + self.run_process([EMCC, testfile, '-o', 'test.js'] + args) + self.run_js('test.js', engine=engine) + + return f + + +for testdir in get_pthread_tests(): + basename = os.path.basename(testdir) + for test_file in glob.glob(os.path.join(testdir, '*.c')): + if not os.path.basename(test_file)[0].isdigit(): + continue + test_suffix = os.path.splitext(os.path.basename(test_file))[0] + test_suffix = test_suffix.replace('-', '_') + test_name = 'test_' + basename + '_' + test_suffix + setattr(posixtest, test_name, make_test(test_name, test_file, browser=False)) + setattr(test_posixtest_browser.posixtest_browser, test_name, make_test(test_name, test_file, browser=True)) diff --git a/tests/test_posixtest_browser.py b/tests/test_posixtest_browser.py new file mode 100644 index 0000000000000..02730cba615ae --- /dev/null +++ b/tests/test_posixtest_browser.py @@ -0,0 +1,18 @@ +"""See test_posixtest.py + +This class is only in its own file so that we can +run the non-browser versions in parallel. + +Currently only entire modules can be marked as +parallel. +""" + +from runner import BrowserCore + + +class posixtest_browser(BrowserCore): + """Run the suite in the browser (serially) + + This class get populated dynamically below. + """ + pass diff --git a/tests/third_party/posixtestsuite b/tests/third_party/posixtestsuite new file mode 160000 index 0000000000000..26372421f53ae --- /dev/null +++ b/tests/third_party/posixtestsuite @@ -0,0 +1 @@ +Subproject commit 26372421f53aeeeeeb4b23561c417886f1930ef6