Add multithread support to ASan (emscripten-core#9076)

Author: quantum5

emcc.py

@@ -1544,9 +1544,6 @@ def check(input_file):
           # by SAFE_HEAP as a null pointer dereference.
           exit_with_error('ASan does not work with SAFE_HEAP')
 
-        if shared.Settings.USE_PTHREADS:
-          exit_with_error('ASan currently does not support threads')
-
       if sanitize and '-g4' in args:
         shared.Settings.LOAD_SOURCE_MAP = 1
 

src/library_pthread.js

@@ -507,8 +507,8 @@ var LibraryPThread = {
     {{{ makeSetValue('__num_logical_cores', 0, 'cores', 'i32') }}};
   },
 
-  {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_create__deps: ['_spawn_thread', 'pthread_getschedparam', 'pthread_self', 'memalign'],
-  {{{ USE_LSAN ? 'emscripten_builtin_' : '' }}}pthread_create: function(pthread_ptr, attr, start_routine, arg) {
+  {{{ USE_LSAN || USE_ASAN ? 'emscripten_builtin_' : '' }}}pthread_create__deps: ['_spawn_thread', 'pthread_getschedparam', 'pthread_self', 'memalign'],
+  {{{ USE_LSAN || USE_ASAN ? 'emscripten_builtin_' : '' }}}pthread_create: function(pthread_ptr, attr, start_routine, arg) {
     if (typeof SharedArrayBuffer === 'undefined') {
       err('Current environment does not support SharedArrayBuffer, pthreads are not available!');
       return {{{ cDefine('EAGAIN') }}};

system/lib/compiler-rt/lib/asan/asan_emscripten.cc

@@ -1,6 +1,8 @@
+#include "asan_interceptors.h"
 #include "asan_internal.h"
 #include "asan_mapping.h"
 #include "asan_poisoning.h"
+#include "asan_stack.h"
 #include "asan_thread.h"
 
 #if SANITIZER_EMSCRIPTEN
@@ -29,6 +31,74 @@ void *AsanDoesNotSupportStaticLinkage() {
 
 void InitializeAsanInterceptors() {}
 
+// We can use a plain thread_local variable for TSD.
+static thread_local void *per_thread;
+
+void *AsanTSDGet() { return per_thread; }
+
+void AsanTSDSet(void *tsd) { per_thread = tsd; }
+
+// There's no initialization needed, and the passed-in destructor
+// will never be called.  Instead, our own thread destruction hook
+// (below) will call AsanThread::TSDDtor directly.
+void AsanTSDInit(void (*destructor)(void *tsd)) {
+  DCHECK(destructor == &PlatformTSDDtor);
+}
+
+void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); }
+
+extern "C" {
+  void *emscripten_builtin_malloc(size_t size);
+  void emscripten_builtin_free(void *memory);
+  int emscripten_builtin_pthread_create(void *thread, void *attr,
+                                        void *(*callback)(void *), void *arg);
+  int pthread_attr_getdetachstate(void *attr, int *detachstate);
+}
+
+static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
+  atomic_uintptr_t *param = reinterpret_cast<atomic_uintptr_t *>(arg);
+  AsanThread *t = nullptr;
+  while ((t = reinterpret_cast<AsanThread *>(
+              atomic_load(param, memory_order_acquire))) == nullptr)
+    internal_sched_yield();
+  emscripten_builtin_free(param);
+  SetCurrentThread(t);
+  return t->ThreadStart(GetTid(), nullptr);
+}
+
+INTERCEPTOR(int, pthread_create, void *thread,
+    void *attr, void *(*start_routine)(void*), void *arg) {
+  EnsureMainThreadIDIsCorrect();
+  // Strict init-order checking is thread-hostile.
+  if (flags()->strict_init_order)
+    StopInitOrderChecking();
+  GET_STACK_TRACE_THREAD;
+  int detached = 0;
+  if (attr)
+    pthread_attr_getdetachstate(attr, &detached);
+  atomic_uintptr_t *param = (atomic_uintptr_t *)
+      emscripten_builtin_malloc(sizeof(atomic_uintptr_t));
+  atomic_store(param, 0, memory_order_relaxed);
+  int result;
+  {
+    // Ignore all allocations made by pthread_create: thread stack/TLS may be
+    // stored by pthread for future reuse even after thread destruction, and
+    // the linked list it's stored in doesn't even hold valid pointers to the
+    // objects, the latter are calculated by obscure pointer arithmetic.
+#if CAN_SANITIZE_LEAKS
+    __lsan::ScopedInterceptorDisabler disabler;
+#endif
+    result = REAL(pthread_create)(thread, attr, asan_thread_start, param);
+  }
+  if (result == 0) {
+    u32 current_tid = GetCurrentTidOrInvalid();
+    AsanThread *t =
+        AsanThread::Create(start_routine, arg, current_tid, &stack, detached);
+    atomic_store(param, reinterpret_cast<uptr>(t), memory_order_release);
+  }
+  return result;
+}
+
 } // namespace __asan
 
 namespace __lsan {

system/lib/compiler-rt/lib/asan/asan_posix.cc

@@ -84,7 +84,7 @@ void PlatformTSDDtor(void *tsd) {
   atomic_signal_fence(memory_order_seq_cst);
   AsanThread::TSDDtor(tsd);
 }
-#else
+#elif !SANITIZER_EMSCRIPTEN
 static pthread_key_t tsd_key;
 static bool tsd_key_inited = false;
 void AsanTSDInit(void (*destructor)(void *tsd)) {

system/lib/compiler-rt/lib/asan/asan_thread.h

@@ -30,8 +30,10 @@ struct DTLS;
 namespace __asan {
 
 const u32 kInvalidTid = 0xffffff;  // Must fit into 24 bits.
-#if SANITIZER_EMSCRIPTEN
+#if SANITIZER_EMSCRIPTEN && !defined(USE_THREADS)
 const u32 kMaxNumberOfThreads = 1;
+#elif SANITIZER_EMSCRIPTEN
+const u32 kMaxNumberOfThreads = 128;
 #else
 const u32 kMaxNumberOfThreads = (1 << 22);  // 4M
 #endif

system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cc

@@ -172,4 +172,11 @@ void ProcessThreads(SuspendedThreadsList const &suspended_threads,
 
 } // namespace __lsan
 
+extern "C" void __lsan_disable_in_this_thread() {
+  __lsan::DisableInThisThread();
+}
+
+extern "C" void __lsan_enable_in_this_thread() {
+  __lsan::EnableInThisThread();
+}
 #endif // CAN_SANITIZE_LEAKS && SANITIZER_EMSCRIPTEN

system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cc

@@ -434,7 +434,6 @@ uptr internal_rename(const char *oldpath, const char *newpath) {
 
 uptr internal_sched_yield() {
 #if SANITIZER_EMSCRIPTEN
-  Report("WARNING: sched_yield doesn't do anything on emscripten");
   return 0;
 #else
   return internal_syscall(SYSCALL(sched_yield));

system/lib/libc/musl/src/misc/emscripten_pthread.c

@@ -24,8 +24,10 @@ EM_JS(void, initPthreadsJS, (void), {
   PThread.initRuntime();
 })
 
+// This must run before any userland ctors
+// Note that ASan constructor priority is 50, and we must be higher.
 EMSCRIPTEN_KEEPALIVE
-__attribute__((constructor(99))) // This must run before any userland ctors
+__attribute__((constructor(48)))
 void __emscripten_pthread_data_constructor(void) {
   initPthreadsJS();
   pthread_self()->locale = &libc.global_locale;

system/lib/pthread/library_pthread.c

@@ -188,14 +188,37 @@ void emscripten_async_waitable_close(em_queued_call* call) { em_queued_call_free
 
 extern double emscripten_receive_on_main_thread_js(int, int, double*);
 
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define HAS_ASAN
+void __lsan_disable_in_this_thread(void);
+void __lsan_enable_in_this_thread(void);
+int emscripten_builtin_pthread_create(void *thread, void *attr,
+                                      void *(*callback)(void *), void *arg);
+#endif
+#endif
+
 static void _do_call(em_queued_call* q) {
   if (!q->js) {
     // C function pointer
     assert(EM_FUNC_SIG_NUM_FUNC_ARGUMENTS(q->functionEnum) <= EM_QUEUED_CALL_MAX_ARGS);
     switch (q->functionEnum) {
       case EM_PROXIED_PTHREAD_CREATE:
+#ifdef HAS_ASAN
+        // ASan wraps the emscripten_builtin_pthread_create call in __lsan::ScopedInterceptorDisabler.
+        // Unfortunately, that only disables it on the thread that made the call.
+        // This is sufficient on the main thread.
+        // On non-main threads, pthread_create gets proxied to the main thread, where LSan is not
+        // disabled. This makes it necessary for us to disable LSan here, so that it does not detect
+        // pthread's internal allocations as leaks.
+        __lsan_disable_in_this_thread();
+        q->returnValue.i =
+          emscripten_builtin_pthread_create(q->args[0].vp, q->args[1].vp, q->args[2].vp, q->args[3].vp);
+        __lsan_enable_in_this_thread();
+#else
         q->returnValue.i =
           pthread_create(q->args[0].vp, q->args[1].vp, q->args[2].vp, q->args[3].vp);
+#endif
         break;
       case EM_PROXIED_SYSCALL:
         q->returnValue.i = emscripten_syscall(q->args[0].i, q->args[1].vp);

system/lib/pthread/library_pthread_wasm.c

@@ -173,7 +173,8 @@ extern void __wasm_init_tls(void *memory);
 void *emscripten_builtin_memalign(size_t align, size_t size);
 void emscripten_builtin_free(void *memory);
 
-__attribute__((constructor(100)))
+// Note that ASan constructor priority is 50, and we must be higher.
+__attribute__((constructor(49)))
 void EMSCRIPTEN_KEEPALIVE emscripten_tls_init(void) {
   size_t tls_size = __builtin_wasm_tls_size();
   size_t tls_align = __builtin_wasm_tls_align();

tests/pthread/test_pthread_asan_use_after_free.cpp

@@ -0,0 +1,16 @@
+#include <cstdio>
+#include <thread>
+
+std::atomic<bool> thread_done;
+
+void f(int *a) {
+  delete [] a;
+  a[0] = 1;
+  std::atomic_store(&thread_done, true);
+}
+
+int main(int argc, char **argv) {
+  std::thread t(f, new int[10]);
+  t.detach();
+  while (!std::atomic_load(&thread_done));
+}

tests/pthread/test_pthread_asan_use_after_free.js

@@ -0,0 +1,25 @@
+(function () {
+  var output = [];
+  Module['printErr'] = function (text) {
+    if (text == '==42==ABORTING') {
+      var result = output.join('\n');
+      var passed = [
+        'ERROR: AddressSanitizer: heap-use-after-free on address',
+        'WRITE of size 4',
+        'is located 0 bytes inside of 40-byte region',
+        'freed by thread T2 here:',
+        'previously allocated by thread T1 here:',
+        'Thread T2 created by T1 here:',
+        'SUMMARY: AddressSanitizer: heap-use-after-free',
+        'Shadow bytes around the buggy address:',
+        'Shadow byte legend (one shadow byte represents 8 application bytes):',
+      ].every(function (snippet) {
+        return result.indexOf(snippet) >= 0;
+      });
+      reportResultToServer(passed ? 1 : 0);
+      return;
+    }
+    output.push(text);
+    console.log(text);
+  };
+})();

tests/pthread/test_pthread_lsan_leak.js

@@ -17,7 +17,7 @@
         'Direct leak of 42 byte(s) in 1 object(s) allocated from',
         'test_pthread_lsan_leak.cpp:12:21',
         'test_pthread_lsan_leak.cpp:38:3',
-        'SUMMARY: LeakSanitizer: 8513 byte(s) leaked in 6 allocation(s).',
+        '8513 byte(s) leaked in 6 allocation(s).',
         ''
       ].every(function (snippet) {
         return result.indexOf(snippet) >= 0;

tests/test_browser.py

@@ -3916,6 +3916,20 @@ def test_pthread_safe_stack(self):
   def test_pthread_lsan(self, name, args=[]):
     self.btest(path_from_root('tests', 'pthread', name + '.cpp'), expected='1', args=['-fsanitize=leak', '-s', 'TOTAL_MEMORY=256MB', '-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-std=c++11', '--pre-js', path_from_root('tests', 'pthread', name + '.js')] + args)
 
+  @parameterized({
+    # Reusing the LSan test files for ASan.
+    'leak': ['test_pthread_lsan_leak', ['-g4']],
+    'no_leak': ['test_pthread_lsan_no_leak'],
+  })
+  @no_fastcomp('ASan is only supported on WASM backend')
+  @requires_threads
+  def test_pthread_asan(self, name, args=[]):
+    self.btest(path_from_root('tests', 'pthread', name + '.cpp'), expected='1', args=['-fsanitize=address', '-s', 'TOTAL_MEMORY=256MB', '-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-std=c++11', '--pre-js', path_from_root('tests', 'pthread', name + '.js')] + args)
+
+  @no_fastcomp('ASan is only supported on WASM backend')
+  def test_pthread_asan_use_after_free(self):
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_asan_use_after_free.cpp'), expected='1', args=['-fsanitize=address', '-s', 'TOTAL_MEMORY=256MB', '-s', 'USE_PTHREADS', '-s', 'PROXY_TO_PTHREAD', '-std=c++11', '--pre-js', path_from_root('tests', 'pthread', 'test_pthread_asan_use_after_free.js')])
+
   # Tests MAIN_THREAD_EM_ASM_INT() function call signatures.
   @no_wasm_backend('MAIN_THREAD_EM_ASM() not yet implemented in Wasm backend')
   def test_main_thread_em_asm_signatures(self):

tools/system_libs.py

@@ -528,11 +528,6 @@ def vary_on(cls):
       vary_on += ['is_asan']
     return vary_on
 
-  @classmethod
-  def variations(cls):
-    return [variation for variation in super(AsanInstrumentedLibrary, cls).variations()
-            if not shared.Settings.WASM_BACKEND or not variation['is_asan'] or not variation.get('is_mt')]
-
   @classmethod
   def get_default_variation(cls, **kwargs):
     return super(AsanInstrumentedLibrary, cls).get_default_variation(is_asan=shared.Settings.USE_ASAN, **kwargs)