emscripten-core · juj · Jun 3, 2015 · Sep 5, 2014 · Sep 6, 2014 · Sep 6, 2014
diff --git a/emcc b/emcc
@@ -1000,6 +1000,28 @@ try:
   if js_opts:
     shared.Settings.RUNNING_JS_OPTS = 1
 
+  if shared.Settings.USE_PTHREADS:
+    if not any(s.startswith('PTHREAD_POOL_SIZE=') for s in settings_changes):
+      settings_changes.append('PTHREAD_POOL_SIZE=0')
+    js_libraries.append(shared.path_from_root('src', 'library_pthread.js'))
+    newargs.append('-D__EMSCRIPTEN_PTHREADS__=1')
+  else:
+    js_libraries.append(shared.path_from_root('src', 'library_pthread_stub.js'))
+
+  if shared.Settings.USE_PTHREADS:
+    if shared.Settings.PROXY_TO_WORKER:
+      logging.error('-s PROXY_TO_WORKER=1 is not yet supported with -s USE_PTHREADS=1!')
+      exit(1)
+    if shared.Settings.LINKABLE:
+      logging.error('-s LINKABLE=1 is not supported with -s USE_PTHREADS=1!')
+      exit(1)
+    if shared.Settings.SIDE_MODULE:
+      logging.error('-s SIDE_MODULE=1 is not supported with -s USE_PTHREADS=1!')
+      exit(1)
+    if shared.Settings.MAIN_MODULE:
+      logging.error('-s MAIN_MODULE=1 is not supported with -s USE_PTHREADS=1!')
+      exit(1)
+
   shared.Settings.EMSCRIPTEN_VERSION = shared.EMSCRIPTEN_VERSION
   shared.Settings.OPT_LEVEL = opt_level
   shared.Settings.DEBUG_LEVEL = debug_level
@@ -1351,6 +1373,9 @@ try:
       final += '.mem.js'
       src = None
 
+  if shared.Settings.USE_PTHREADS:
+    shutil.copyfile(shared.path_from_root('src', 'pthread-main.js'), os.path.join(os.path.dirname(os.path.abspath(target)), 'pthread-main.js'))
+
   log_time('source transforms')
 
   # It is useful to run several js optimizer passes together, to save on unneeded unparsing/reparsing
@@ -1606,7 +1631,7 @@ try:
 <script>
   if ((',' + window.location.search.substr(1) + ',').indexOf(',noProxy,') < 0) {
     console.log('running code in a web worker');
-''' + open(shared.path_from_root('src', 'webGLClient.js')).read() + '\n' + open(shared.path_from_root('src', 'proxyClient.js')).read().replace('{{{ filename }}}', child_js).replace('{{{ IDBStore.js }}}', open(shared.path_from_root('src', 'IDBStore.js')).read()) + '''
+''' + open(shared.path_from_root('src', 'webGLClient.js')).read() + '\n' + shared.read_and_preprocess(shared.path_from_root('src', 'proxyClient.js')).replace('{{{ filename }}}', child_js).replace('{{{ IDBStore.js }}}', open(shared.path_from_root('src', 'IDBStore.js')).read()) + '''
   } else {
     // note: no support for code mods (PRECISE_F32==2)
     console.log('running code on the main thread');

diff --git a/emscripten.py b/emscripten.py
@@ -65,6 +65,8 @@ def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
     backend_args = [backend_compiler, infile, '-march=js', '-filetype=asm', '-o', temp_js]
     if settings['PRECISE_F32']:
       backend_args += ['-emscripten-precise-f32']
+    if settings['USE_PTHREADS']:
+      backend_args += ['-emscripten-enable-pthreads']
     if settings['WARN_UNALIGNED']:
       backend_args += ['-emscripten-warn-unaligned']
     if settings['RESERVED_FUNCTION_POINTERS'] > 0:
@@ -220,8 +222,8 @@ def save_settings():
     staticbump = mem_init.count(',')+1
     while staticbump % 16 != 0: staticbump += 1
     pre = pre.replace('STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d;
-  /* global initializers */ __ATINIT__.push(%s);
-  %s''' % (staticbump, global_initializers, mem_init)) # XXX wrong size calculation!
+  /* global initializers */ %s __ATINIT__.push(%s);
+  %s''' % (staticbump, 'if (!ENVIRONMENT_IS_PTHREAD)' if settings['USE_PTHREADS'] else '', global_initializers, mem_init)) # XXX wrong size calculation!
 
     if settings['SIDE_MODULE']:
       pre = pre.replace('Runtime.GLOBAL_BASE', 'gb').replace('{{{ STATIC_BUMP }}}', str(staticbump))
@@ -460,7 +462,12 @@ def make_emulated_param(i):
                                 'shiftRightArithmeticByScalar',
                                 'shiftRightLogicalByScalar',
                                 'shiftLeftByScalar'];
-    fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array', 'NaN', 'Infinity']
+    fundamentals = ['Math']
+    if settings['USE_PTHREADS']:
+      fundamentals += ['SharedInt8Array', 'SharedInt16Array', 'SharedInt32Array', 'SharedUint8Array', 'SharedUint16Array', 'SharedUint32Array', 'SharedFloat32Array', 'SharedFloat64Array', 'Atomics']
+    else:
+      fundamentals += ['Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array']
+    fundamentals += ['NaN', 'Infinity']
     if metadata['simd']:
         fundamentals += ['SIMD']
     if settings['ALLOW_MEMORY_GROWTH']: fundamentals.append('byteLength')
@@ -523,7 +530,7 @@ def keyfunc(other):
       if not settings['SIDE_MODULE']:
         asm_setup += 'var gb = Runtime.GLOBAL_BASE, fb = 0;\n'
 
-    asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew']
+    asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace', 'setThrew']
     if not settings['RELOCATABLE']:
       asm_runtime_funcs += ['setTempRet0', 'getTempRet0']
     else:
@@ -649,6 +656,10 @@ def math_fix(g):
       asm_global_funcs += ''.join(['  var SIMD_' + ty + '=global' + access_quote('SIMD') + access_quote(ty) + ';\n' for ty in simdtypes])
       asm_global_funcs += ''.join(['  var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdinttypes for g in simdintfuncs])
       asm_global_funcs += ''.join(['  var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdfloattypes for g in simdfloatfuncs])
+    if settings['USE_PTHREADS']:
+#      asm_global_funcs += ''.join(['  var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'exchange', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']])
+# TODO: Once bug https://bugzilla.mozilla.org/show_bug.cgi?id=1141986 is implemented, replace the following line with the above one!
+      asm_global_funcs += ''.join(['  var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']])
     asm_global_vars = ''.join(['  var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars])
 
     # sent data
@@ -724,14 +735,15 @@ def math_fix(g):
   var HEAPU32 = new global%s(buffer);
   var HEAPF32 = new global%s(buffer);
   var HEAPF64 = new global%s(buffer);
-''' % (access_quote('Int8Array'),
-     access_quote('Int16Array'),
-     access_quote('Int32Array'),
-     access_quote('Uint8Array'),
-     access_quote('Uint16Array'),
-     access_quote('Uint32Array'),
-     access_quote('Float32Array'),
-     access_quote('Float64Array')) if not settings['ALLOW_MEMORY_GROWTH'] else '''
+''' % (access_quote('SharedInt8Array' if settings['USE_PTHREADS'] else 'Int8Array'),
+     access_quote('SharedInt16Array' if settings['USE_PTHREADS'] else 'Int16Array'),
+     access_quote('SharedInt32Array' if settings['USE_PTHREADS'] else 'Int32Array'),
+     access_quote('SharedUint8Array' if settings['USE_PTHREADS'] else 'Uint8Array'),
+     access_quote('SharedUint16Array' if settings['USE_PTHREADS'] else 'Uint16Array'),
+     access_quote('SharedUint32Array' if settings['USE_PTHREADS'] else 'Uint32Array'),
+     access_quote('SharedFloat32Array' if settings['USE_PTHREADS'] else 'Float32Array'),
+     access_quote('SharedFloat64Array' if settings['USE_PTHREADS'] else 'Float64Array'))
+     if not settings['ALLOW_MEMORY_GROWTH'] else '''
   var Int8View = global%s;
   var Int16View = global%s;
   var Int32View = global%s;
@@ -800,6 +812,12 @@ def math_fix(g):
   top = top|0;
   STACKTOP = top;
 }
+function establishStackSpace(stackBase, stackMax) {
+  stackBase = stackBase|0;
+  stackMax = stackMax|0;
+  STACKTOP = stackBase;
+  STACK_MAX = stackMax;
+}
 ''' + ('''
 function setAsync() {
   ___async = 1;
@@ -869,6 +887,7 @@ def math_fix(g):
 Runtime.stackAlloc = asm['stackAlloc'];
 Runtime.stackSave = asm['stackSave'];
 Runtime.stackRestore = asm['stackRestore'];
+Runtime.establishStackSpace = asm['establishStackSpace'];
 ''')
     if not settings['RELOCATABLE']:
       funcs_js.append('''

diff --git a/site/source/docs/porting/pthreads.rst b/site/source/docs/porting/pthreads.rst
@@ -0,0 +1,52 @@
+.. Pthreads support:
+
+==============================
+Pthreads support
+==============================
+
+Low level multithreading is currently supported in Firefox Nightly release channel via an experimental extension to JavaScript Web Workers. This is a work-in-progress research project, and the prototype specification can be located `here <https://docs.google.com/document/d/1NDGA_gZJ7M7w1Bh8S0AoDyEqwDdRh4uSoTPSNn77PFk/edit?usp=sharing>`.
+
+The proposed specification allows Emscripten applications to share the main memory heap between web workers. This along with primitives for low level atomics and futex support enables Emscripten to implement support for the Pthreads (POSIX threads) API.
+
+Compiling with pthreads enabled
+===============================
+
+By default, support for pthreads is not enabled, since the specification is still in a prototyping stage. To enable code generation for pthreads, the following command line flags exist:
+
+- Pass the compiler flag -s USE_PTHREADS=1 when compiling any .c/.cpp files, AND when linking to generate the final output .js file.
+- Optionally, pass the linker flag -s PTHREAD_POOL_SIZE=<integer> to specify a predefined pool of web workers to populate at page preRun time before application main() is called. If -1 is passed to both PTHREAD_POOL_SIZE and PTHREAD_HINT_NUM_CORES, then a popup dialog will ask the user the size of the pool (useful for testing).
+- Optionally, pass the linker flag -s PTHREAD_HINT_NUM_CORES=<integer> to choose what the function emscripten_num_logical_cores(); will return if navigator.hardwareConcurrency is not supported. If -1 is specified here, a popup dialog will be shown at startup to let the user specify the value that is returned here. This can be helpful in order to dynamically test how an application behaves with different values here. 
+
+There should be no other changes required. In C/C++ code, the preprocessor check #ifdef __EMSCRIPTEN_PTHREADS__ can be used to detect whether Emscripten is currently targeting pthreads.
+
+Special considerations
+======================
+
+The Emscripten implementation for the pthreads API should follow the POSIX standard closely, but some behavioral differences do exist:
+
+- When -s PTHREAD_POOL_SIZE=<integer> is not specified and pthread_create() is called, the new thread will not actually start to run immediately, but the main JS thread must yield execution back to browser first. This behavior is a result of `#1049079 <https://bugzilla.mozilla.org/show_bug.cgi?id=1049079>`.
+
+- Currently several of the functions in the C runtime, such as filesystem functions like fopen(), fread(), printf(), fprintf() etc. are not multithreaded, but instead their execution is proxied over to the main application thread. Memory allocation via malloc() and free() is fully multithreaded though.
+
+- The Emscripten implementation does not support `POSIX signals <http://man7.org/linux/man-pages/man7/signal.7.html>`, which are sometimes used in conjunction with pthreads. This is because it is not possible to send signals to web workers and pre-empt their execution. The only exception to this is pthread_kill() which can be used as normal to forcibly terminate a running thread.
+
+- The Emscripten implementation does also not support multiprocessing via fork() and join().
+
+- For web security purposes, there exists a fixed limit (by default 20) of threads that can be spawned when running in Firefox Nightly. `#1052398 <https://bugzilla.mozilla.org/show_bug.cgi?id=1052398>`. To adjust the limit, navigate to about:config and change the value of the pref "dom.workers.maxPerDomain".
+
+- Some of the features in the pthreads specification are unsupported since the upstream musl library that Emscripten utilizes does not support them, or they are marked optional and a conformant implementation need not support them. Such unsupported features in Emscripten include prioritization of threads, and pthread_rwlock_unlock() is not performed in thread priority order. The functions pthread_mutexattr_set/getprotocol(), pthread_mutexattr_set/getprioceiling() and pthread_attr_set/getscope() are no-ops.
+
+- One particular note to pay attention to when porting is that sometimes in existing codebases the callback function pointers to pthread_create() and pthread_cleanup_push() omit the void* argument, which strictly speaking is undefined behavior in C/C++, but works in several x86 calling conventions. Doing this in Emscripten will issue a compiler warning, and can abort at runtime when attempting to call a function pointer with incorrect signature, so in the presence of such errors, it is good to check the signatures of the thread callback functions.
+
+Also note that when compiling code that uses pthreads, an additional JavaScript file `pthread-main.js` is generated alongside the output .js file. That file must be deployed with the rest of the generated code files.
+
+Running code and tests
+======================
+
+Any code that is compiled with pthreads support enabled will currently only work in the Firefox Nightly channel, since the SharedArrayBuffer specification is still in an experimental research stage before standardization. There exists two test suites that can be used to verify the behavior of the pthreads API implementation in Emscripten:
+
+- The Emscripten unit test suite contains several pthreads-specific tests in the "browser." suite. Run any of the tests named browser.test_pthread_*.
+
+- An Emscripten-specialized version of the `Open POSIX Test Suite <http://posixtest.sourceforge.net/>` is available at `juj/posixtestsuite <https://github.com/juj/posixtestsuite>` GitHub repository. This suite contains about 300 tests for pthreads conformance. To run this suite, the pref dom.workers.maxPerDomain should first be increased to at least 50.
+
+Please check these first in case of any issues. Bugs can be reported to the Emscripten bug tracker as usual.
diff --git a/src/compiler.js b/src/compiler.js
@@ -194,7 +194,7 @@ if (!BOOTSTRAPPING_STRUCT_INFO) {
 load('modules.js');
 load('parseTools.js');
 load('jsifier.js');
-globalEval(processMacros(preprocess(read('runtime.js'))));
+globalEval(processMacros(preprocess(read('runtime.js'), 'runtime.js')));
 Runtime.QUANTUM_SIZE = QUANTUM_SIZE;
 
 //===============================

diff --git a/src/deps_info.json b/src/deps_info.json
@@ -32,6 +32,7 @@
   "eglGetProcAddress": ["emscripten_GetProcAddress"],
   "glfwGetProcAddress": ["emscripten_GetProcAddress"],
   "emscripten_GetProcAddress": ["strstr"],
-  "__cxa_begin_catch": ["__cxa_can_catch", "__cxa_is_pointer_type"]
+  "__cxa_begin_catch": ["__cxa_can_catch", "__cxa_is_pointer_type"],
+  "sleep": ["usleep"]
 }
 
diff --git a/src/emrun_postjs.js b/src/emrun_postjs.js
@@ -1,23 +1,25 @@
-function emrun_register_handlers() {
-  function post(msg) {
-    var http = new XMLHttpRequest();
-    http.open("POST", "stdio.html", true);
-    http.send(msg);
-  }
-  // If the address contains localhost, or we are running the page from port 6931, we can assume we're running the test runner and should post stdout logs.
-  if (document.URL.search("localhost") != -1 || document.URL.search(":6931/") != -1) {
-    var emrun_http_sequence_number = 1;
-    var prevPrint = Module['print'];
-    var prevErr = Module['printErr'];
-    function emrun_exit() { post('^exit^'+EXITSTATUS); };
-    Module['addOnExit'](emrun_exit);
-    Module['print'] = function emrun_print(text) { post('^out^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevPrint(text); }
-    Module['printErr'] = function emrun_printErr(text) { post('^err^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevErr(text); }
+if (typeof window === "object" && !ENVIRONMENT_IS_PTHREAD) {
+  function emrun_register_handlers() {
+    function post(msg) {
+      var http = new XMLHttpRequest();
+      http.open("POST", "stdio.html", true);
+      http.send(msg);
+    }
+    // If the address contains localhost, or we are running the page from port 6931, we can assume we're running the test runner and should post stdout logs.
+    if (document.URL.search("localhost") != -1 || document.URL.search(":6931/") != -1) {
+      var emrun_http_sequence_number = 1;
+      var prevPrint = Module['print'];
+      var prevErr = Module['printErr'];
+      function emrun_exit() { post('^exit^'+EXITSTATUS); };	
+      Module['addOnExit'](emrun_exit);
+      Module['print'] = function emrun_print(text) { post('^out^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevPrint(text); }
+      Module['printErr'] = function emrun_printErr(text) { post('^err^'+(emrun_http_sequence_number++)+'^'+encodeURIComponent(text)); prevErr(text); }
+
+      // Notify emrun web server that this browser has successfully launched the page.
+      post('^pageload^');
+    }
   }
-  // Notify emrun web server that this browser has successfully launched the page.
-  post('^pageload^');
 }
-window.addEventListener('load', emrun_register_handlers);
 
 // POSTs the given binary data represented as a (typed) array data back to the emrun-based web server.
 // To use from C code, call e.g. EM_ASM_({emrun_file_dump("file.dat", HEAPU8.subarray($0, $0 + $1));}, my_data_pointer, my_data_pointer_byte_length);