Support some synchronous Fetch API calls with the wasm backend, without a fetch-worker (#9490)

@dschuff looked into this and found that we can only use the fetch-worker to
help with synchronous fetches when we are in a worker, since the main thread
can't block on another one. But if we are in a worker, then we can just do a
sync xhr directly!

Thanks to that we can enable test_fetch_sync_xhr_in_wasm, so this helps #7024,
but it does not fix all the cases. There is still emscripten_fetch_wait, and I
believe some IDB operations may also not work. I noticed that some fetch test
coverage may be misleading here, as we skip browser tests that use WASM=0 with
pthreads (since wasm2js doesn't support that); I added an explicit skip to the
test for emscripten_fetch_wait to make things clearer.

This gets rid of a block of code starting with Should we try IndexedDB first?.
That code seems unnecessary: It checks if we will need IDB and then errors if
we will but it isn't ready. However, the actual operation later would error on
missing IDB anyhow, so this is redundant. Worse, it looks wrong to me, as it
says a simple fetch should use IDB but that doesn't seem right - in particular,
it made the test here fail. Regardless, just removing the code seems best.

Author: kripken

emcc.py

@@ -1265,6 +1265,16 @@ def is_supported_link_flag(f):
       if shared.Settings.USE_PTHREADS:
         shared.Settings.FETCH_WORKER_FILE = unsuffixed(os.path.basename(target)) + '.fetch.js'
 
+    if shared.Settings.FETCH:
+      # In asm.js+pthreads we can use a fetch worker, which is made from the main
+      # asm.js code. That lets us do sync operations by blocking on the worker etc.
+      # In the wasm backend we don't have a fetch worker implemented yet, however,
+      # we can still do basic synchronous fetches in the same places: if we can
+      # block on another thread then we aren't the main thread, and if we aren't
+      # the main thread then synchronous xhrs are legitimate.
+      if shared.Settings.USE_PTHREADS and not shared.Settings.WASM_BACKEND:
+        shared.Settings.USE_FETCH_WORKER = 1
+
     if shared.Settings.DEMANGLE_SUPPORT:
       shared.Settings.EXPORTED_FUNCTIONS += ['___cxa_demangle']
       forced_stdlibs.append('libc++abi')
@@ -2358,11 +2368,8 @@ def repl(m):
           f.write(shared.read_and_preprocess(shared.path_from_root('src', 'worker.js'), expand_macros=True))
 
       # Generate the fetch.js worker script for multithreaded emscripten_fetch() support if targeting pthreads.
-      if shared.Settings.FETCH and shared.Settings.USE_PTHREADS:
-        if shared.Settings.WASM_BACKEND:
-          logger.warning('Bug/TODO: Blocking calls to the fetch API do not currently work under WASM backend (https://github.com/emscripten-core/emscripten/issues/7024)')
-        else:
-          shared.make_fetch_worker(final, shared.Settings.FETCH_WORKER_FILE)
+      if shared.Settings.USE_FETCH_WORKER:
+        shared.make_fetch_worker(final, shared.Settings.FETCH_WORKER_FILE)
 
     # exit block 'memory initializer'
     log_time('memory initializer')

src/Fetch.js

@@ -42,7 +42,7 @@ var Fetch = {
   },
 #endif
 
-#if USE_PTHREADS
+#if USE_FETCH_WORKER
   initFetchWorker: function() {
     var stackSize = 128*1024;
     var stack = allocate(stackSize>>2, "i32*", ALLOC_DYNAMIC);
@@ -51,7 +51,7 @@ var Fetch = {
 #endif
 
   staticInit: function() {
-#if USE_PTHREADS
+#if USE_FETCH_WORKER
     var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined' && !ENVIRONMENT_IS_PTHREAD);
 #else
     var isMainThread = (typeof ENVIRONMENT_IS_FETCH_WORKER === 'undefined');
@@ -65,7 +65,7 @@ var Fetch = {
       Fetch.dbInstance = db;
 
       if (isMainThread) {
-#if USE_PTHREADS
+#if USE_FETCH_WORKER
         Fetch.initFetchWorker();
 #endif
         removeRunDependency('library_fetch_init');
@@ -78,7 +78,7 @@ var Fetch = {
       Fetch.dbInstance = false;
 
       if (isMainThread) {
-#if USE_PTHREADS
+#if USE_FETCH_WORKER
         Fetch.initFetchWorker();
 #endif
         removeRunDependency('library_fetch_init');
@@ -87,7 +87,7 @@ var Fetch = {
     Fetch.openDatabase('emscripten_filesystem', 1, onsuccess, onerror);
 #endif // ~FETCH_SUPPORT_INDEXEDDB
 
-#if USE_PTHREADS
+#if USE_FETCH_WORKER
     if (isMainThread) {
 #if FETCH_SUPPORT_INDEXEDDB
       addRunDependency('library_fetch_init');
@@ -543,16 +543,6 @@ function emscripten_start_fetch(fetch, successcb, errorcb, progresscb, readystat
     __emscripten_fetch_xhr(fetch, cacheResultAndReportSuccess, reportError, reportProgress, reportReadyStateChange);
   };
 
-  // Should we try IndexedDB first?
-  var needsIndexedDbConnection = !fetchAttrReplace || requestMethod === 'EM_IDB_STORE' || requestMethod === 'EM_IDB_DELETE';
-  if (needsIndexedDbConnection && !Fetch.dbInstance) {
-#if FETCH_DEBUG
-    console.error('fetch: failed to read IndexedDB! Database is not open.');
-#endif
-    reportError(fetch, 0, 'IndexedDB is not open');
-    return 0; // todo: free
-  }
-
   if (requestMethod === 'EM_IDB_STORE') {
     // TODO(?): Here we perform a clone of the data, because storing shared typed arrays to IndexedDB does not seem to be allowed.
     var ptr = HEAPU32[fetch_attr + {{{ C_STRUCTS.emscripten_fetch_attr_t.requestData }}} >> 2];

src/settings.js

@@ -1395,6 +1395,12 @@ var FETCH_DEBUG = 0;
 // If nonzero, enables emscripten_fetch API.
 var FETCH = 0;
 
+// Whether to use an asm.js fetch worker when using FETCH. Note that this is
+// only relevant for fastcomp, where we support asm.js. As a result, some
+// synchronous fetch operations that depend on the fetch worker may not work
+// with the wasm backend, like waiting or IndexedDB.
+var USE_FETCH_WORKER = 0;
+
 // Internal: name of the file containing the Fetch *.fetch.js, if relevant
 // Do not set yourself.
 var FETCH_WORKER_FILE = '';

system/lib/fetch/emscripten_fetch.cpp

@@ -148,7 +148,13 @@ emscripten_fetch_t* emscripten_fetch(emscripten_fetch_attr_t* fetch_attr, const
 
 #undef STRDUP_OR_ABORT
 
-#if __EMSCRIPTEN_PTHREADS__
+// In asm.js we can use a fetch worker, which is created from the main asm.js
+// code. That lets us do sync operations by blocking on the worker etc.
+// In the wasm backend we don't have a fetch worker implemented yet, however,
+// we can still do basic synchronous fetches in the same places: if we can
+// block on another thread then we aren't the main thread, and if we aren't
+// the main thread then synchronous xhrs are legitimate.
+#if __EMSCRIPTEN_PTHREADS__ && !defined(__wasm__)
   const bool waitable = (fetch_attr->attributes & EMSCRIPTEN_FETCH_WAITABLE) != 0;
   // Depending on the type of fetch, we can either perform it in the same Worker/thread than the
   // caller, or we might need to run it in a separate Worker. There is a dedicated fetch worker that

tests/test_browser.py

@@ -4349,10 +4349,10 @@ def test_fetch_implicit_append(self):
     self.btest('fetch/example_synchronous_fetch.cpp', expected='200', args=['-s', 'FETCH=1', '-s', 'WASM=0', '-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1'])
 
   # Tests synchronous emscripten_fetch() usage from wasm pthread in fastcomp.
-  @no_wasm_backend("fetch API uses an asm.js based web worker to run synchronous XHRs and IDB operations")
+  @requires_threads
   def test_fetch_sync_xhr_in_wasm(self):
     shutil.copyfile(path_from_root('tests', 'gears.png'), 'gears.png')
-    self.btest('fetch/example_synchronous_fetch.cpp', expected='200', args=['-s', 'FETCH=1', '-s', 'WASM=1', '-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1'])
+    self.btest('fetch/example_synchronous_fetch.cpp', expected='200', args=['-s', 'FETCH=1', '-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1'])
 
   # Tests that the Fetch API works for synchronous XHRs when used with --proxy-to-worker.
   @requires_threads
@@ -4364,6 +4364,7 @@ def test_fetch_sync_xhr_in_proxy_to_worker(self):
                also_asmjs=True)
 
   # Tests waiting on EMSCRIPTEN_FETCH_WAITABLE request from a worker thread
+  @no_wasm_backend("emscripten_fetch_wait uses an asm.js based web worker")
   @requires_threads
   def test_fetch_sync_fetch_in_main_thread(self):
     shutil.copyfile(path_from_root('tests', 'gears.png'), 'gears.png')