Benchmark fixes and improvements (#10870)

We had an annoying little bug where we set CFLAGS so that
feature flags work, but some build systems don't append that -
they just replace all their defaults with it. As a result, on the zlib
test we were measuring unoptimized code (lol) which was
over 30% larger and almost half the speed it should be...

Add an LTO mode for emscripten. LTO sometimes helps by a
small amount, but it's mixed, and can also hurt by a little bit too
on code size, due to much more inlining. But in some cases it
helps a lot, like 25% smaller code in fasta, 8% smaller in coremark,
and a 25% speedup on skinning.

Run v8 without liftoff. The baseline compiler can add a lot of
noise since how much time we spend there ends up mattering
a lot (the wasm runs at half speed until the full compiler is done,
and that's noisy).

Also add nicer code for error handling in the zlib benchmark.

Author: kripken

tests/benchmark/test_zlib_benchmark.c

@@ -29,7 +29,13 @@ void __attribute__ ((noinline)) doit(unsigned char *buffer, int size, int i) {
   unsigned long decompressedSize = size;
   uncompress(buffer3, &decompressedSize, buffer2, (int)compressedSize);
   assert(decompressedSize == size);
-  if (i == 0) assert(strcmp((char*)buffer, (char*)buffer3) == 0);
+  if (i == 0) {
+    if (strcmp((char*)buffer, (char*)buffer3) != 0) {
+      puts("incorrect output!");
+      abort();
+    }
+    puts("output looks good");
+  }
 }
 
 int main(int argc, char **argv) {

tests/test_benchmark.py

@@ -191,7 +191,10 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat
     llvm_root = self.env.get('LLVM') or LLVM_ROOT
     if lib_builder:
       env_init = self.env.copy()
-      env_init['CFLAGS'] = ' '.join(LLVM_FEATURE_FLAGS)
+      # Note that we need to pass in all the flags here because some build
+      # systems (like zlib) if they see a CFLAGS it will override all their
+      # default flags, including optimizations.
+      env_init['CFLAGS'] = ' '.join(LLVM_FEATURE_FLAGS + [OPTIMIZATIONS] + self.extra_args)
       emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=env_init)
     final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
     final = final.replace('.cpp', '')
@@ -313,22 +316,35 @@ def cleanup(self):
     # NativeBenchmarker('clang', CLANG_CC, CLANG),
     # NativeBenchmarker('gcc',   'gcc',    'g++')
   ]
-if SPIDERMONKEY_ENGINE and SPIDERMONKEY_ENGINE in shared.JS_ENGINES:
-  benchmarkers += [
-    # EmscriptenBenchmarker('sm', SPIDERMONKEY_ENGINE),
-  ]
+
 if V8_ENGINE and V8_ENGINE in shared.JS_ENGINES:
+  # avoid the baseline compiler running, because it adds a lot of noise
+  # (the nondeterministic time it takes to get to the full compiler ends up
+  # mattering as much as the actual benchmark)
+  aot_v8 = V8_ENGINE + ['--no-liftoff']
+  default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8'
   benchmarkers += [
-    EmscriptenBenchmarker(os.environ.get('EMBENCH_NAME') or 'v8', V8_ENGINE),
+    EmscriptenBenchmarker(default_v8_name, aot_v8),
+    EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']),
   ]
-if shared.NODE_JS and shared.NODE_JS in shared.JS_ENGINES:
+  if os.path.exists(CHEERP_BIN):
+    benchmarkers += [
+      # CheerpBenchmarker('cheerp-v8-wasm', aot_v8),
+    ]
+
+if SPIDERMONKEY_ENGINE and SPIDERMONKEY_ENGINE in shared.JS_ENGINES:
+  # TODO: ensure no baseline compiler is used, see v8
   benchmarkers += [
-    EmscriptenBenchmarker('Node.js', shared.NODE_JS),
+    # EmscriptenBenchmarker('sm', SPIDERMONKEY_ENGINE),
   ]
-if os.path.exists(CHEERP_BIN):
+  if os.path.exists(CHEERP_BIN):
+    benchmarkers += [
+      # CheerpBenchmarker('cheerp-sm-wasm', SPIDERMONKEY_ENGINE),
+    ]
+
+if shared.NODE_JS and shared.NODE_JS in shared.JS_ENGINES:
   benchmarkers += [
-    # CheerpBenchmarker('cheerp-sm-wasm', SPIDERMONKEY_ENGINE),
-    # CheerpBenchmarker('cheerp-v8-wasm', V8_ENGINE),
+    # EmscriptenBenchmarker('Node.js', shared.NODE_JS),
   ]