Improve job scheduling when running regression tests with -jN

Currently the regression test suite directories are listed and executed in
alphabetical order. However, because the runtimes of the various test suites
are not uniform, this leads to poor utilisation of the parallel jobs. For
instance, consider four tests, A, B, C and D. Lets assume the runtimes for these
hypothetical tests are:

  A 100
  B 50
  C 50
  D 100

further, lets assume we have two cores to run on. The tests are run
in alphabetical order, leading to a job schedule that looks something like:

Time  Core     Test
0     1        A
0     2        B
50    2        C
100   1        D
100   2        <idle>
200 - All tests complete

However, if jobs are scheduled in roughly decending runtime order, we get a
job schedule that looks roughly like:

Time  Core     Test
0     1        A
0     2        D
100   1        B
100   2        C
150 - All tests complete

This commit re-orders the list of regression suite directories into roughly
decending runtime order.

Author: chrisr-diffblue

regression/CMakeLists.txt

@@ -20,25 +20,29 @@ macro(add_test_pl_tests cmdline)
     add_test_pl_profile("${TEST_DIR_NAME}" "${cmdline}" -K KNOWNBUG ${ARGN})
 endmacro(add_test_pl_tests)
 
-add_subdirectory(ansi-c)
+# For the best possible utilisation of multiple cores when
+# running tests in parallel, it is important that these directories are
+# listed with decreasing runtimes (i.e. longest running at the top)
 add_subdirectory(cbmc)
-add_subdirectory(cbmc-cover)
-add_subdirectory(cbmc-cpp)
 add_subdirectory(cbmc-java)
-add_subdirectory(cbmc-java-inheritance)
-add_subdirectory(cpp)
 add_subdirectory(goto-analyzer)
-add_subdirectory(goto-analyzer-taint)
-add_subdirectory(goto-cc-cbmc)
-add_subdirectory(goto-cc-goto-analyzer)
-add_subdirectory(goto-diff)
+add_subdirectory(ansi-c)
+add_subdirectory(jbmc-strings)
 add_subdirectory(goto-instrument)
+add_subdirectory(cpp)
+add_subdirectory(strings-smoke-tests)
+add_subdirectory(cbmc-cover)
 add_subdirectory(goto-instrument-typedef)
+add_subdirectory(strings)
+add_subdirectory(invariants)
+add_subdirectory(goto-diff)
+add_subdirectory(test-script)
+add_subdirectory(goto-analyzer-taint)
+add_subdirectory(cbmc-java-inheritance)
 if(NOT WIN32)
   add_subdirectory(goto-gcc)
 endif()
-add_subdirectory(invariants)
-add_subdirectory(jbmc-strings)
-add_subdirectory(strings)
-add_subdirectory(strings-smoke-tests)
-add_subdirectory(test-script)
+add_subdirectory(goto-cc-cbmc)
+add_subdirectory(cbmc-cpp)
+add_subdirectory(goto-cc-goto-analyzer)
+

regression/Makefile

@@ -1,23 +1,26 @@
-DIRS = ansi-c \
-       cbmc \
-       cbmc-cover \
-       cbmc-cpp \
+# For the best possible utilisation of multiple cores when
+# running tests in parallel, it is important that these directories are
+# listed with decreasing runtimes (i.e. longest running at the top)
+DIRS = cbmc \
        cbmc-java \
-       cbmc-java-inheritance \
-       cpp \
        goto-analyzer \
-       goto-analyzer-taint \
-       goto-cc-cbmc \
-       goto-cc-goto-analyzer \
-       goto-diff \
-       goto-gcc \
+       ansi-c \
+       jbmc-strings \
        goto-instrument \
+       cpp \
+       strings-smoke-tests \
+       cbmc-cover \
        goto-instrument-typedef \
-       invariants \
        strings \
-       jbmc-strings \
-       strings-smoke-tests \
+       invariants \
+       goto-diff \
        test-script \
+       goto-analyzer-taint \
+       cbmc-java-inheritance \
+       goto-gcc \
+       goto-cc-cbmc \
+       cbmc-cpp \
+       goto-cc-goto-analyzer \
        # Empty last line
 
 # Tests under goto-gcc cannot be run on Windows, so appveyor.yml unlinks