Enable a coarse-grained prallelism when running regression test jobs via Makefiles

Previously, the makefiles ran the regression test target sequentially,
regardless of any -jN option passed to make. In this commit, an extra
'test-batch' target is added which enables running the same regression
tests, but with each top-level test directory run as a seperate target.
To avoid horribly interleaved output from parallel test jobs, GNU parallel
is used to execute the test jobs. This provides line bufferering, such that
output from each job is prefixed with a job number. This matches the behaviour
of ctest used in the CMake based builds.

Author: chrisr-diffblue

.travis.yml

@@ -62,6 +62,7 @@ jobs:
             - libwww-perl
             - g++-5
             - libubsan0
+            - parallel
       before_install:
         - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc
       # env: COMPILER=g++-5 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer"
@@ -76,7 +77,7 @@ jobs:
       compiler: gcc
       cache: ccache
       before_install:
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel
         - export PATH=$PATH:/usr/local/opt/ccache/libexec
       env: COMPILER="ccache g++"
 
@@ -87,7 +88,7 @@ jobs:
       compiler: clang
       cache: ccache
       before_install:
-        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache
+        - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel
         - export PATH=$PATH:/usr/local/opt/ccache/libexec
       env:
         - COMPILER="ccache clang++"
@@ -132,6 +133,7 @@ jobs:
             - clang-3.7
             - libstdc++-5-dev
             - libubsan0
+            - parallel
       before_install:
         - mkdir bin ; ln -s /usr/bin/clang-3.7 bin/gcc
         - export CCACHE_CPP2=yes
@@ -258,7 +260,7 @@ install:
 
 script:
   - if [ -e bin/gcc ] ; then export PATH=$PWD/bin:$PATH ; fi ;
-  - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
+  - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test-parallel "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 JOBS=2
   - make -C unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
   - make -C unit test
 

regression/Makefile

@@ -20,15 +20,40 @@ DIRS = ansi-c \
        test-script \
        # Empty last line
 
-# Check for the existence of $dir. Tests under goto-gcc cannot be run on
-# Windows, so appveyor.yml unlinks the entire directory under Windows.
+# Tests under goto-gcc cannot be run on Windows, so appveyor.yml unlinks
+# the entire directory under Windows. This variable will contain the list
+# of directories that actually exist on the current platform.
+PLATFORM_DIRS = $(wildcard $(DIRS))
+
+# Run all test directories in sequence
+.PHONY: test
 test:
-	@for dir in $(DIRS); do \
-	  if [ -d "$$dir" ]; then \
-	    $(MAKE) -C "$$dir" test || exit 1; \
-	  fi; \
+	@for dir in $(PLATFORM_DIRS); do \
+		$(MAKE) "$$dir" || exit 1; \
 	done;
 
+# Pattern to execute a single test suite directory
+.PHONY: $(PLATFORM_DIRS)
+$(PLATFORM_DIRS):
+	@echo "Running $@..." ;
+	$(MAKE) -C "$@" test || exit 1;
+
+# Run all test directories using GNU Parallel
+.PHONY: test-parallel
+.NOTPARALLEL: test-parallel
+test-parallel:
+	@echo "Building with $(JOBS) jobs"
+	parallel \
+		--halt soon,fail=1 \
+		--tag \
+		--tagstring '{#}:' \
+		--linebuffer \
+		--jobs $(JOBS) \
+		$(MAKE) "{}" \
+		::: $(PLATFORM_DIRS)
+
+
+.PHONY: clean
 clean:
 	@for dir in *; do \
 		if [ -d "$$dir" ]; then \