ci: Optimize build matrix

.cirrus.yml

@@ -4,10 +4,11 @@ env:
   # Specific warnings can be disabled with -Wno-error=foo.
   # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual.
   WERROR_CFLAGS: -Werror -pedantic-errors
-  MAKEFLAGS: -j2
-  BUILD: check
+  MAKEFLAGS: -j4
+  # Run check and then distcheck.
+  BUILD: "distcheck check"
   ### secp256k1 config
-  STATICPRECOMPUTATION: yes
+  ECMULTWINDOW: auto
   ECMULTGENPRECISION: auto
   ASM: no
   WIDEMUL: auto
@@ -50,28 +51,33 @@ merge_base_script_snippet: &MERGE_BASE
     - git config --global user.name "ci"
     - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
 
-task:
-  name: "x86_64: Linux (Debian stable)"
+linux_container_snippet: &LINUX_CONTAINER
   container:
     dockerfile: ci/linux-debian.Dockerfile
     # Reduce number of CPUs to be able to do more builds in parallel.
     cpu: 1
+    # Gives us more CPUs for free if they're available.
+    greedy: true
     # More than enough for our scripts.
     memory: 1G
+
+task:
+  name: "x86_64: Linux (Debian stable)"
+  << : *LINUX_CONTAINER
   matrix: &ENV_MATRIX
-    - env: {WIDEMUL:  int64,  RECOVERY: yes}
-    - env: {WIDEMUL:  int64,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128}
-    - env: {WIDEMUL: int128,  RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
-    - env: {WIDEMUL: int128,  ASM: x86_64}
-    - env: {                  RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
-    - env: {                  STATICPRECOMPUTATION: no}
-    - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETEST: no, BENCH: no}
-    - env: {CPPFLAGS: -DDETERMINISTIC}
-    - env: {CFLAGS: -O0, CTIMETEST: no}
-    - env: { ECMULTGENPRECISION: 2 }
-    - env: { ECMULTGENPRECISION: 8 }
+    # Enable all modules and test with all common configuration (WIDEMUL, ASM, some ECMULTWINDOW, some ECMULTGENPRECISION).
+    # Sneak in rare build options (CFLAGS: "-O0 -g", CTIMETEST: no, WITH_VALGRIND: no, BENCH: no, CPPFLAGS: -DDETERMINISTIC) at random positions.
+    - env: { WIDEMUL:  int64, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes }
+    - env: { WIDEMUL:  int64, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes, CFLAGS: "-O0 -g", CTIMETEST: no, WITH_VALGRIND: no }
+    - env: { WIDEMUL: int128, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes }
+    - env: { WIDEMUL: int128, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes, ASM: x86_64 }
+    - env: { WIDEMUL: int128, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes, ASM: x86_64, ECMULTWINDOW: 4, ECMULTGENPRECISION: 2 }
+    - env: { WIDEMUL: int128, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes,              ECMULTWINDOW: 8, ECMULTGENPRECISION: 8, CPPFLAGS: -DDETERMINISTIC }
+    - env: { WIDEMUL: int128, RECOVERY: yes, ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes, CFLAGS: "-O0 -g", CTIMETEST: no, BENCH: no }
+    # No special settings, this is also useful to test autodetection
+    - env: { }
+    # Only non-experimental modules
+    - env: {                  RECOVERY: yes, ECDH: yes }
   matrix:
     - env:
         CC: gcc
@@ -84,10 +90,7 @@ task:
 
 task:
   name: "i686: Linux (Debian stable)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     HOST: i686-linux-gnu
     ECDH: yes
@@ -113,8 +116,7 @@ task:
     HOMEBREW_NO_INSTALL_CLEANUP: 1
     # Cirrus gives us a fixed number of 12 virtual CPUs. Not that we even have that many jobs at the moment...
     MAKEFLAGS: -j13
-  matrix:
-    << : *ENV_MATRIX
+  << : *ENV_MATRIX
   matrix:
     - env:
         CC: gcc-9
@@ -134,8 +136,9 @@ task:
   ##   - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
   ##
   brew_valgrind_pre_script:
+    - brew update
     - brew config
-    - brew tap --shallow LouisBrunner/valgrind
+    - brew tap LouisBrunner/valgrind
     # Fetch valgrind source but don't build it yet.
     - brew fetch --HEAD LouisBrunner/valgrind/valgrind
   brew_valgrind_cache:
@@ -165,10 +168,7 @@ task:
 
 task:
   name: "s390x (big-endian): Linux (Debian stable, QEMU)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     WRAPPER_CMD: qemu-s390x
     SECP256K1_TEST_ITERS: 16
@@ -188,10 +188,7 @@ task:
 
 task:
   name: "ARM32: Linux (Debian stable, QEMU)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     WRAPPER_CMD: qemu-arm
     SECP256K1_TEST_ITERS: 16
@@ -212,10 +209,7 @@ task:
 
 task:
   name: "ARM64: Linux (Debian stable, QEMU)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     WRAPPER_CMD: qemu-aarch64
     SECP256K1_TEST_ITERS: 16
@@ -233,10 +227,7 @@ task:
 
 task:
   name: "ppc64le: Linux (Debian stable, QEMU)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     WRAPPER_CMD: qemu-ppc64le
     SECP256K1_TEST_ITERS: 16
@@ -254,10 +245,7 @@ task:
 
 task:
   name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     WRAPPER_CMD: wine64-stable
     SECP256K1_TEST_ITERS: 16
@@ -275,9 +263,8 @@ task:
 
 # Sanitizers
 task:
+  << : *LINUX_CONTAINER
   container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
     memory: 2G
   env:
     ECDH: yes
@@ -302,11 +289,10 @@ task:
   matrix:
     - env:
         ASM: auto
-        STATICPRECOMPUTATION: yes
     - env:
         ASM: no
-        STATICPRECOMPUTATION: no
         ECMULTGENPRECISION: 2
+        ECMULTWINDOW: 4
   matrix:
     - env:
         CC: clang
@@ -320,15 +306,12 @@ task:
 
 task:
   name: "C++ -fpermissive"
-  container:
-    dockerfile: ci/linux-debian.Dockerfile
-    cpu: 1
-    memory: 1G
+  << : *LINUX_CONTAINER
   env:
     # ./configure correctly errors out when given CC=g++.
     # We hack around this by passing CC=g++ only to make.
     CC: gcc
-    MAKEFLAGS: -j2 CC=g++ CFLAGS=-fpermissive\ -g
+    MAKEFLAGS: -j4 CC=g++ CFLAGS=-fpermissive\ -g
     WERROR_CFLAGS:
     EXPERIMENTAL: yes
     ECDH: yes

ci/cirrus.sh

@@ -15,7 +15,8 @@ valgrind --version || true
 ./configure \
     --enable-experimental="$EXPERIMENTAL" \
     --with-test-override-wide-multiply="$WIDEMUL" --with-asm="$ASM" \
-    --enable-ecmult-static-precomputation="$STATICPRECOMPUTATION" --with-ecmult-gen-precision="$ECMULTGENPRECISION" \
+    --with-ecmult-window="$ECMULTWINDOW" \
+    --with-ecmult-gen-precision="$ECMULTGENPRECISION" \
     --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \
     --enable-module-schnorrsig="$SCHNORRSIG" \
     --with-valgrind="$WITH_VALGRIND" \

src/precompute_ecmult.c

@@ -32,7 +32,7 @@ static void print_table(FILE *fp, const char *name, int window_g, const secp256k
 
     j = 1;
     for(i = 3; i <= window_g; ++i) {
-        fprintf(fp, "#if ECMULT_TABLE_SIZE(WINDOW_G) > %ld\n", ECMULT_TABLE_SIZE(i-1));
+        fprintf(fp, "#if WINDOW_G > %d\n", i-1);
         for(;j < ECMULT_TABLE_SIZE(i); ++j) {
             fprintf(fp, ",S(%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32
                           ",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32",%"PRIx32")\n",
@@ -57,6 +57,8 @@ static void print_two_tables(FILE *fp, int window_g) {
 }
 
 int main(void) {
+    /* Always compute all tables for window sizes up to 15. */
+    int window_g = (ECMULT_WINDOW_SIZE < 15) ? 15 : ECMULT_WINDOW_SIZE;
     FILE* fp;
 
     fp = fopen("src/precomputed_ecmult.c","w");
@@ -77,15 +79,15 @@ int main(void) {
     fprintf(fp, "#include \"ecmult.h\"\n");
     fprintf(fp, "#include \"precomputed_ecmult.h\"\n");
     fprintf(fp, "#define S(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) SECP256K1_GE_STORAGE_CONST(0x##a##u,0x##b##u,0x##c##u,0x##d##u,0x##e##u,0x##f##u,0x##g##u,0x##h##u,0x##i##u,0x##j##u,0x##k##u,0x##l##u,0x##m##u,0x##n##u,0x##o##u,0x##p##u)\n");
-    fprintf(fp, "#if ECMULT_TABLE_SIZE(ECMULT_WINDOW_SIZE) > %ld\n", ECMULT_TABLE_SIZE(ECMULT_WINDOW_SIZE));
+    fprintf(fp, "#if ECMULT_WINDOW_SIZE > %d\n", window_g);
     fprintf(fp, "   #error configuration mismatch, invalid ECMULT_WINDOW_SIZE. Try deleting precomputed_ecmult.c before the build.\n");
     fprintf(fp, "#endif\n");
     fprintf(fp, "#ifdef EXHAUSTIVE_TEST_ORDER\n");
     fprintf(fp, "#    error Cannot compile precomputed_ecmult.c in exhaustive test mode\n");
     fprintf(fp, "#endif /* EXHAUSTIVE_TEST_ORDER */\n");
     fprintf(fp, "#define WINDOW_G ECMULT_WINDOW_SIZE\n");
 
-    print_two_tables(fp, ECMULT_WINDOW_SIZE);
+    print_two_tables(fp, window_g);
 
     fprintf(fp, "#undef S\n");
     fclose(fp);