From 3c6e105da6f40e694767f799b72ecd13a0d4d487 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 12:16:22 +0000 Subject: [PATCH 1/6] Set test options via cabal.project This means we can set the default properly for people using the project, and also set things more simply in the test workflow. --- .github/workflows/test.yml | 70 ++++++++++++++++------------------ cabal.project | 5 +++ hls-test-utils/src/Test/Hls.hs | 3 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11a634389e..462607c906 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,151 +104,147 @@ jobs: run: cabal build - name: Set test options - # run the tests without parallelism, otherwise tasty will attempt to run - # all functional test cases simultaneously which causes way too many hls - # instances to be spun up for the poor github actions runner to handle - # # See https://github.com/ocharles/tasty-rerun/issues/22 for why we need # to include 'new' in the filters, since many of our test suites are in the # same package. run: | - echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions,new" >> $GITHUB_ENV + echo "test-options: --rerun-update --rerun-filter failures,exceptions,new" >> cabal.project.local - if: matrix.test name: Test hls-graph - run: cabal test hls-graph --test-options="$TEST_OPTS" + run: cabal test hls-graph - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test name: Test ghcide # run the tests without parallelism to avoid running out of memory - run: cabal test ghcide --test-options="$TEST_OPTS" || cabal test ghcide --test-options="$TEST_OPTS" + run: cabal test ghcide || cabal test ghcide - if: matrix.test name: Test hls-plugin-api - run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || cabal test hls-plugin-api --test-options="$TEST_OPTS" + run: cabal test hls-plugin-api || cabal test hls-plugin-api - if: matrix.test name: Test func-test suite env: HLS_TEST_EXE: hls HLS_WRAPPER_TEST_EXE: hls-wrapper - run: cabal test func-test --test-options="$TEST_OPTS" || cabal test func-test --test-options="$TEST_OPTS" + run: cabal test func-test || cabal test func-test - if: matrix.test name: Test wrapper-test suite env: HLS_TEST_EXE: hls HLS_WRAPPER_TEST_EXE: hls-wrapper - run: cabal test wrapper-test --test-options="$TEST_OPTS --rerun-log-file .tasty-rerun-log-wrapper" + run: cabal test wrapper-test - if: matrix.test name: Test hls-refactor-plugin - run: cabal test hls-refactor-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-refactor-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-refactor-plugin-tests || cabal test hls-refactor-plugin-tests - if: matrix.test name: Test hls-floskell-plugin - run: cabal test hls-floskell-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-floskell-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-floskell-plugin-tests || cabal test hls-floskell-plugin-tests - if: matrix.test name: Test hls-class-plugin - run: cabal test hls-class-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-class-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-class-plugin-tests || cabal test hls-class-plugin-tests - if: matrix.test name: Test hls-pragmas-plugin - run: cabal test hls-pragmas-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-pragmas-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-pragmas-plugin-tests || cabal test hls-pragmas-plugin-tests - if: matrix.test name: Test hls-eval-plugin - run: cabal test hls-eval-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-eval-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-eval-plugin-tests || cabal test hls-eval-plugin-tests - if: matrix.test name: Test hls-splice-plugin - run: cabal test hls-splice-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-splice-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-splice-plugin-tests || cabal test hls-splice-plugin-tests - if: matrix.test && matrix.ghc != '9.2' name: Test hls-stan-plugin - run: cabal test hls-stan-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-stan-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-stan-plugin-tests || cabal test hls-stan-plugin-tests - if: matrix.test name: Test hls-stylish-haskell-plugin - run: cabal test hls-stylish-haskell-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-stylish-haskell-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-stylish-haskell-plugin-tests || cabal test hls-stylish-haskell-plugin-tests - if: matrix.test name: Test hls-ormolu-plugin - run: cabal test hls-ormolu-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-ormolu-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-ormolu-plugin-tests || cabal test hls-ormolu-plugin-tests - if: matrix.test name: Test hls-fourmolu-plugin - run: cabal test hls-fourmolu-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-fourmolu-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-fourmolu-plugin-tests || cabal test hls-fourmolu-plugin-tests - if: matrix.test name: Test hls-explicit-imports-plugin test suite - run: cabal test hls-explicit-imports-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-imports-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-explicit-imports-plugin-tests || cabal test hls-explicit-imports-plugin-tests - if: matrix.test name: Test hls-call-hierarchy-plugin test suite - run: cabal test hls-call-hierarchy-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-call-hierarchy-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-call-hierarchy-plugin-tests || cabal test hls-call-hierarchy-plugin-tests - if: matrix.test && matrix.os != 'windows-latest' name: Test hls-rename-plugin test suite - run: cabal test hls-rename-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-rename-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-rename-plugin-tests || cabal test hls-rename-plugin-tests - if: matrix.test name: Test hls-hlint-plugin test suite - run: cabal test hls-hlint-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-hlint-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-hlint-plugin-tests || cabal test hls-hlint-plugin-tests - if: matrix.test name: Test hls-module-name-plugin test suite - run: cabal test hls-module-name-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-module-name-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-module-name-plugin-tests || cabal test hls-module-name-plugin-tests - if: matrix.test name: Test hls-alternate-number-format-plugin test suite - run: cabal test hls-alternate-number-format-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-alternate-number-format-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-alternate-number-format-plugin-tests || cabal test hls-alternate-number-format-plugin-tests - if: matrix.test name: Test hls-qualify-imported-names-plugin test suite - run: cabal test hls-qualify-imported-names-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-qualify-imported-names-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-qualify-imported-names-plugin-tests || cabal test hls-qualify-imported-names-plugin-tests - if: matrix.test name: Test hls-code-range-plugin test suite - run: cabal test hls-code-range-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-code-range-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-code-range-plugin-tests || cabal test hls-code-range-plugin-tests - if: matrix.test name: Test hls-change-type-signature test suite - run: cabal test hls-change-type-signature-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-change-type-signature-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-change-type-signature-plugin-tests || cabal test hls-change-type-signature-plugin-tests - if: matrix.test name: Test hls-gadt-plugin test suit - run: cabal test hls-gadt-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-gadt-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-gadt-plugin-tests || cabal test hls-gadt-plugin-tests - if: matrix.test name: Test hls-explicit-fixity-plugin test suite - run: cabal test hls-explicit-fixity-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-fixity-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-explicit-fixity-plugin-tests || cabal test hls-explicit-fixity-plugin-tests - if: matrix.test name: Test hls-explicit-record-fields-plugin test suite - run: cabal test hls-explicit-record-fields-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-record-fields-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-explicit-record-fields-plugin-tests || cabal test hls-explicit-record-fields-plugin-tests ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions - if: matrix.test && matrix.ghc == '9.2' name: Test hls-cabal-fmt-plugin test suite - run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS" + run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests - if: matrix.test name: Test hls-cabal-plugin test suite - run: cabal test hls-cabal-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-cabal-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-cabal-plugin-tests || cabal test hls-cabal-plugin-tests - if: matrix.test name: Test hls-retrie-plugin test suite - run: cabal test hls-retrie-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-retrie-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-retrie-plugin-tests || cabal test hls-retrie-plugin-tests - if: matrix.test name: Test hls-overloaded-record-dot-plugin test suite - run: cabal test hls-overloaded-record-dot-plugin-tests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-overloaded-record-dot-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-overloaded-record-dot-plugin-tests || cabal test hls-overloaded-record-dot-plugin-tests - if: matrix.test name: Test hls-semantic-tokens-plugin test suite - run: cabal test hls-semantic-tokens-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-semantic-tokens-plugin-tests --test-options="$TEST_OPTS" + run: cabal test hls-semantic-tokens-plugin-tests || cabal test hls-semantic-tokens-plugin-tests test_post_job: diff --git a/cabal.project b/cabal.project index adf19ed228..dd45e316e3 100644 --- a/cabal.project +++ b/cabal.project @@ -18,6 +18,11 @@ benchmarks: True write-ghc-environment-files: never +-- Many of our tests only work single-threaded, and the only way to +-- ensure tasty runs everything purely single-threaded is to pass +-- this at the top-level +test-options: -j1 + -- Make sure dependencies are build with haddock so we get -- haddock shown on hover package * diff --git a/hls-test-utils/src/Test/Hls.hs b/hls-test-utils/src/Test/Hls.hs index a9fc31ef71..17fb48ff99 100644 --- a/hls-test-utils/src/Test/Hls.hs +++ b/hls-test-utils/src/Test/Hls.hs @@ -122,7 +122,6 @@ import Test.Tasty.ExpectedFailure import Test.Tasty.Golden import Test.Tasty.HUnit import Test.Tasty.Ingredients.Rerun -import Test.Tasty.Runners (NumThreads (..)) data Log = LogIDEMain IDEMain.Log @@ -147,7 +146,7 @@ instance Pretty LogTestHarness where -- | Run 'defaultMainWithRerun', limiting each single test case running at most 10 minutes defaultTestRunner :: TestTree -> IO () -defaultTestRunner = defaultMainWithRerun . adjustOption (const $ NumThreads 1) . adjustOption (const $ mkTimeout 600000000) +defaultTestRunner = defaultMainWithRerun . adjustOption (const $ mkTimeout 600000000) gitDiff :: FilePath -> FilePath -> [String] gitDiff fRef fNew = ["git", "-c", "core.fileMode=false", "diff", "--no-index", "--text", "--exit-code", fRef, fNew] From 73fa8013157c0040e5639e35167e7cd383757297 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 12:31:10 +0000 Subject: [PATCH 2/6] Set options after testing hls-graph --- .github/workflows/test.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 462607c906..f8280abe93 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,17 +103,19 @@ jobs: - name: Build run: cabal build - - name: Set test options + - if: matrix.test + name: Test hls-graph + run: cabal test hls-graph + + # Do this after running the tests that don't use tasty-rerun, otherwise they complain + # about not recognizing the arguments + - name: Set rerun test options # See https://github.com/ocharles/tasty-rerun/issues/22 for why we need # to include 'new' in the filters, since many of our test suites are in the # same package. run: | echo "test-options: --rerun-update --rerun-filter failures,exceptions,new" >> cabal.project.local - - if: matrix.test - name: Test hls-graph - run: cabal test hls-graph - - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test name: Test ghcide # run the tests without parallelism to avoid running out of memory From bbcdb120ee1804be36ed4bbd11c82264a0c232ba Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 12:43:46 +0000 Subject: [PATCH 3/6] Revert "Set options after testing hls-graph" This reverts commit 73fa8013157c0040e5639e35167e7cd383757297. --- .github/workflows/test.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8280abe93..462607c906 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,19 +103,17 @@ jobs: - name: Build run: cabal build - - if: matrix.test - name: Test hls-graph - run: cabal test hls-graph - - # Do this after running the tests that don't use tasty-rerun, otherwise they complain - # about not recognizing the arguments - - name: Set rerun test options + - name: Set test options # See https://github.com/ocharles/tasty-rerun/issues/22 for why we need # to include 'new' in the filters, since many of our test suites are in the # same package. run: | echo "test-options: --rerun-update --rerun-filter failures,exceptions,new" >> cabal.project.local + - if: matrix.test + name: Test hls-graph + run: cabal test hls-graph + - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test name: Test ghcide # run the tests without parallelism to avoid running out of memory From 2193f9449fb0ee3b01c8684410d4f180881370e9 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 12:44:06 +0000 Subject: [PATCH 4/6] Quote filters --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 462607c906..6dbca74393 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,7 +108,7 @@ jobs: # to include 'new' in the filters, since many of our test suites are in the # same package. run: | - echo "test-options: --rerun-update --rerun-filter failures,exceptions,new" >> cabal.project.local + echo 'test-options: --rerun-update --rerun-filter "failures,exceptions,new"' >> cabal.project.local - if: matrix.test name: Test hls-graph From 6f2bf1f7c70158acab9c340678c9d6948aafa94b Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 16:36:50 +0000 Subject: [PATCH 5/6] Use cabal configure instead --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dbca74393..56a5fe468c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,7 +108,7 @@ jobs: # to include 'new' in the filters, since many of our test suites are in the # same package. run: | - echo 'test-options: --rerun-update --rerun-filter "failures,exceptions,new"' >> cabal.project.local + echo 'cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new"' >> cabal.project.local - if: matrix.test name: Test hls-graph From a29a00522a152f16ce9cddbb46093c219fb8a0d0 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Sat, 3 Feb 2024 18:03:28 +0000 Subject: [PATCH 6/6] Try better --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56a5fe468c..dfed301f55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,7 +108,7 @@ jobs: # to include 'new' in the filters, since many of our test suites are in the # same package. run: | - echo 'cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new"' >> cabal.project.local + cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new" - if: matrix.test name: Test hls-graph