Skip to content

Set test options via cabal.project #4039

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Feb 3, 2024
Merged

Set test options via cabal.project #4039

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3c6e105
Set test options via cabal.project
michaelpj Feb 3, 2024
73fa801
Set options after testing hls-graph
michaelpj Feb 3, 2024
bbcdb12
Revert "Set options after testing hls-graph"
michaelpj Feb 3, 2024
2193f94
Quote filters
michaelpj Feb 3, 2024
3d6b45a
Merge branch 'master' into mpj/project-test-options
michaelpj Feb 3, 2024
6f2bf1f
Use cabal configure instead
michaelpj Feb 3, 2024
8cc2b3e
Merge branch 'master' into mpj/project-test-options
michaelpj Feb 3, 2024
a29a005
Try better
michaelpj Feb 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 33 additions & 37 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,151 +104,147 @@ jobs:
run: cabal build

- name: Set test options
# run the tests without parallelism, otherwise tasty will attempt to run
# all functional test cases simultaneously which causes way too many hls
# instances to be spun up for the poor github actions runner to handle
#
# See https://github.com/ocharles/tasty-rerun/issues/22 for why we need
# to include 'new' in the filters, since many of our test suites are in the
# same package.
run: |
echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions,new" >> $GITHUB_ENV
echo 'cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new"' >> cabal.project.local
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean this, which has the side effect of writing those options into cabal.project.local

Suggested change
echo 'cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new"' >> cabal.project.local
cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new" 
$ cabal configure --test-options="--rerun-update --rerun-filter failures,exceptions,new"
$ cat cabal.project.local 
ignore-project: False
test-options: "--rerun-update"
              "--rerun-filter"
              "failures,exceptions,new"

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's what I meant to type, just 🤦


- if: matrix.test
name: Test hls-graph
run: cabal test hls-graph --test-options="$TEST_OPTS"
run: cabal test hls-graph

- if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
name: Test ghcide
# run the tests without parallelism to avoid running out of memory
run: cabal test ghcide --test-options="$TEST_OPTS" || cabal test ghcide --test-options="$TEST_OPTS"
run: cabal test ghcide || cabal test ghcide

- if: matrix.test
name: Test hls-plugin-api
run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || cabal test hls-plugin-api --test-options="$TEST_OPTS"
run: cabal test hls-plugin-api || cabal test hls-plugin-api

- if: matrix.test
name: Test func-test suite
env:
HLS_TEST_EXE: hls
HLS_WRAPPER_TEST_EXE: hls-wrapper
run: cabal test func-test --test-options="$TEST_OPTS" || cabal test func-test --test-options="$TEST_OPTS"
run: cabal test func-test || cabal test func-test

- if: matrix.test
name: Test wrapper-test suite
env:
HLS_TEST_EXE: hls
HLS_WRAPPER_TEST_EXE: hls-wrapper
run: cabal test wrapper-test --test-options="$TEST_OPTS --rerun-log-file .tasty-rerun-log-wrapper"
run: cabal test wrapper-test

- if: matrix.test
name: Test hls-refactor-plugin
run: cabal test hls-refactor-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-refactor-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-refactor-plugin-tests || cabal test hls-refactor-plugin-tests

- if: matrix.test
name: Test hls-floskell-plugin
run: cabal test hls-floskell-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-floskell-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-floskell-plugin-tests || cabal test hls-floskell-plugin-tests

- if: matrix.test
name: Test hls-class-plugin
run: cabal test hls-class-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-class-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-class-plugin-tests || cabal test hls-class-plugin-tests

- if: matrix.test
name: Test hls-pragmas-plugin
run: cabal test hls-pragmas-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-pragmas-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-pragmas-plugin-tests || cabal test hls-pragmas-plugin-tests

- if: matrix.test
name: Test hls-eval-plugin
run: cabal test hls-eval-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-eval-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-eval-plugin-tests || cabal test hls-eval-plugin-tests

- if: matrix.test
name: Test hls-splice-plugin
run: cabal test hls-splice-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-splice-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-splice-plugin-tests || cabal test hls-splice-plugin-tests

- if: matrix.test && matrix.ghc != '9.2'
name: Test hls-stan-plugin
run: cabal test hls-stan-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-stan-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-stan-plugin-tests || cabal test hls-stan-plugin-tests

- if: matrix.test
name: Test hls-stylish-haskell-plugin
run: cabal test hls-stylish-haskell-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-stylish-haskell-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-stylish-haskell-plugin-tests || cabal test hls-stylish-haskell-plugin-tests

- if: matrix.test
name: Test hls-ormolu-plugin
run: cabal test hls-ormolu-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-ormolu-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-ormolu-plugin-tests || cabal test hls-ormolu-plugin-tests

- if: matrix.test
name: Test hls-fourmolu-plugin
run: cabal test hls-fourmolu-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-fourmolu-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-fourmolu-plugin-tests || cabal test hls-fourmolu-plugin-tests

- if: matrix.test
name: Test hls-explicit-imports-plugin test suite
run: cabal test hls-explicit-imports-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-imports-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-explicit-imports-plugin-tests || cabal test hls-explicit-imports-plugin-tests

- if: matrix.test
name: Test hls-call-hierarchy-plugin test suite
run: cabal test hls-call-hierarchy-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-call-hierarchy-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-call-hierarchy-plugin-tests || cabal test hls-call-hierarchy-plugin-tests

- if: matrix.test && matrix.os != 'windows-latest'
name: Test hls-rename-plugin test suite
run: cabal test hls-rename-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-rename-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-rename-plugin-tests || cabal test hls-rename-plugin-tests

- if: matrix.test
name: Test hls-hlint-plugin test suite
run: cabal test hls-hlint-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-hlint-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-hlint-plugin-tests || cabal test hls-hlint-plugin-tests

- if: matrix.test
name: Test hls-module-name-plugin test suite
run: cabal test hls-module-name-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-module-name-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-module-name-plugin-tests || cabal test hls-module-name-plugin-tests

- if: matrix.test
name: Test hls-alternate-number-format-plugin test suite
run: cabal test hls-alternate-number-format-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-alternate-number-format-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-alternate-number-format-plugin-tests || cabal test hls-alternate-number-format-plugin-tests

- if: matrix.test
name: Test hls-qualify-imported-names-plugin test suite
run: cabal test hls-qualify-imported-names-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-qualify-imported-names-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-qualify-imported-names-plugin-tests || cabal test hls-qualify-imported-names-plugin-tests

- if: matrix.test
name: Test hls-code-range-plugin test suite
run: cabal test hls-code-range-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-code-range-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-code-range-plugin-tests || cabal test hls-code-range-plugin-tests

- if: matrix.test
name: Test hls-change-type-signature test suite
run: cabal test hls-change-type-signature-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-change-type-signature-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-change-type-signature-plugin-tests || cabal test hls-change-type-signature-plugin-tests

- if: matrix.test
name: Test hls-gadt-plugin test suit
run: cabal test hls-gadt-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-gadt-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-gadt-plugin-tests || cabal test hls-gadt-plugin-tests

- if: matrix.test
name: Test hls-explicit-fixity-plugin test suite
run: cabal test hls-explicit-fixity-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-fixity-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-explicit-fixity-plugin-tests || cabal test hls-explicit-fixity-plugin-tests

- if: matrix.test
name: Test hls-explicit-record-fields-plugin test suite
run: cabal test hls-explicit-record-fields-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-explicit-record-fields-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-explicit-record-fields-plugin-tests || cabal test hls-explicit-record-fields-plugin-tests

## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions
- if: matrix.test && matrix.ghc == '9.2'
name: Test hls-cabal-fmt-plugin test suite
run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS"
run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests

- if: matrix.test
name: Test hls-cabal-plugin test suite
run: cabal test hls-cabal-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-cabal-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-cabal-plugin-tests || cabal test hls-cabal-plugin-tests

- if: matrix.test
name: Test hls-retrie-plugin test suite
run: cabal test hls-retrie-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-retrie-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-retrie-plugin-tests || cabal test hls-retrie-plugin-tests

- if: matrix.test
name: Test hls-overloaded-record-dot-plugin test suite
run: cabal test hls-overloaded-record-dot-plugin-tests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-overloaded-record-dot-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-overloaded-record-dot-plugin-tests || cabal test hls-overloaded-record-dot-plugin-tests

- if: matrix.test
name: Test hls-semantic-tokens-plugin test suite
run: cabal test hls-semantic-tokens-plugin-tests --test-options="$TEST_OPTS" || cabal test hls-semantic-tokens-plugin-tests --test-options="$TEST_OPTS"
run: cabal test hls-semantic-tokens-plugin-tests || cabal test hls-semantic-tokens-plugin-tests


test_post_job:
Expand Down
5 changes: 5 additions & 0 deletions cabal.project
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ benchmarks: True

write-ghc-environment-files: never

-- Many of our tests only work single-threaded, and the only way to
-- ensure tasty runs everything purely single-threaded is to pass
-- this at the top-level
test-options: -j1

-- Make sure dependencies are build with haddock so we get
-- haddock shown on hover
package *
Expand Down
3 changes: 1 addition & 2 deletions hls-test-utils/src/Test/Hls.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,6 @@ import Test.Tasty.ExpectedFailure
import Test.Tasty.Golden
import Test.Tasty.HUnit
import Test.Tasty.Ingredients.Rerun
import Test.Tasty.Runners (NumThreads (..))

data Log
= LogIDEMain IDEMain.Log
Expand All @@ -147,7 +146,7 @@ instance Pretty LogTestHarness where

-- | Run 'defaultMainWithRerun', limiting each single test case running at most 10 minutes
defaultTestRunner :: TestTree -> IO ()
defaultTestRunner = defaultMainWithRerun . adjustOption (const $ NumThreads 1) . adjustOption (const $ mkTimeout 600000000)
defaultTestRunner = defaultMainWithRerun . adjustOption (const $ mkTimeout 600000000)

gitDiff :: FilePath -> FilePath -> [String]
gitDiff fRef fNew = ["git", "-c", "core.fileMode=false", "diff", "--no-index", "--text", "--exit-code", fRef, fNew]
Expand Down