Parallelize benchmark CI (#1320)

* [benchmark] command line config argument

* [benchmark] Add a phony target for every example

* [benchmark] add config to toggle heap profiling

* [shake-bench] Phony rules for binaries

* [CI] benchmark jobs tree

* update mergify conditions

Author: pepeiborra

.github/mergify.yml

@@ -5,7 +5,8 @@ pull_request_rules:
       method: squash
   name: Automatically merge pull requests
   conditions:
-  - status-success=bench (8.10.3, ubuntu-latest)
+  - status-success=bench-example (8.10.3, ubuntu-latest, Cabal-3.0.0.0)
+  - status-success=bench-example (8.10.3, ubuntu-latest, lsp-types-1.0.0.1)
 
   - status-success=nix (default, ubuntu-latest)
   - status-success=nix (default, macOS-latest)

.github/workflows/bench.yml

@@ -2,7 +2,7 @@ name: Benchmark
 
 on: [pull_request]
 jobs:
-  bench:
+  bench-init:
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -38,20 +38,80 @@ jobs:
 
     - name: Build
       shell: bash
-      # Retry it three times to workaround compiler segfaults in windows
-      run: cabal build ghcide:benchHist || cabal build ghcide:benchHist || cabal build ghcide:benchHist
+      run: cabal build ghcide:benchHist
+
+    - name: Bench init
+      shell: bash
+      run: cabal bench ghcide:benchHist  -j --benchmark-options="all-binaries"
+
+    # tar is required to preserve file permissions
+    # compression speeds up upload/download nicely
+    - name: tar workspace
+      shell: bash
+      run: tar -czf workspace.tar.gz * .git
+
+    - name: tar cabal
+      run: |
+        cd ~/.cabal
+        tar -czf cabal.tar.gz *
+
+    - name: Upload workspace
+      uses: actions/upload-artifact@v2
+      with:
+        name: workspace
+        retention-days: 1
+        path: workspace.tar.gz
+
+    - name: Upload .cabal
+      uses: actions/upload-artifact@v2
+      with:
+        name: cabal-home
+        retention-days: 1
+        path: ~/.cabal/cabal.tar.gz
+
+  bench-example:
+    needs: [bench-init]
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ghc: ['8.10.3']
+        os: [ubuntu-latest]
+        example: ['Cabal-3.0.0.0', 'lsp-types-1.0.0.1']
+
+    steps:
+    - uses: haskell/actions/setup@v1
+      with:
+        ghc-version: ${{ matrix.ghc }}
+        cabal-version: '3.2'
+        enable-stack: false
+
+    - name: Download cabal home
+      uses: actions/download-artifact@v2
+      with:
+        name: cabal-home
+        path: .
+
+    - name: Download workspace
+      uses: actions/download-artifact@v2
+      with:
+        name: workspace
+        path: .
+
+    - name: untar
+      run: |
+        tar xzf workspace.tar.gz
+        tar xzf cabal.tar.gz --directory ~/.cabal
 
     - name: Bench
       shell: bash
-      # run the tests without parallelism, otherwise tasty will attempt to run
-      # all test cases simultaneously which causes way too many hls
-      # instances to be spun up for the poor github actions runner to handle
-      run: cabal bench ghcide:benchHist
+      run: cabal bench ghcide:benchHist  -j --benchmark-options="${{ matrix.example }}"
 
     - name: Display results
       shell: bash
       run: |
-        column -s, -t < ghcide/bench-results/results.csv | tee ghcide/bench-results/results.txt
+        column -s, -t < ghcide/bench-results/unprofiled/${{ matrix.example }}/results.csv | tee ghcide/bench-results/unprofiled/${{ matrix.example }}/results.txt
 
     - name: Archive benchmarking artifacts
       uses: actions/upload-artifact@v2

ghcide/.hlint.yaml

@@ -97,7 +97,7 @@
 - flags:
   - default: false
   - {name: [-Wno-missing-signatures, -Wno-orphans, -Wno-overlapping-patterns, -Wno-incomplete-patterns, -Wno-missing-fields, -Wno-unused-matches]}
-  - {name: [-Wno-dodgy-imports,-Wno-incomplete-uni-patterns], within: [Main, Development.IDE.GHC.Compat]}
+  - {name: [-Wno-dodgy-imports,-Wno-incomplete-uni-patterns], within: [Main, Development.IDE.GHC.Compat, Development.Benchmark.Rules]}
 # - modules:
 #   - {name: [Data.Set, Data.HashSet], as: Set} # if you import Data.Set qualified, it must be as 'Set'
 #   - {name: Control.Arrow, within: []} # Certain modules are banned entirely

ghcide/bench/README.md

@@ -6,10 +6,64 @@ performance analysis of ghcide:
 
 - `exe/Main.hs` - a standalone benchmark runner. Run with `stack run ghcide-bench`
 - `hist/Main.hs` - a Shake script for running the benchmark suite over a set of commits.
-  - Run with `stack bench` or `cabal bench`,
+  - Run with `stack bench ghcide` or `cabal bench ghcide`,
   - Requires a `ghcide-bench` binary in the PATH (usually provided by stack/cabal),
   - Calls `cabal` (or `stack`, configurable) internally to build the project,
-  - Driven by the `config.yaml` configuration file.
+  - Driven by the `bench/config.yaml` configuration file.
     By default it compares HEAD with "master"
 
-Further details available in the config file and the module header comments.
+# Examples and experiments
+
+The benchmark suites runs a set of experiments (hover, completion, edit, etc.)
+over all the defined examples (currently Cabal and lsp-types). Examples are defined
+in `ghcide/bench/config.yaml` whereas experiments are coded in `ghcide/bench/lib/Experiments.hs`.
+
+# Phony targets
+
+The Shake script supports a number of phony targets that allow running a subset of the benchmarks:
+
+* all
+:  runs all the examples, unprofiled
+
+* profiled-all
+:  runs all the examples with heap profiling, assuming `profilingInterval` is defined
+
+* Cabal-3.0.0.0
+:  runs the Cabal example, unprofiled
+
+* profiled-Cabal-3.0.0.0
+:  runs the Cabal example, with heap profiling
+
+* etc
+
+`--help` lists all the phony targets. Invoke it with:
+
+    cabal bench ghcide --benchmark-options="--help"
+
+```
+Targets:
+  - bench-results/binaries/*/commitid
+  - bench-results/binaries/HEAD/ghcide
+  - bench-results/binaries/HEAD/ghc.path
+  - bench-results/binaries/*/ghcide
+  - bench-results/binaries/*/ghc.path
+  - bench-results/binaries/*/*.warmup
+  - bench-results/*/*/*/*.csv
+  - bench-results/*/*/*/*.gcStats.log
+  - bench-results/*/*/*/*.output.log
+  - bench-results/*/*/*/*.eventlog
+  - bench-results/*/*/*/*.hp
+  - bench-results/*/*/*/results.csv
+  - bench-results/*/*/results.csv
+  - bench-results/*/results.csv
+  - bench-results/*/*/*/*.svg
+  - bench-results/*/*/*/*.diff.svg
+  - bench-results/*/*/*.svg
+  - bench-results/*/*/*/*.heap.svg
+  - Cabal-3.0.0.0
+  - lsp-types-1.0.0.1
+  - all
+  - profiled-Cabal-3.0.0.0
+  - profiled-lsp-types-1.0.0.1
+  - profiled-all
+  ```

ghcide/bench/config.yaml

@@ -67,3 +67,7 @@ versions:
 # - ghcide-v0.7.3
 - upstream: origin/master
 - HEAD
+
+# Heap profile interval in seconds (+RTS -i)
+# Comment out to disable heap profiling
+profileInterval: 1

ghcide/bench/hist/Main.hs

@@ -52,10 +52,16 @@ import qualified Experiments.Types as E
 import GHC.Generics (Generic)
 import Numeric.Natural (Natural)
 import Development.Shake.Classes
+import System.Console.GetOpt
+import Data.Maybe
+import Control.Monad.Extra
 
 
-config :: FilePath
-config = "bench/config.yaml"
+configPath :: FilePath
+configPath = "bench/config.yaml"
+
+configOpt :: OptDescr (Either String FilePath)
+configOpt = Option [] ["config"] (ReqArg Right configPath) "config file"
 
 -- | Read the config without dependency
 readConfigIO :: FilePath -> IO (Config BuildSystem)
@@ -65,17 +71,17 @@ instance IsExample Example where getExampleName = E.getExampleName
 type instance RuleResult GetExample = Maybe Example
 type instance RuleResult GetExamples = [Example]
 
+shakeOpts :: ShakeOptions
+shakeOpts =
+    shakeOptions{shakeChange = ChangeModtimeAndDigestInput, shakeThreads = 0}
+
 main :: IO ()
-main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigestInput, shakeThreads = 0} $ do
-  createBuildSystem $ \resource -> do
-      configStatic <- liftIO $ readConfigIO config
-      let build = outputFolder configStatic
-      buildRules build ghcideBuildRules
-      benchRules build resource (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
-      csvRules build
-      svgRules build
-      heapProfileRules build
-      action $ allTargets build
+main = shakeArgsWith shakeOpts [configOpt] $ \configs wants -> pure $ Just $ do
+  let config = fromMaybe configPath $ listToMaybe configs
+  _configStatic <- createBuildSystem config
+  case wants of
+      [] -> want ["all"]
+      _ -> want wants
 
 ghcideBuildRules :: MkBuildRules BuildSystem
 ghcideBuildRules = MkBuildRules findGhcForBuildSystem "ghcide" buildGhcide
@@ -89,13 +95,14 @@ data Config buildSystem = Config
     versions :: [GitCommit],
     -- | Output folder ('foo' works, 'foo/bar' does not)
     outputFolder :: String,
-    buildTool :: buildSystem
+    buildTool :: buildSystem,
+    profileInterval :: Maybe Double
   }
   deriving (Generic, Show)
   deriving anyclass (FromJSON)
 
-createBuildSystem :: (Resource -> Rules a) -> Rules a
-createBuildSystem userRules = do
+createBuildSystem :: FilePath -> Rules (Config BuildSystem )
+createBuildSystem config = do
   readConfig <- newCache $ \fp -> need [fp] >> liftIO (readConfigIO fp)
 
   _ <- addOracle $ \GetExperiments {} -> experiments <$> readConfig config
@@ -105,9 +112,20 @@ createBuildSystem userRules = do
   _ <- addOracle $ \GetBuildSystem {} -> buildTool <$> readConfig config
   _ <- addOracle $ \GetSamples{} -> samples <$> readConfig config
 
-  benchResource <- newResource "ghcide-bench" 1
+  configStatic <- liftIO $ readConfigIO config
+  let build = outputFolder configStatic
+
+  buildRules build ghcideBuildRules
+  benchRules build (MkBenchRules (askOracle $ GetSamples ()) benchGhcide "ghcide")
+  csvRules build
+  svgRules build
+  heapProfileRules build
+  phonyRules "" "ghcide" NoProfiling build (examples configStatic)
+
+  whenJust (profileInterval configStatic) $ \i -> do
+    phonyRules "profiled-" "ghcide" (CheapHeapProfiling i) build (examples configStatic)
 
-  userRules benchResource
+  return configStatic
 
 newtype GetSamples = GetSamples () deriving newtype (Binary, Eq, Hashable, NFData, Show)
 type instance RuleResult GetSamples = Natural

ghcide/ghcide.cabal

@@ -246,7 +246,9 @@ benchmark benchHist
         base == 4.*,
         shake-bench == 0.1.*,
         directory,
+        extra,
         filepath,
+        optparse-applicative,
         shake,
         text,
         yaml