diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b8fda76503..428a035800 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -138,6 +138,15 @@ jobs:
           sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
                      src/**/*.hs exe/*.hs
 
+      # this is only safe if the test environment is isolated
+      - name: setup ide cache
+        run: |
+          export XDG_CACHE_HOME=$RUNNER_TEMP/cache
+          rm -rf ~/.cache/ghcide
+          rm -rf ~/.cache/hie-bios
+          rm -rf $XDG_CACHE_HOME/ghcide
+          rm -rf $XDG_CACHE_HOME/hie-bios
+
       - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
         name: Build
         # Retry it three times to workaround compiler segfaults in windows