diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 99f30e571a..af1dacb531 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -15,7 +15,7 @@ jobs:
         uses: fkirc/skip-duplicate-actions@v3.4.0
         with:
           cancel_others: true
-          paths_ignore: '["**/docs/**", "**.md", "**/LICENSE", ".circleci/**", "install/**", "nix/**", "**/test/**"]'
+          paths_ignore: '["**/docs/**", "**.md", "**/LICENSE", ".circleci/**", "install/**", "**.nix", "**/test/**", "flake.lock"]'
 
   bench-init:
     needs: pre_job
diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
index 299928b600..4e16290ecf 100644
--- a/.github/workflows/nix.yml
+++ b/.github/workflows/nix.yml
@@ -12,13 +12,19 @@ jobs:
   pre_job:
     runs-on: ubuntu-latest
     outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+      should_skip_develop: ${{ steps.skip_check.outputs.should_skip }}
+      should_skip_build: ${{ steps.skip_check_no_nix.outputs.should_skip }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@master
         with:
           cancel_others: true
           paths_ignore: '["**/docs/**", "**.md", "**/LICENSE", ".circleci/**", "install/**"]'
+      - id: skip_check_no_nix
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          cancel_others: false
+          paths: '["**.nix"]'
 
   # Enter the development shell and run `cabal build`
   develop:
@@ -31,11 +37,11 @@ jobs:
         os: [ubuntu-latest, macOS-latest]
 
     steps:
-    - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    - if: ${{ needs.pre_job.outputs.should_skip_develop != 'true' }}
       uses: actions/checkout@v2
       with:
         submodules: true
-    - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    - if: ${{ needs.pre_job.outputs.should_skip_develop != 'true' }}
       uses: cachix/install-nix-action@v13
       with:
         install_url: https://nixos-nix-install-tests.cachix.org/serve/lb41az54kzk6j12p81br4bczary7m145/install
@@ -43,22 +49,27 @@ jobs:
         extra_nix_config: |
           experimental-features = nix-command flakes
         nix_path: nixpkgs=channel:nixos-unstable
-    - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    - if: ${{ needs.pre_job.outputs.should_skip_develop != 'true' }}
       uses: cachix/cachix-action@v10
       with:
         name: haskell-language-server
         # Disable pushing, we will do that in job `build`
         skipPush: true
-    - if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    - if: ${{ needs.pre_job.outputs.should_skip_develop != 'true' }}
       run: |
         nix develop --command cabal update
         nix develop --command cabal build
 
   # Build and then push HLS binaries with developmet shell to cachix
-  # This job runs when PRs are merged to master, and should be excluded from branch protections
+  # This job runs when
+  #   1. PRs are merged to master (runs on master)
+  #   2. Nix files are changed (runs on PR)
   build:
+    needs: pre_job
     runs-on: ${{ matrix.os }}
-    if: ${{ github.repository_owner == 'haskell' && github.ref == 'refs/heads/master' }}
+    env:
+      HAS_TOKEN: ${{ secrets.HLS_CACHIX_AUTH_TOKEN != '' }}
+    if: ${{ needs.pre_job.outputs.should_skip_build != 'true' || (github.repository_owner == 'haskell' && github.ref == 'refs/heads/master') }}
     strategy:
       fail-fast: false
       matrix:
@@ -79,9 +90,13 @@ jobs:
       with:
         name: haskell-language-server
         authToken: ${{ secrets.HLS_CACHIX_AUTH_TOKEN }}
+    - name: Build development shell
+      run: nix develop --profile dev
     - name: Push development shell
-      run: nix develop --profile dev && cachix push haskell-language-server dev
-    - name: Build and push binaries
-      run: |
-        nix build
-        nix path-info --json | jq -r '.[].path' | cachix push haskell-language-server
+      if: ${{ env.HAS_TOKEN == 'true' }}
+      run: cachix push haskell-language-server dev
+    - name: Build binaries
+      run: nix build
+    - name: Push binaries
+      if: ${{ env.HAS_TOKEN == 'true' }}
+      run: nix path-info --json | jq -r '.[].path' | cachix push haskell-language-server
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 714be66cc6..f4cbca11c6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         uses: fkirc/skip-duplicate-actions@v3.4.0
         with:
           cancel_others: true
-          paths_ignore: '["**/docs/**", "**.md", "**/LICENSE", ".circleci/**", "install/**", "nix/**"]'
+          paths_ignore: '["**/docs/**", "**.md", "**/LICENSE", ".circleci/**", "install/**", "**.nix", "flake.lock"]'
       # If we only change ghcide downstream packages we have not test ghcide itself
       - id: skip_ghcide_check
         uses: fkirc/skip-duplicate-actions@v3.4.0
diff --git a/flake.nix b/flake.nix
index 95e7a83693..604f909b51 100644
--- a/flake.nix
+++ b/flake.nix
@@ -69,13 +69,29 @@
           } // pluginSourceDirs;
 
           # Tweak our packages
+          # Don't use `callHackage`, it requires us to override `all-cabal-hashes`
           tweaks = hself: hsuper:
             with haskell.lib; {
-              hls-hlint-plugin =
-                hsuper.hls-hlint-plugin.override { hlint = hself.hlint_3_2_7; };
+
+              # https://github.com/haskell/haskell-language-server/pull/1858
+              # Remove this override when nixpkgs has this package
+              apply-refact_0_9_3_0 = hself.callCabal2nix "apply-refact"
+                (builtins.fetchTarball {
+                  url =
+                    "https://hackage.haskell.org/package/apply-refact-0.9.3.0/apply-refact-0.9.3.0.tar.gz";
+                  sha256 =
+                    "1jfq1aw91finlpq5nn7a96za4c8j13jk6jmx2867fildxwrik2qj";
+                }) { };
+
+              hls-hlint-plugin = hsuper.hls-hlint-plugin.override {
+                hlint = hself.hlint_3_2_7;
+                apply-refact = hself.apply-refact_0_9_3_0;
+              };
+
               hls-tactics-plugin = hsuper.hls-tactics-plugin.override {
                 refinery = hself.refinery_0_3_0_0;
               };
+
             };
 
           hlsSources =