Replace worker-farm with jest-worker (#8496)

* Replace worker-farm with jest-worker

* Apply suggestions from code review

Co-Authored-By: Joe Haddad <[email protected]>

* Remove semaphores on top of jest-worker, unwind
terser worker, and remove extra error log

Author: ijjk

packages/next/build/index.ts

@@ -1,4 +1,3 @@
-import { Sema } from 'async-sema'
 import chalk from 'chalk'
 import fs from 'fs'
 import mkdirpOrig from 'mkdirp'
@@ -15,7 +14,7 @@ import loadConfig, {
 import nanoid from 'next/dist/compiled/nanoid/index.js'
 import path from 'path'
 import { promisify } from 'util'
-import workerFarm from 'worker-farm'
+import Worker from 'jest-worker'
 
 import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages'
 import { recursiveDelete } from '../lib/recursive-delete'
@@ -204,16 +203,10 @@ export default async function build(dir: string, conf = null): Promise<void> {
 
   process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
 
-  const staticCheckSema = new Sema(config.experimental.cpus, {
-    capacity: pageKeys.length,
+  const staticCheckWorkers = new Worker(staticCheckWorker, {
+    numWorkers: config.experimental.cpus,
+    enableWorkerThreads: true,
   })
-  const staticCheckWorkers = workerFarm(
-    {
-      maxConcurrentWorkers: config.experimental.cpus,
-    },
-    staticCheckWorker,
-    ['default']
-  )
 
   await Promise.all(
     pageKeys.map(async page => {
@@ -268,17 +261,10 @@ export default async function build(dir: string, conf = null): Promise<void> {
 
       if (nonReservedPage) {
         try {
-          await staticCheckSema.acquire()
-          const result: any = await new Promise((resolve, reject) => {
-            staticCheckWorkers.default(
-              { serverBundle, runtimeEnvConfig },
-              (error: Error | null, result: any) => {
-                if (error) return reject(error)
-                resolve(result || {})
-              }
-            )
+          let result: any = await (staticCheckWorkers as any).default({
+            serverBundle,
+            runtimeEnvConfig,
           })
-          staticCheckSema.release()
 
           if (result.isHybridAmp) {
             hybridAmpPages.add(page)
@@ -293,15 +279,13 @@ export default async function build(dir: string, conf = null): Promise<void> {
         } catch (err) {
           if (err.message !== 'INVALID_DEFAULT_EXPORT') throw err
           invalidPages.add(page)
-          staticCheckSema.release()
         }
       }
 
       pageInfos.set(page, { size, chunks, serverBundle, static: isStatic })
     })
   )
-
-  workerFarm.end(staticCheckWorkers)
+  staticCheckWorkers.end()
 
   if (invalidPages.size > 0) {
     throw new Error(

packages/next/build/static-checker.ts

@@ -1,14 +1,9 @@
 import { isPageStatic } from './utils'
 
-export default function worker(
-  options: any,
-  callback: (err: Error | null, data?: any) => void
-) {
-  try {
-    const { serverBundle, runtimeEnvConfig } = options || ({} as any)
-    const result = isPageStatic(serverBundle, runtimeEnvConfig)
-    callback(null, result)
-  } catch (error) {
-    callback(error)
-  }
+export default function worker(options: {
+  serverBundle: string
+  runtimeEnvConfig: any
+}) {
+  const { serverBundle, runtimeEnvConfig } = options || ({} as any)
+  return isPageStatic(serverBundle, runtimeEnvConfig)
 }

packages/next/build/webpack/plugins/terser-webpack-plugin/src/TaskRunner.js

@@ -1,13 +1,11 @@
 import { join } from 'path'
 import minify from './minify'
 import { promisify } from 'util'
-import workerFarm from 'worker-farm'
+import Worker from 'jest-worker'
 import { writeFile, readFile } from 'fs'
-import serialize from 'serialize-javascript'
-import { Sema } from 'async-sema'
 import mkdirp from 'mkdirp'
 
-const worker = require.resolve('./worker')
+const worker = require.resolve('./minify')
 const writeFileP = promisify(writeFile)
 const readFileP = promisify(readFile)
 
@@ -16,11 +14,9 @@ export default class TaskRunner {
     if (cache) {
       mkdirp.sync((this.cacheDir = join(distDir, 'cache', 'next-minifier')))
     }
-
     // In some cases cpus() returns undefined
     // https://github.com/nodejs/node/issues/19022
     this.maxConcurrentWorkers = cpus
-    this.sema = new Sema(cpus * 3)
   }
 
   run(tasks, callback) {
@@ -31,36 +27,18 @@ export default class TaskRunner {
     }
 
     if (this.maxConcurrentWorkers > 1) {
-      const workerOptions =
-        process.platform === 'win32'
-          ? {
-              maxConcurrentWorkers: this.maxConcurrentWorkers,
-              maxConcurrentCallsPerWorker: 1,
-            }
-          : { maxConcurrentWorkers: this.maxConcurrentWorkers }
-      this.workers = workerFarm(workerOptions, worker)
-      this.boundWorkers = (options, cb) => {
-        try {
-          this.workers(serialize(options), cb)
-        } catch (error) {
-          // worker-farm can fail with ENOMEM or something else
-          cb(error)
-        }
-      }
+      this.workers = new Worker(worker, {
+        enableWorkerThreads: true,
+        numWorkers: this.maxConcurrentWorkers,
+      })
+      this.boundWorkers = options => this.workers.default(options)
     } else {
-      this.boundWorkers = (options, cb) => {
-        try {
-          cb(null, minify(options))
-        } catch (error) {
-          cb(error)
-        }
-      }
+      this.boundWorkers = async options => minify(options)
     }
 
     let toRun = tasks.length
     const results = []
     const step = (index, data) => {
-      this.sema.release()
       toRun -= 1
       results[index] = data
 
@@ -71,37 +49,33 @@ export default class TaskRunner {
 
     tasks.forEach((task, index) => {
       const cachePath = this.cacheDir && join(this.cacheDir, task.cacheKey)
-
-      const enqueue = () => {
-        this.boundWorkers(task, (error, data) => {
-          const result = error ? { error } : data
+      const enqueue = async () => {
+        try {
+          const result = await this.boundWorkers(task)
           const done = () => step(index, result)
-
-          if (this.cacheDir && !result.error) {
-            writeFileP(cachePath, JSON.stringify(data), 'utf8')
+          if (cachePath) {
+            writeFileP(cachePath, JSON.stringify(result), 'utf8')
               .then(done)
               .catch(done)
-          } else {
-            done()
           }
-        })
+        } catch (error) {
+          step(index, { error })
+        }
       }
 
-      this.sema.acquire().then(() => {
-        if (this.cacheDir) {
-          readFileP(cachePath, 'utf8')
-            .then(data => step(index, JSON.parse(data)))
-            .catch(() => enqueue())
-        } else {
-          enqueue()
-        }
-      })
+      if (this.cacheDir) {
+        readFileP(cachePath, 'utf8')
+          .then(data => step(index, JSON.parse(data)))
+          .catch(() => enqueue())
+      } else {
+        enqueue()
+      }
     })
   }
 
   exit() {
     if (this.workers) {
-      workerFarm.end(this.workers)
+      this.workers.end()
     }
   }
 }

packages/next/build/webpack/plugins/terser-webpack-plugin/src/worker.js

@@ -80,6 +80,7 @@
     "find-up": "4.0.0",
     "fork-ts-checker-webpack-plugin": "1.3.4",
     "fresh": "0.5.2",
+    "jest-worker": "24.9.0",
     "launch-editor": "2.2.1",
     "loader-utils": "1.2.3",
     "mkdirp": "0.5.1",
@@ -88,7 +89,6 @@
     "prop-types-exact": "1.2.0",
     "react-error-overlay": "5.1.6",
     "react-is": "16.8.6",
-    "serialize-javascript": "1.7.0",
     "source-map": "0.6.1",
     "string-hash": "1.1.3",
     "strip-ansi": "5.2.0",
@@ -101,8 +101,7 @@
     "webpack": "4.39.0",
     "webpack-dev-middleware": "3.7.0",
     "webpack-hot-middleware": "2.25.0",
-    "webpack-sources": "1.3.0",
-    "worker-farm": "1.7.0"
+    "webpack-sources": "1.3.0"
   },
   "peerDependencies": {
     "react": "^16.6.0",

packages/next/package.json

@@ -8345,6 +8345,14 @@ jest-watcher@^24.8.0:
     jest-util "^24.8.0"
     string-length "^2.0.0"
 
+[email protected]:
+  version "24.9.0"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.9.0.tgz#5dbfdb5b2d322e98567898238a9697bcce67b3e5"
+  integrity sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw==
+  dependencies:
+    merge-stream "^2.0.0"
+    supports-color "^6.1.0"
+
 jest-worker@^24.6.0:
   version "24.6.0"
   resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-24.6.0.tgz#7f81ceae34b7cde0c9827a6980c35b7cdc0161b3"
@@ -12315,7 +12323,7 @@ sentence-case@^2.1.0:
     no-case "^2.2.0"
     upper-case-first "^1.1.2"
 
-serialize-javascript@1.7.0, serialize-javascript@^1.7.0:
+serialize-javascript@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-1.7.0.tgz#d6e0dfb2a3832a8c94468e6eb1db97e55a192a65"
   integrity sha512-ke8UG8ulpFOxO8f8gRYabHQe/ZntKlcig2Mp+8+URDP1D8vJZ0KUt7LYo07q25Z/+JVSgpr/cui9PIp5H6/+nA==
@@ -14267,7 +14275,7 @@ wordwrap@~1.0.0:
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb"
   integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=
 
-worker-farm@1.7.0, worker-farm@^1.7.0:
+worker-farm@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/worker-farm/-/worker-farm-1.7.0.tgz#26a94c5391bbca926152002f69b84a4bf772e5a8"
   integrity sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==