Build default language with locale (adds preserveLocale flag) (#1940)

Currently when navigating to a page that does not have a locale, it will default the english one, this breaks the navigation (it doesn't really break it, but everything including links now changes locale to english)
![2018-12-15 01 11 30](https://user-images.githubusercontent.com/11986564/50038916-c2886300-0007-11e9-8755-0ad42a12c67e.gif)
This PR adds a preserveLocale flag that will run a second build of the files not included in the locale that exist in the "en" locale but using the current locale site.json. 
Build time is considerably longer (*en* locale has almost 700 files, compared to others with 20 or less)
The result:
![2018-12-15 01 16 14](https://user-images.githubusercontent.com/11986564/50039015-86ee9880-0009-11e9-82ca-39405727747b.gif)

Open for discussion.

Author: flakolefluk

README.md

@@ -41,6 +41,15 @@ If you want to submit a new feature or a bugfix, the best way is to create the c
   * The top of each Markdown file is a block of YAML for page specific localization information that is passed to various templates.
   * The bulk of the Markdown content for each page is referenced as `{{{content}}}` in the corresponding template.
 
+### Serve/Build Options
+
+* `DEFAULT_LOCALE={{locale}} npm run serve` builds only the files present in the specified locale folder (will display 404 if file is not present)
+* `DEFAULT_LOCALE={{locale}} npm run serve -- --preserveLocale` builds the files present in the specified locale folder and adds the pages present in the english locale that are missing.
+* `npm run serve` builds all languages and returns 404 when a file is not present in the current locale
+* `npm run serve -- --preserveLocale` builds all languages and adds the pages present in the english locale that are missing.
+* Multiple locales can be built by using comma separated values in the DEFAULT_LOCALE option. i.e: DEFAULT_LOCALE=en,es,it
+
+
 ### Deployment
 
 Full set up is in https://github.com/nodejs/build/tree/master/setup/www minus secrets and certificates. The webhook is setup on GitHub for this project and talks to a small Node server on the host which does the work. See the [github-webhook](https://github.com/rvagg/github-webhook) package for this.

build.js

@@ -56,15 +56,16 @@ function i18nJSON (lang) {
 // This is the function where the actual magic happens. This contains the main
 // Metalsmith build cycle used for building a locale subsite, such as the
 // english one.
-function buildLocale (source, locale) {
+function buildLocale (source, locale, opts) {
   console.log(`[metalsmith] build/${locale} started`)
   console.time(`[metalsmith] build/${locale} finished`)
   const metalsmith = Metalsmith(__dirname)
   metalsmith
-    // Sets global metadata imported from the locale's respective site.json.
+  // Sets global metadata imported from the locale's respective site.json.
     .metadata({ site: i18nJSON(locale), project: source.project })
-    // Sets the build source as the locale folder.
+  // Sets the build source as the locale folder.
     .source(path.join(__dirname, 'locale', locale))
+    .use(withPreserveLocale(opts && opts.preserveLocale))
     // Extracts the main menu and sub-menu links form locale's site.json and
     // adds them to the metadata. This data is used in the navigation template
     .use(navigation(source.project.latestVersions))
@@ -170,6 +171,31 @@ function buildLocale (source, locale) {
   })
 }
 
+// This plugin reads the files present in the english locale that are missing
+// in the current locale being built (requires preserveLocale flag)
+function withPreserveLocale (preserveLocale) {
+  return (files, m, next) => {
+    if (preserveLocale) {
+      var path = m.path('locale/en')
+      m.read(path, function (err, newfiles) {
+        if (err) {
+          console.error(err)
+          return next(err)
+        }
+
+        Object.keys(newfiles).forEach(function (key) {
+          if (!files[key]) {
+            files[key] = newfiles[key]
+          }
+        })
+        next()
+      })
+    } else {
+      next()
+    }
+  }
+}
+
 // This middleware adds "Edit on GitHub" links to every editable page
 function githubLinks (options) {
   return (files, m, next) => {
@@ -268,7 +294,8 @@ function getSource (callback) {
 
 // This is where the build is orchestrated from, as indicated by the function
 // name. It brings together all build steps and dependencies and executes them.
-function fullBuild () {
+function fullBuild (opts) {
+  const { preserveLocale, selectedLocales } = opts
   // Build static files.
   copyStatic()
   // Build layouts
@@ -278,16 +305,19 @@ function fullBuild () {
 
     // Executes the build cycle for every locale.
     fs.readdir(path.join(__dirname, 'locale'), (e, locales) => {
-      locales.filter(junk.not).forEach((locale) => {
-        buildLocale(source, locale)
-      })
+      locales.filter(file => junk.not(file) && (selectedLocales ? selectedLocales.includes(file) : true))
+        .forEach((locale) => {
+          buildLocale(source, locale, { preserveLocale })
+        })
     })
   })
 }
 
 // Starts the build if the file was executed from the command line
 if (require.main === module) {
-  fullBuild()
+  const preserveLocale = process.argv.includes('--preserveLocale')
+  const selectedLocales = process.env.DEFAULT_LOCALE ? process.env.DEFAULT_LOCALE.toLowerCase().split(',') : process.env.DEFAULT_LOCALE
+  fullBuild({ selectedLocales, preserveLocale })
 }
 
 exports.getSource = getSource

package.json

@@ -5,10 +5,11 @@
   "homepage": "https://nodejs.org",
   "scripts": {
     "build": "node build.js",
+    "build:deploy":"node build.js --preserveLocale",
     "serve": "node server.js",
     "external:survey": "rsync -avz --exclude 'node_modules*' --exclude 'package*' external/survey-2018/ build/en/user-survey-report",
     "gzip": "find build -type f \\( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.xml' -o -name '*.json' \\) -exec gzip -kf9 {} \\;",
-    "deploy": "npm run load-schedule && npm run build && npm run external:survey && npm run gzip",
+    "deploy": "npm run load-schedule && npm run build:deploy && npm run external:survey && npm run gzip",
     "load-versions": "node scripts/load-versions.js",
     "load-schedule": "curl -sS https://github.com/raw/nodejs/Release/master/schedule.json -o source/schedule.json",
     "start": "npm run serve",

server.js

@@ -18,6 +18,9 @@ const build = require('./build')
 
 const port = process.env.PORT || 8080
 
+const selectedLocales = process.env.DEFAULT_LOCALE ? process.env.DEFAULT_LOCALE.toLowerCase().split(',') : process.env.DEFAULT_LOCALE
+const preserveLocale = process.argv.includes('--preserveLocale')
+
 // Watches for file changes in the locale, layout and static directories, and
 // rebuilds the modified one.
 const opts = {
@@ -34,38 +37,6 @@ const opts = {
 const locales = chokidar.watch(path.join(__dirname, 'locale'), opts)
 const layouts = chokidar.watch(path.join(__dirname, 'layouts'), opts)
 const statics = chokidar.watch(path.join(__dirname, 'static'), opts)
-const fs = require('fs')
-// Read all the langs under `locale`
-const SUPPORTED_LANGUAGES = new Set(fs.readdirSync(path.join(__dirname, 'locale')))
-
-// Redirect mechanism meant as a fix for languages where some pages
-// have not been translated yet, therefore redirect to the english equivalent,
-// which isn't the correct language, but better than a 404-page
-function redirectToEnglishUrl (req, res) {
-  return () => {
-    // Url should be case insensitive.(e.g: zh-CN = zh-cn),
-    // So we should make a convert to the lower case and check the route values.
-    let url = req.url.toLowerCase()
-    const splitedValues = url.split('/')
-    // For urls like `/blog`, add `en` before that
-    if (splitedValues.length === 2) {
-      splitedValues[0] = 'en'
-      url = splitedValues.join('/').trim()
-    } else if (splitedValues[1] !== 'en' && SUPPORTED_LANGUAGES.has(splitedValues[1])) {
-      // For urls like `/lang/docs/`.
-      // If we found the lang in our set, this means the specific lang
-      // doesn't have proper translated pages yet, so force the default
-      // lang to `en`.
-      splitedValues[1] = 'en'
-      url = splitedValues.join('/').trim()
-    }
-
-    res.writeHead(302, {
-      location: url
-    })
-    res.end()
-  }
-}
 
 // Gets the locale name by path.
 function getLocale (filePath) {
@@ -77,18 +48,24 @@ build.getSource((err, source) => {
   if (err) { throw err }
 
   locales.on('change', (filePath) => {
-    build.buildLocale(source, getLocale(filePath))
+    const locale = getLocale(filePath)
+    if (!selectedLocales || selectedLocales.includes(locale)) {
+      build.buildLocale(source, locale)
+    }
   })
   locales.on('add', (filePath) => {
-    build.buildLocale(source, getLocale(filePath))
-    locales.add(filePath)
+    const locale = getLocale(filePath)
+    if (!selectedLocales || selectedLocales.includes(locale)) {
+      build.buildLocale(source, locale)
+      locales.add(filePath)
+    }
   })
 })
 
-layouts.on('change', build.fullBuild)
+layouts.on('change', () => build.fullBuild({ selectedLocales, preserveLocale }))
 layouts.on('add', (filePath) => {
   layouts.add(filePath)
-  build.fullBuild()
+  build.fullBuild({ selectedLocales, preserveLocale })
 })
 
 statics.on('change', build.copyStatic)
@@ -97,15 +74,18 @@ statics.on('add', (filePath) => {
   build.copyStatic()
 })
 
+const mainLocale = (selectedLocales && selectedLocales[0]) || 'en'
+
 // Initializes the server and mounts it in the generated build directory.
 http.createServer((req, res) => {
   // If we are accessing the root, it should be redirected to `/en` instead.
   // We shouldn't get a 404 page.
+
   if (req.url === '/') {
-    req.url = '/en'
+    req.url = `/${mainLocale}`
   }
-  mount(req, res, redirectToEnglishUrl(req, res))
-}).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/en/\x1B[39m`))
+  mount(req, res)
+}).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/${mainLocale}/\x1B[39m`))
 
 // Start the initial build of static HTML pages
-build.fullBuild()
+build.fullBuild({ selectedLocales, preserveLocale })