From 6153962ee5ef363c52d623ca13c390389204a008 Mon Sep 17 00:00:00 2001 From: flakolefluk Date: Thu, 13 Dec 2018 01:37:10 -0300 Subject: [PATCH 1/5] adds preserveLocale flag, builds default language pages not included in locale --- build.js | 123 ++++++++++++++++++++++++++++++++++++++++++++++++--- package.json | 3 +- server.js | 4 +- 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/build.js b/build.js index 580a1190fbf21..0626ed799a001 100755 --- a/build.js +++ b/build.js @@ -56,7 +56,7 @@ function i18nJSON (lang) { // This is the function where the actual magic happens. This contains the main // Metalsmith build cycle used for building a locale subsite, such as the // english one. -function buildLocale (source, locale) { +function buildLocale (source, locale, opts) { console.log(`[metalsmith] build/${locale} started`) console.time(`[metalsmith] build/${locale} finished`) const metalsmith = Metalsmith(__dirname) @@ -164,12 +164,124 @@ function buildLocale (source, locale) { // This actually executes the build and stops the internal timer after // completion. - metalsmith.build((err) => { + metalsmith.build((err, files) => { if (err) { throw err } console.timeEnd(`[metalsmith] build/${locale} finished`) + if (opts && opts.preserveLocale) { + buildMissing(source, locale, files) + } }) } +function buildMissing (source, locale, localeFiles) { + console.log(`[metalsmith] build/${locale} missing files started`) + console.time(`[metalsmith] build/${locale} missing files finished`) + + const metalsmith = Metalsmith(__dirname) + metalsmith + .clean(false) + .metadata({ site: i18nJSON(locale), project: source.project }) + // Sets the build source as the DEFAULT_LANG (en) locale folder. + .source(path.join(__dirname, 'locale', DEFAULT_LANG)) + // Ignores files already built with current locale + .ignore([...Object.keys(localeFiles).map(file => + path.join(__dirname, 'locale', 'en', file.replace('.html', '.md')) + ), ...Object.keys(localeFiles) + .filter(key => `${localeFiles[key].path}/index.html` === key) + .map(key => path.join(__dirname, 'locale', 'en', `${localeFiles[key].path}.md`))]) + .use(navigation(source.project.latestVersions)) + .use(collections({ + blog: { + pattern: 'blog/**/*.md', + sortBy: 'date', + reverse: true, + refer: false + }, + blogAnnounce: { + pattern: 'blog/announcements/*.md', + sortBy: 'date', + reverse: true, + refer: false + }, + blogReleases: { + pattern: 'blog/release/*.md', + sortBy: 'date', + reverse: true, + refer: false + }, + blogVulnerability: { + pattern: 'blog/vulnerability/*.md', + sortBy: 'date', + reverse: true, + refer: false + }, + lastWeekly: { + pattern: 'blog/weekly-updates/*.md', + sortBy: 'date', + reverse: true, + refer: false, + limit: 1 + }, + knowledgeBase: { + pattern: 'knowledge/**/*.md', + refer: false + }, + guides: { + pattern: 'docs/guides/!(index).md' + } + })) + .use(pagination({ + path: 'blog/year', + iteratee: (post, idx) => ({ + post, + displaySummary: idx < 10 + }) + })) + .use(markdown(markedOptions)) + .use(githubLinks({ locale: locale })) + .use(prism()) + .use(permalinks({ + relative: false + })) + .use(feed({ + collection: 'blog', + destination: 'feed/blog.xml', + title: 'Node.js Blog' + })) + .use(feed({ + collection: 'blogAnnounce', + destination: 'feed/announce.xml', + title: 'Node.js Announcements' + })) + .use(feed({ + collection: 'blogReleases', + destination: 'feed/releases.xml', + title: 'Node.js Blog: Releases' + })) + .use(feed({ + collection: 'blogVulnerability', + destination: 'feed/vulnerability.xml', + title: 'Node.js Blog: Vulnerability Reports' + })) + .use(discoverPartials({ + directory: 'layouts/partials', + pattern: /\.hbs$/ + })) + .use(discoverHelpers({ + directory: 'scripts/helpers', + pattern: /\.js$/ + })) + .use(layouts()) + .destination(path.join(__dirname, 'build', locale)) + + if (locale !== DEFAULT_LANG) { + metalsmith.build((err) => { + if (err) { throw err } + console.timeEnd(`[metalsmith] build/${locale} missing files finished`) + }) + } +} + // This middleware adds "Edit on GitHub" links to every editable page function githubLinks (options) { return (files, m, next) => { @@ -267,7 +379,7 @@ function getSource (callback) { // This is where the build is orchestrated from, as indicated by the function // name. It brings together all build steps and dependencies and executes them. -function fullBuild () { +function fullBuild (opts) { // Build static files. copyStatic() // Build layouts @@ -278,7 +390,7 @@ function fullBuild () { // Executes the build cycle for every locale. fs.readdir(path.join(__dirname, 'locale'), (e, locales) => { locales.filter(junk.not).forEach((locale) => { - buildLocale(source, locale) + buildLocale(source, locale, opts) }) }) }) @@ -286,7 +398,8 @@ function fullBuild () { // Starts the build if the file was executed from the command line if (require.main === module) { - fullBuild() + const preserveLocale = process.argv.includes('--preserveLocale') + fullBuild({ preserveLocale }) } exports.getSource = getSource diff --git a/package.json b/package.json index d18f3049c3f34..be48134f47e92 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,11 @@ "homepage": "https://nodejs.org", "scripts": { "build": "node build.js", + "build:deploy":"node build.js --preserveLocale", "serve": "node server.js", "external:survey": "rsync -avz --exclude 'node_modules*' --exclude 'package*' external/survey-2018/ build/en/user-survey-report", "gzip": "find build -type f \\( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.xml' -o -name '*.json' \\) -exec gzip -kf9 {} \\;", - "deploy": "npm run load-schedule && npm run build && npm run external:survey && npm run gzip", + "deploy": "npm run load-schedule && npm run build:deploy && npm run external:survey && npm run gzip", "load-versions": "node scripts/load-versions.js", "load-schedule": "curl -sS https://raw.githubusercontent.com/nodejs/Release/master/schedule.json -o source/schedule.json", "start": "npm run serve", diff --git a/server.js b/server.js index df975c8cfd180..96ac4598865fa 100644 --- a/server.js +++ b/server.js @@ -18,6 +18,8 @@ const build = require('./build') const port = process.env.PORT || 8080 +const preserveLocale = process.argv.includes('--preserveLocale') + // Watches for file changes in the locale, layout and static directories, and // rebuilds the modified one. const opts = { @@ -108,4 +110,4 @@ http.createServer((req, res) => { }).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/en/\x1B[39m`)) // Start the initial build of static HTML pages -build.fullBuild() +build.fullBuild({ preserveLocale }) From cb37e5abc952d9205b51bb4d4d10935d5de861c8 Mon Sep 17 00:00:00 2001 From: flakolefluk Date: Thu, 27 Dec 2018 02:08:07 -0300 Subject: [PATCH 2/5] missing files plugin, configure languages to build --- build.js | 145 ++++++++++++------------------------------------------ server.js | 53 ++++++-------------- 2 files changed, 45 insertions(+), 153 deletions(-) diff --git a/build.js b/build.js index 0626ed799a001..541c7aade7a75 100755 --- a/build.js +++ b/build.js @@ -61,10 +61,11 @@ function buildLocale (source, locale, opts) { console.time(`[metalsmith] build/${locale} finished`) const metalsmith = Metalsmith(__dirname) metalsmith - // Sets global metadata imported from the locale's respective site.json. + // Sets global metadata imported from the locale's respective site.json. .metadata({ site: i18nJSON(locale), project: source.project }) - // Sets the build source as the locale folder. + // Sets the build source as the locale folder. .source(path.join(__dirname, 'locale', locale)) + .use(withPreserveLocale(opts && opts.preserveLocale)) // Extracts the main menu and sub-menu links form locale's site.json and // adds them to the metadata. This data is used in the navigation template .use(navigation(source.project.latestVersions)) @@ -118,7 +119,7 @@ function buildLocale (source, locale, opts) { }) })) .use(markdown(markedOptions)) - .use(githubLinks({ locale: locale })) + .use(githubLinks({ locale })) .use(prism()) // Set pretty permalinks, we don't want .html suffixes everywhere. .use(permalinks({ @@ -167,118 +168,31 @@ function buildLocale (source, locale, opts) { metalsmith.build((err, files) => { if (err) { throw err } console.timeEnd(`[metalsmith] build/${locale} finished`) - if (opts && opts.preserveLocale) { - buildMissing(source, locale, files) - } }) } -function buildMissing (source, locale, localeFiles) { - console.log(`[metalsmith] build/${locale} missing files started`) - console.time(`[metalsmith] build/${locale} missing files finished`) +// This plugin reads the files present in the english locale that are missing +// in the current locale being built (requires preserveLocale flag) +function withPreserveLocale (preserveLocale) { + return (files, m, next) => { + if (preserveLocale) { + var path = m.path('locale/en') + m.read(path, function (err, newfiles) { + if (err) { + console.error(err) + return next(err) + } - const metalsmith = Metalsmith(__dirname) - metalsmith - .clean(false) - .metadata({ site: i18nJSON(locale), project: source.project }) - // Sets the build source as the DEFAULT_LANG (en) locale folder. - .source(path.join(__dirname, 'locale', DEFAULT_LANG)) - // Ignores files already built with current locale - .ignore([...Object.keys(localeFiles).map(file => - path.join(__dirname, 'locale', 'en', file.replace('.html', '.md')) - ), ...Object.keys(localeFiles) - .filter(key => `${localeFiles[key].path}/index.html` === key) - .map(key => path.join(__dirname, 'locale', 'en', `${localeFiles[key].path}.md`))]) - .use(navigation(source.project.latestVersions)) - .use(collections({ - blog: { - pattern: 'blog/**/*.md', - sortBy: 'date', - reverse: true, - refer: false - }, - blogAnnounce: { - pattern: 'blog/announcements/*.md', - sortBy: 'date', - reverse: true, - refer: false - }, - blogReleases: { - pattern: 'blog/release/*.md', - sortBy: 'date', - reverse: true, - refer: false - }, - blogVulnerability: { - pattern: 'blog/vulnerability/*.md', - sortBy: 'date', - reverse: true, - refer: false - }, - lastWeekly: { - pattern: 'blog/weekly-updates/*.md', - sortBy: 'date', - reverse: true, - refer: false, - limit: 1 - }, - knowledgeBase: { - pattern: 'knowledge/**/*.md', - refer: false - }, - guides: { - pattern: 'docs/guides/!(index).md' - } - })) - .use(pagination({ - path: 'blog/year', - iteratee: (post, idx) => ({ - post, - displaySummary: idx < 10 + Object.keys(newfiles).forEach(function (key) { + if (!files[key]) { + files[key] = newfiles[key] + } + }) + next() }) - })) - .use(markdown(markedOptions)) - .use(githubLinks({ locale: locale })) - .use(prism()) - .use(permalinks({ - relative: false - })) - .use(feed({ - collection: 'blog', - destination: 'feed/blog.xml', - title: 'Node.js Blog' - })) - .use(feed({ - collection: 'blogAnnounce', - destination: 'feed/announce.xml', - title: 'Node.js Announcements' - })) - .use(feed({ - collection: 'blogReleases', - destination: 'feed/releases.xml', - title: 'Node.js Blog: Releases' - })) - .use(feed({ - collection: 'blogVulnerability', - destination: 'feed/vulnerability.xml', - title: 'Node.js Blog: Vulnerability Reports' - })) - .use(discoverPartials({ - directory: 'layouts/partials', - pattern: /\.hbs$/ - })) - .use(discoverHelpers({ - directory: 'scripts/helpers', - pattern: /\.js$/ - })) - .use(layouts()) - .destination(path.join(__dirname, 'build', locale)) - - if (locale !== DEFAULT_LANG) { - metalsmith.build((err) => { - if (err) { throw err } - console.timeEnd(`[metalsmith] build/${locale} missing files finished`) - }) + } else { + next() + } } } @@ -380,6 +294,7 @@ function getSource (callback) { // This is where the build is orchestrated from, as indicated by the function // name. It brings together all build steps and dependencies and executes them. function fullBuild (opts) { + const { preserveLocale, selectedLocales } = opts // Build static files. copyStatic() // Build layouts @@ -389,9 +304,10 @@ function fullBuild (opts) { // Executes the build cycle for every locale. fs.readdir(path.join(__dirname, 'locale'), (e, locales) => { - locales.filter(junk.not).forEach((locale) => { - buildLocale(source, locale, opts) - }) + locales.filter(file => junk.not(file) && (selectedLocales ? selectedLocales.includes(file) : true)) + .forEach((locale) => { + buildLocale(source, locale, { preserveLocale }) + }) }) }) } @@ -399,7 +315,8 @@ function fullBuild (opts) { // Starts the build if the file was executed from the command line if (require.main === module) { const preserveLocale = process.argv.includes('--preserveLocale') - fullBuild({ preserveLocale }) + const selectedLocales = process.env.DEFAULT_LOCALE ? process.env.DEFAULT_LOCALE.toLowerCase().split(',') : process.env.DEFAULT_LOCALE + fullBuild({ selectedLocales, preserveLocale }) } exports.getSource = getSource diff --git a/server.js b/server.js index 96ac4598865fa..7f616ed94f160 100644 --- a/server.js +++ b/server.js @@ -18,6 +18,7 @@ const build = require('./build') const port = process.env.PORT || 8080 +const selectedLocales = process.env.DEFAULT_LOCALE ? process.env.DEFAULT_LOCALE.toLowerCase().split(',') : process.env.DEFAULT_LOCALE const preserveLocale = process.argv.includes('--preserveLocale') // Watches for file changes in the locale, layout and static directories, and @@ -36,38 +37,6 @@ const opts = { const locales = chokidar.watch(path.join(__dirname, 'locale'), opts) const layouts = chokidar.watch(path.join(__dirname, 'layouts'), opts) const statics = chokidar.watch(path.join(__dirname, 'static'), opts) -const fs = require('fs') -// Read all the langs under `locale` -const SUPPORTED_LANGUAGES = new Set(fs.readdirSync(path.join(__dirname, 'locale'))) - -// Redirect mechanism meant as a fix for languages where some pages -// have not been translated yet, therefore redirect to the english equivalent, -// which isn't the correct language, but better than a 404-page -function redirectToEnglishUrl (req, res) { - return () => { - // Url should be case insensitive.(e.g: zh-CN = zh-cn), - // So we should make a convert to the lower case and check the route values. - let url = req.url.toLowerCase() - const splitedValues = url.split('/') - // For urls like `/blog`, add `en` before that - if (splitedValues.length === 2) { - splitedValues[0] = 'en' - url = splitedValues.join('/').trim() - } else if (splitedValues[1] !== 'en' && SUPPORTED_LANGUAGES.has(splitedValues[1])) { - // For urls like `/lang/docs/`. - // If we found the lang in our set, this means the specific lang - // doesn't have proper translated pages yet, so force the default - // lang to `en`. - splitedValues[1] = 'en' - url = splitedValues.join('/').trim() - } - - res.writeHead(302, { - location: url - }) - res.end() - } -} // Gets the locale name by path. function getLocale (filePath) { @@ -79,18 +48,24 @@ build.getSource((err, source) => { if (err) { throw err } locales.on('change', (filePath) => { - build.buildLocale(source, getLocale(filePath)) + const locale = getLocale(filePath) + if (!selectedLocales || selectedLocales.includes(locale)) { + build.buildLocale(source, locale) + } }) locales.on('add', (filePath) => { - build.buildLocale(source, getLocale(filePath)) - locales.add(filePath) + const locale = getLocale(filePath) + if (!selectedLocales || selectedLocales.includes(locale)) { + build.buildLocale(source, locale) + locales.add(filePath) + } }) }) -layouts.on('change', build.fullBuild) +layouts.on('change', () => build.fullBuild({ selectedLocales, preserveLocale })) layouts.on('add', (filePath) => { layouts.add(filePath) - build.fullBuild() + build.fullBuild({ selectedLocales, preserveLocale }) }) statics.on('change', build.copyStatic) @@ -106,8 +81,8 @@ http.createServer((req, res) => { if (req.url === '/') { req.url = '/en' } - mount(req, res, redirectToEnglishUrl(req, res)) + mount(req, res) }).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/en/\x1B[39m`)) // Start the initial build of static HTML pages -build.fullBuild({ preserveLocale }) +build.fullBuild({ selectedLocales, preserveLocale }) From 539c3d69238e584d12a759f730cd569c4d677b03 Mon Sep 17 00:00:00 2001 From: flakolefluk Date: Sat, 19 Jan 2019 14:33:21 -0300 Subject: [PATCH 3/5] display default language --- server.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index 7f616ed94f160..7b70ab4584b2c 100644 --- a/server.js +++ b/server.js @@ -74,15 +74,18 @@ statics.on('add', (filePath) => { build.copyStatic() }) +const mainLocale = (selectedLocales && selectedLocales[0]) || 'en' + // Initializes the server and mounts it in the generated build directory. http.createServer((req, res) => { // If we are accessing the root, it should be redirected to `/en` instead. // We shouldn't get a 404 page. + if (req.url === '/') { - req.url = '/en' + req.url = `/${mainLocale}` } mount(req, res) -}).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/en/\x1B[39m`)) +}).listen(port, () => console.log(`\x1B[32mServer running at http://localhost:${port}/${mainLocale}/\x1B[39m`)) // Start the initial build of static HTML pages build.fullBuild({ selectedLocales, preserveLocale }) From c16b5ef57a35da65ed92c342ff94a828ec00d7c9 Mon Sep 17 00:00:00 2001 From: flakolefluk Date: Sat, 19 Jan 2019 14:49:51 -0300 Subject: [PATCH 4/5] add serve/build options --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 9e77ab16b9cbd..4ff0db62109b0 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,15 @@ If you want to submit a new feature or a bugfix, the best way is to create the c * The top of each Markdown file is a block of YAML for page specific localization information that is passed to various templates. * The bulk of the Markdown content for each page is referenced as `{{{content}}}` in the corresponding template. +### Serve/Build Options + +* `DEFAULT_LOCALE={{locale}} npm run serve` builds only the files present in the specified locale folder (will display 404 if file is not present) +* `DEFAULT_LOCALE={{locale}} npm run serve -- --preserveLocale` builds the files present in the specified locale folder and adds the pages present in the english locale that are missing. +* `npm run serve` builds all languages and returns 404 when a file is not present in the current locale +* `npm run serve -- --preserveLocale` builds all languages and adds the pages present in the english locale that are missing. +* Multiple locales can be built by using comma separated values in the DEFAULT_LOCALE option. i.e: DEFAULT_LOCALE=en,es,it + + ### Deployment Full set up is in https://github.com/nodejs/build/tree/master/setup/www minus secrets and certificates. The webhook is setup on GitHub for this project and talks to a small Node server on the host which does the work. See the [github-webhook](https://github.com/rvagg/github-webhook) package for this. From d851b97e76e62dcfaea7bb5b7b3d8cbf4ee11e2d Mon Sep 17 00:00:00 2001 From: flakolefluk Date: Sun, 20 Jan 2019 11:02:54 -0300 Subject: [PATCH 5/5] remove files from build function --- build.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.js b/build.js index b8003f9bb2fa9..476e86a7e58f1 100755 --- a/build.js +++ b/build.js @@ -165,7 +165,7 @@ function buildLocale (source, locale, opts) { // This actually executes the build and stops the internal timer after // completion. - metalsmith.build((err, files) => { + metalsmith.build((err) => { if (err) { throw err } console.timeEnd(`[metalsmith] build/${locale} finished`) })