Reinclude --loader temporarily, moving new work to Phase3

.eslintrc.js

@@ -41,6 +41,7 @@ module.exports = {
         'test/es-module/test-esm-type-flag.js',
         'test/es-module/test-esm-type-flag-alias.js',
         '*.mjs',
+        'test/es-module/test-esm-example-loader.js',
       ],
       parserOptions: { sourceType: 'module' },
     },

doc/api/cli.md

@@ -268,6 +268,13 @@ default) is not firewall-protected.**
 
 See the [debugging security implications][] section for more information.
 
+### `--loader=file`
+<!-- YAML
+added: v9.0.0
+-->
+
+Specify the `file` of the custom [experimental ECMAScript Module][] loader.
+
 ### `--max-http-header-size=size`
 <!-- YAML
 added: v11.6.0
@@ -716,6 +723,7 @@ Node.js options that are allowed are:
 - `--inspect`
 - `--inspect-brk`
 - `--inspect-port`
+- `--loader`
 - `--max-http-header-size`
 - `--napi-modules`
 - `--no-deprecation`
@@ -903,6 +911,7 @@ greater than `4` (its current default value). For more information, see the
 [debugger]: debugger.html
 [debugging security implications]: https://nodejs.org/en/docs/guides/debugging-getting-started/#security-implications
 [emit_warning]: process.html#process_process_emitwarning_warning_type_code_ctor
+[experimental ECMAScript Module]: esm.html#esm_experimental_loader_hooks
 [libuv threadpool documentation]: http://docs.libuv.org/en/latest/threadpool.html
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
 [secureProtocol]: tls.html#tls_tls_createsecurecontext_options

doc/api/esm.md

@@ -289,6 +289,128 @@ READ_PACKAGE_JSON(_packageURL_)
 >    1. Throw an _Invalid Package Configuration_ error.
 > 1. Return the parsed JSON source of the file at _pjsonURL_.
 
+## Experimental Loader hooks
+
+**Note: This API is currently being redesigned and will still change.**.
+
+<!-- type=misc -->
+
+To customize the default module resolution, loader hooks can optionally be
+provided via a `--loader ./loader-name.mjs` argument to Node.js.
+
+When hooks are used they only apply to ES module loading and not to any
+CommonJS modules loaded.
+
+### Resolve hook
+
+The resolve hook returns the resolved file URL and module format for a
+given module specifier and parent file URL:
+
+```js
+const baseURL = new URL('file://');
+baseURL.pathname = `${process.cwd()}/`;
+
+export async function resolve(specifier,
+                              parentModuleURL = baseURL,
+                              defaultResolver) {
+  return {
+    url: new URL(specifier, parentModuleURL).href,
+    format: 'esm'
+  };
+}
+```
+
+The `parentModuleURL` is provided as `undefined` when performing main Node.js
+load itself.
+
+The default Node.js ES module resolution function is provided as a third
+argument to the resolver for easy compatibility workflows.
+
+In addition to returning the resolved file URL value, the resolve hook also
+returns a `format` property specifying the module format of the resolved
+module. This can be one of the following:
+
+| `format` | Description |
+| --- | --- |
+| `'module'` | Load a standard JavaScript module |
+| `'commonjs'` | Load a Node.js CommonJS module |
+| `'builtin'` | Load a Node.js builtin module |
+| `'dynamic'` | Use a [dynamic instantiate hook][] |
+
+For example, a dummy loader to load JavaScript restricted to browser resolution
+rules with only JS file extension and Node.js builtin modules support could
+be written:
+
+```js
+import path from 'path';
+import process from 'process';
+import Module from 'module';
+
+const builtins = Module.builtinModules;
+const JS_EXTENSIONS = new Set(['.js', '.mjs']);
+
+const baseURL = new URL('file://');
+baseURL.pathname = `${process.cwd()}/`;
+
+export function resolve(specifier, parentModuleURL = baseURL, defaultResolve) {
+  if (builtins.includes(specifier)) {
+    return {
+      url: specifier,
+      format: 'builtin'
+    };
+  }
+  if (/^\.{0,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) {
+    // For node_modules support:
+    // return defaultResolve(specifier, parentModuleURL);
+    throw new Error(
+      `imports must begin with '/', './', or '../'; '${specifier}' does not`);
+  }
+  const resolved = new URL(specifier, parentModuleURL);
+  const ext = path.extname(resolved.pathname);
+  if (!JS_EXTENSIONS.has(ext)) {
+    throw new Error(
+      `Cannot load file with non-JavaScript file extension ${ext}.`);
+  }
+  return {
+    url: resolved.href,
+    format: 'esm'
+  };
+}
+```
+
+With this loader, running:
+
+```console
+NODE_OPTIONS='--experimental-modules --loader ./custom-loader.mjs' node x.js
+```
+
+would load the module `x.js` as an ES module with relative resolution support
+(with `node_modules` loading skipped in this example).
+
+### Dynamic instantiate hook
+
+To create a custom dynamic module that doesn't correspond to one of the
+existing `format` interpretations, the `dynamicInstantiate` hook can be used.
+This hook is called only for modules that return `format: 'dynamic'` from
+the `resolve` hook.
+
+```js
+export async function dynamicInstantiate(url) {
+  return {
+    exports: ['customExportName'],
+    execute: (exports) => {
+      // Get and set functions provided for pre-allocated export names
+      exports.customExportName.set('value');
+    }
+  };
+}
+```
+
+With the list of module exports provided upfront, the `execute` function will
+then be called at the exact point of module evaluation order for that module
+in the import tree.
+
 [Node.js EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md
+[dynamic instantiate hook]: #esm_dynamic_instantiate_hook
 [`module.createRequireFromPath()`]: modules.html#modules_module_createrequirefrompath_filename
 [ESM Minimal Kernel]: https://github.com/nodejs/modules/blob/master/doc/plan-for-new-modules-implementation.md

lib/internal/process/esm_loader.js

@@ -7,13 +7,15 @@ const {
   ERR_INVALID_TYPE_FLAG,
   ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING,
 } = require('internal/errors').codes;
+const { emitExperimentalWarning } = require('internal/util');
 
 const type = require('internal/options').getOptionValue('--type');
 if (type && type !== 'commonjs' && type !== 'module')
   throw new ERR_INVALID_TYPE_FLAG(type);
 exports.typeFlag = type;
 
 const { Loader } = require('internal/modules/esm/loader');
+const { pathToFileURL } = require('internal/url');
 const {
   wrapToModuleMap,
 } = require('internal/vm/source_text_module');
@@ -44,8 +46,16 @@ exports.loaderPromise = new Promise((resolve) => loaderResolve = resolve);
 exports.ESMLoader = undefined;
 
 exports.initializeLoader = function(cwd, userLoader) {
-  const ESMLoader = new Loader();
+  let ESMLoader = new Loader();
   const loaderPromise = (async () => {
+    if (userLoader) {
+      emitExperimentalWarning('--loader');
+      const hooks = await ESMLoader.import(
+        userLoader, pathToFileURL(`${cwd}/`).href);
+      ESMLoader = new Loader();
+      ESMLoader.hook(hooks);
+      exports.ESMLoader = ESMLoader;
+    }
     return ESMLoader;
   })();
   loaderResolve(loaderPromise);

src/node_options.cc

@@ -95,6 +95,10 @@ void PerIsolateOptions::CheckOptions(std::vector<std::string>* errors) {
 }
 
 void EnvironmentOptions::CheckOptions(std::vector<std::string>* errors) {
+  if (!userland_loader.empty() && !experimental_modules) {
+    errors->push_back("--loader requires --experimental-modules be enabled");
+  }
+
   if (syntax_check_only && has_eval_string) {
     errors->push_back("either --check or --eval can be used, not both");
   }
@@ -236,6 +240,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "(default: llhttp).",
             &EnvironmentOptions::http_parser,
             kAllowedInEnvironment);
+  AddOption("--loader",
+            "(with --experimental-modules) use the specified file as a "
+            "custom loader",
+            &EnvironmentOptions::userland_loader,
+            kAllowedInEnvironment);
   AddOption("--no-deprecation",
             "silence deprecation warnings",
             &EnvironmentOptions::no_deprecation,

src/node_options.h

@@ -105,6 +105,7 @@ class EnvironmentOptions : public Options {
   bool trace_deprecation = false;
   bool trace_sync_io = false;
   bool trace_warnings = false;
+  std::string userland_loader;
 
   bool syntax_check_only = false;
   bool has_eval_string = false;

test/es-module/test-esm-example-loader.js

@@ -0,0 +1,6 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/example-loader.mjs
+/* eslint-disable node-core/required-modules */
+import assert from 'assert';
+import ok from '../fixtures/es-modules/test-esm-ok.mjs';
+
+assert(ok);

test/es-module/test-esm-loader-dependency.mjs

@@ -0,0 +1,5 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/loader-with-dep.mjs
+/* eslint-disable node-core/required-modules */
+import '../fixtures/es-modules/test-esm-ok.mjs';
+
+// We just test that this module doesn't fail loading

test/es-module/test-esm-loader-invalid-format.mjs

@@ -0,0 +1,12 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/loader-invalid-format.mjs
+/* eslint-disable node-core/required-modules */
+import { expectsError, mustCall } from '../common/index.mjs';
+import assert from 'assert';
+
+import('../fixtures/es-modules/test-esm-ok.mjs')
+.then(assert.fail, expectsError({
+  code: 'ERR_INVALID_RETURN_PROPERTY_VALUE',
+  message: 'Expected string to be returned for the "format" from the ' +
+           '"loader resolve" function but got type undefined.'
+}))
+.then(mustCall());

test/es-module/test-esm-loader-invalid-url.mjs

@@ -0,0 +1,14 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/loader-invalid-url.mjs
+/* eslint-disable node-core/required-modules */
+
+import { expectsError, mustCall } from '../common/index.mjs';
+import assert from 'assert';
+
+import('../fixtures/es-modules/test-esm-ok.mjs')
+.then(assert.fail, expectsError({
+  code: 'ERR_INVALID_RETURN_PROPERTY',
+  message: 'Expected a valid url to be returned for the "url" from the ' +
+           '"loader resolve" function but got ' +
+           '../fixtures/es-modules/test-esm-ok.mjs.'
+}))
+.then(mustCall());

test/es-module/test-esm-loader-missing-dynamic-instantiate-hook.mjs

@@ -0,0 +1,10 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/missing-dynamic-instantiate-hook.mjs
+/* eslint-disable node-core/required-modules */
+
+import { expectsError } from '../common/index.mjs';
+
+import('test').catch(expectsError({
+  code: 'ERR_MISSING_DYNAMIC_INSTANTIATE_HOOK',
+  message: 'The ES Module loader may not return a format of \'dynamic\' ' +
+    'when no dynamicInstantiate function was provided'
+}));

test/es-module/test-esm-named-exports.mjs

@@ -0,0 +1,9 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs
+/* eslint-disable node-core/required-modules */
+import '../common/index.mjs';
+import { readFile } from 'fs';
+import assert from 'assert';
+import ok from '../fixtures/es-modules/test-esm-ok.mjs';
+
+assert(ok);
+assert(readFile);

test/es-module/test-esm-preserve-symlinks-not-found-plain.mjs

@@ -0,0 +1,3 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/not-found-assert-loader.mjs
+/* eslint-disable node-core/required-modules */
+import './not-found.js';

test/es-module/test-esm-preserve-symlinks-not-found.mjs

@@ -0,0 +1,3 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/not-found-assert-loader.mjs
+/* eslint-disable node-core/required-modules */
+import './not-found.mjs';

test/es-module/test-esm-resolve-hook.mjs

@@ -0,0 +1,8 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/js-loader.mjs
+/* eslint-disable node-core/required-modules */
+import { namedExport } from '../fixtures/es-module-loaders/js-as-esm.js';
+import assert from 'assert';
+import ok from '../fixtures/es-modules/test-esm-ok.mjs';
+
+assert(ok);
+assert(namedExport);

test/es-module/test-esm-shared-loader-dep.mjs

@@ -0,0 +1,11 @@
+// Flags: --experimental-modules --loader ./test/fixtures/es-module-loaders/loader-shared-dep.mjs
+/* eslint-disable node-core/required-modules */
+import { createRequire } from '../common/index.mjs';
+
+import assert from 'assert';
+import '../fixtures/es-modules/test-esm-ok.mjs';
+
+const require = createRequire(import.meta.url);
+const dep = require('../fixtures/es-module-loaders/loader-dep.js');
+
+assert.strictEqual(dep.format, 'module');

test/es-module/test-esm-throw-undefined.mjs

@@ -0,0 +1,16 @@
+// Flags: --experimental-modules
+/* eslint-disable node-core/required-modules */
+
+import '../common/index.mjs';
+import assert from 'assert';
+
+async function doTest() {
+  await assert.rejects(
+    async () => {
+      await import('../fixtures/es-module-loaders/throw-undefined.mjs');
+    },
+    (e) => e === undefined
+  );
+}
+
+doTest();

test/fixtures/es-module-loaders/builtin-named-exports-loader.mjs

@@ -0,0 +1,29 @@
+import module from 'module';
+
+const builtins = new Set(
+  Object.keys(process.binding('natives')).filter(str =>
+    /^(?!(?:internal|node|v8)\/)/.test(str))
+);
+
+export function dynamicInstantiate(url) {
+  const builtinInstance = module._load(url.substr(5));
+  const builtinExports = ['default', ...Object.keys(builtinInstance)];
+  return {
+    exports: builtinExports,
+    execute: exports => {
+      for (let name of builtinExports)
+        exports[name].set(builtinInstance[name]);
+      exports.default.set(builtinInstance);
+    }
+  };
+}
+
+export function resolve(specifier, base, defaultResolver) {
+  if (builtins.has(specifier)) {
+    return {
+      url: `node:${specifier}`,
+      format: 'dynamic'
+    };
+  }
+  return defaultResolver(specifier, base);
+}