Skip to content

Commit 537402f

Browse files
guybedfordguybedford
committed
module: resolve and instantiate loader pipeline hooks
This enables a --loader flag for Node, which can provide custom "resolve" and "dynamicInstantiate" methods for custom ES module loading. In the process, module providers have been converted from classes into functions and the module APIs have been made to pass URL strings over objects.
1 parent 85a5a2c commit 537402f

23 files changed

+507
-198
lines changed

‎.eslintrc.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ parserOptions:
1111
ecmaVersion: 2017
1212

1313
overrides:
14-
- files: ["doc/api/esm.md", "*.mjs"]
14+
- files: ["doc/api/esm.md", "*.mjs", "test/es-module/test-esm-example-loader.js"]
1515
parserOptions:
1616
sourceType: module
1717

@@ -117,6 +117,7 @@ rules:
117117
keyword-spacing: error
118118
linebreak-style: [error, unix]
119119
max-len: [error, {code: 80,
120+
ignorePattern: "^\/\/ Flags:",
120121
ignoreRegExpLiterals: true,
121122
ignoreUrls: true,
122123
tabWidth: 2}]

‎doc/api/esm.md

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,4 +98,111 @@ fs.readFile('./foo.txt', (err, body) => {
9898
});
9999
```
100100

101+
## Loader hooks
102+
103+
<!-- type=misc -->
104+
105+
To customize the default module resolution, loader hooks can optionally be
106+
provided via a `--loader ./loader-name.mjs` argument to Node.
107+
108+
When hooks are used they only apply to ES module loading and not to any
109+
CommonJS modules loaded.
110+
111+
### Resolve hook
112+
113+
The resolve hook returns the resolved file URL and module format for a
114+
given module specifier and parent file URL:
115+
116+
```js
117+
import url from 'url';
118+
119+
export async function resolve(specifier, parentModuleURL, defaultResolver) {
120+
return {
121+
url: new URL(specifier, parentModuleURL).href,
122+
format: 'esm'
123+
};
124+
}
125+
```
126+
127+
The default NodeJS ES module resolution function is provided as a third
128+
argument to the resolver for easy compatibility workflows.
129+
130+
In addition to returning the resolved file URL value, the resolve hook also
131+
returns a `format` property specifying the module format of the resolved
132+
module. This can be one of `"esm"`, `"cjs"`, `"json"`, `"builtin"` or
133+
`"addon"`.
134+
135+
For example a dummy loader to load JavaScript restricted to browser resolution
136+
rules with only JS file extension and Node builtin modules support could
137+
be written:
138+
139+
```js
140+
import url from 'url';
141+
import path from 'path';
142+
import process from 'process';
143+
144+
const builtins = new Set(
145+
Object.keys(process.binding('natives')).filter((str) =>
146+
/^(?!(?:internal|node|v8)\/)/.test(str))
147+
);
148+
const JS_EXTENSIONS = new Set(['.js', '.mjs']);
149+
150+
export function resolve(specifier, parentModuleURL/*, defaultResolve */) {
151+
if (builtins.has(specifier)) {
152+
return {
153+
url: specifier,
154+
format: 'builtin'
155+
};
156+
}
157+
if (/^\.{0,2}[/]/.test(specifier) !== true && !specifier.startsWith('file:')) {
158+
// For node_modules support:
159+
// return defaultResolve(specifier, parentModuleURL);
160+
throw new Error(
161+
`imports must begin with '/', './', or '../'; '${specifier}' does not`);
162+
}
163+
const resolved = new url.URL(specifier, parentModuleURL);
164+
const ext = path.extname(resolved.pathname);
165+
if (!JS_EXTENSIONS.has(ext)) {
166+
throw new Error(
167+
`Cannot load file with non-JavaScript file extension ${ext}.`);
168+
}
169+
return {
170+
url: resolved.href,
171+
format: 'esm'
172+
};
173+
}
174+
```
175+
176+
With this loader, running:
177+
178+
```
179+
NODE_OPTIONS='--experimental-modules --loader ./custom-loader.mjs' node x.js
180+
```
181+
182+
would load the module `x.js` as an ES module with relative resolution support
183+
(with `node_modules` loading skipped in this example).
184+
185+
### Dynamic instantiate hook
186+
187+
To create a custom dynamic module that doesn't correspond to one of the
188+
existing `format` interpretations, the `dynamicInstantiate` hook can be used.
189+
This hook is called only for modules that return `format: "dynamic"` from
190+
the `resolve` hook.
191+
192+
```js
193+
export async function dynamicInstantiate(url) {
194+
return {
195+
exports: ['customExportName'],
196+
execute: (exports) => {
197+
// get and set functions provided for pre-allocated export names
198+
exports.customExportName.set('value');
199+
}
200+
};
201+
}
202+
```
203+
204+
With the list of module exports provided upfront, the `execute` function will
205+
then be called at the exact point of module evalutation order for that module
206+
in the import tree.
207+
101208
[Node.js EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md

‎lib/internal/errors.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,8 @@ E('ERR_TRANSFORM_WITH_LENGTH_0',
328328
E('ERR_UNESCAPED_CHARACTERS',
329329
(name) => `${name} contains unescaped characters`);
330330
E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s');
331+
E('ERR_UNKNOWN_FILE_EXTENSION', 'Unknown file extension: %s');
332+
E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s');
331333
E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s');
332334
E('ERR_UNKNOWN_STDIN_TYPE', 'Unknown stdin file type');
333335
E('ERR_UNKNOWN_STREAM_TYPE', 'Unknown stream file type');

‎lib/internal/loader/Loader.js

Lines changed: 62 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
'use strict';
22

3-
const { URL } = require('url');
43
const { getURLFromFilePath } = require('internal/url');
54

65
const {
7-
getNamespaceOfModuleWrap
6+
getNamespaceOfModuleWrap,
7+
createDynamicModule
88
} = require('internal/loader/ModuleWrap');
99

1010
const ModuleMap = require('internal/loader/ModuleMap');
1111
const ModuleJob = require('internal/loader/ModuleJob');
12-
const resolveRequestUrl = require('internal/loader/resolveRequestUrl');
12+
const ModuleRequest = require('internal/loader/ModuleRequest');
1313
const errors = require('internal/errors');
14+
const debug = require('util').debuglog('esm');
1415

1516
function getBase() {
1617
try {
17-
return getURLFromFilePath(`${process.cwd()}/`);
18+
return getURLFromFilePath(`${process.cwd()}/`).href;
1819
} catch (e) {
1920
e.stack;
2021
// If the current working directory no longer exists.
@@ -28,45 +29,75 @@ function getBase() {
2829
class Loader {
2930
constructor(base = getBase()) {
3031
this.moduleMap = new ModuleMap();
31-
if (typeof base !== 'undefined' && base instanceof URL !== true) {
32-
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL');
32+
if (typeof base !== 'string') {
33+
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string');
3334
}
3435
this.base = base;
36+
this.resolver = ModuleRequest.resolve.bind(null);
37+
this.dynamicInstantiate = undefined;
3538
}
3639

37-
async resolve(specifier) {
38-
const request = resolveRequestUrl(this.base, specifier);
39-
if (request.url.protocol !== 'file:') {
40-
throw new errors.Error('ERR_INVALID_PROTOCOL',
41-
request.url.protocol, 'file:');
42-
}
43-
return request.url;
40+
hook({ resolve = ModuleRequest.resolve, dynamicInstantiate }) {
41+
this.resolver = resolve.bind(null);
42+
this.dynamicInstantiate = dynamicInstantiate;
4443
}
4544

46-
async getModuleJob(dependentJob, specifier) {
47-
if (!this.moduleMap.has(dependentJob.url)) {
48-
throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url);
45+
async resolve(specifier, parentURL = this.base) {
46+
if (typeof parentURL !== 'string') {
47+
throw new errors.TypeError('ERR_INVALID_ARG_TYPE',
48+
'parentURL', 'string');
49+
}
50+
const { url, format } = await this.resolver(specifier, parentURL,
51+
ModuleRequest.resolve);
52+
53+
if (typeof format !== 'string') {
54+
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'format',
55+
['esm', 'cjs', 'builtin', 'addon', 'json']);
56+
}
57+
if (typeof url !== 'string') {
58+
throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
59+
}
60+
61+
if (format === 'builtin') {
62+
return { url: `node:${url}`, format };
4963
}
50-
const request = await resolveRequestUrl(dependentJob.url, specifier);
51-
const url = `${request.url}`;
52-
if (this.moduleMap.has(url)) {
53-
return this.moduleMap.get(url);
64+
65+
if (format !== 'dynamic') {
66+
if (!ModuleRequest.loaders.has(format)) {
67+
throw new errors.Error('ERR_UNKNOWN_MODULE_FORMAT', format);
68+
}
69+
if (!url.startsWith('file:')) {
70+
throw new errors.Error('ERR_INVALID_PROTOCOL', url, 'file:');
71+
}
5472
}
55-
const dependencyJob = new ModuleJob(this, request);
56-
this.moduleMap.set(url, dependencyJob);
57-
return dependencyJob;
73+
74+
return { url, format };
5875
}
5976

60-
async import(specifier) {
61-
const request = await resolveRequestUrl(this.base, specifier);
62-
const url = `${request.url}`;
63-
let job;
64-
if (this.moduleMap.has(url)) {
65-
job = this.moduleMap.get(url);
66-
} else {
67-
job = new ModuleJob(this, request);
77+
async getModuleJob(specifier, parentURL = this.base) {
78+
const { url, format } = await this.resolve(specifier, parentURL);
79+
let job = this.moduleMap.get(url);
80+
if (job === undefined) {
81+
let loaderInstance;
82+
if (format === 'dynamic') {
83+
loaderInstance = async (url) => {
84+
const { exports, execute } = await this.dynamicInstantiate(url);
85+
return createDynamicModule(exports, url, (reflect) => {
86+
debug(`Loading custom loader ${url}`);
87+
execute(reflect.exports);
88+
});
89+
};
90+
} else {
91+
loaderInstance = ModuleRequest.loaders.get(format);
92+
}
93+
job = new ModuleJob(this, url, loaderInstance);
6894
this.moduleMap.set(url, job);
6995
}
96+
return job;
97+
}
98+
99+
async import(specifier, parentURL = this.base) {
100+
const job = await this.getModuleJob(specifier, parentURL);
70101
const module = await job.run();
71102
return getNamespaceOfModuleWrap(module);
72103
}

‎lib/internal/loader/ModuleJob.js

Lines changed: 24 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,51 +2,40 @@
22

33
const { SafeSet, SafePromise } = require('internal/safe_globals');
44
const resolvedPromise = SafePromise.resolve();
5-
const resolvedArrayPromise = SafePromise.resolve([]);
6-
const { ModuleWrap } = require('internal/loader/ModuleWrap');
75

8-
const NOOP = () => { /* No-op */ };
96
class ModuleJob {
107
/**
118
* @param {module: ModuleWrap?, compiled: Promise} moduleProvider
129
*/
13-
constructor(loader, moduleProvider, url) {
14-
this.url = `${moduleProvider.url}`;
15-
this.moduleProvider = moduleProvider;
10+
constructor(loader, url, moduleProvider) {
1611
this.loader = loader;
1712
this.error = null;
1813
this.hadError = false;
1914

20-
if (moduleProvider instanceof ModuleWrap !== true) {
21-
// linked == promise for dependency jobs, with module populated,
22-
// module wrapper linked
23-
this.modulePromise = this.moduleProvider.createModule();
24-
this.module = undefined;
25-
const linked = async () => {
26-
const dependencyJobs = [];
27-
this.module = await this.modulePromise;
28-
this.module.link(async (dependencySpecifier) => {
29-
const dependencyJobPromise =
30-
this.loader.getModuleJob(this, dependencySpecifier);
31-
dependencyJobs.push(dependencyJobPromise);
32-
const dependencyJob = await dependencyJobPromise;
33-
return dependencyJob.modulePromise;
34-
});
35-
return SafePromise.all(dependencyJobs);
36-
};
37-
this.linked = linked();
15+
// linked == promise for dependency jobs, with module populated,
16+
// module wrapper linked
17+
this.moduleProvider = moduleProvider;
18+
this.modulePromise = this.moduleProvider(url);
19+
this.module = undefined;
20+
this.reflect = undefined;
21+
const linked = async () => {
22+
const dependencyJobs = [];
23+
({ module: this.module,
24+
reflect: this.reflect } = await this.modulePromise);
25+
this.module.link(async (dependencySpecifier) => {
26+
const dependencyJobPromise =
27+
this.loader.getModuleJob(dependencySpecifier, url);
28+
dependencyJobs.push(dependencyJobPromise);
29+
const dependencyJob = await dependencyJobPromise;
30+
return (await dependencyJob.modulePromise).module;
31+
});
32+
return SafePromise.all(dependencyJobs);
33+
};
34+
this.linked = linked();
3835

39-
// instantiated == deep dependency jobs wrappers instantiated,
40-
//module wrapper instantiated
41-
this.instantiated = undefined;
42-
} else {
43-
const getModuleProvider = async () => moduleProvider;
44-
this.modulePromise = getModuleProvider();
45-
this.moduleProvider = { finish: NOOP };
46-
this.module = moduleProvider;
47-
this.linked = resolvedArrayPromise;
48-
this.instantiated = this.modulePromise;
49-
}
36+
// instantiated == deep dependency jobs wrappers instantiated,
37+
// module wrapper instantiated
38+
this.instantiated = undefined;
5039
}
5140

5241
instantiate() {

0 commit comments

Comments
 (0)