Skip to content

test: build test add-ons like third-party add-ons #12231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 14 additions & 107 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ endif
# otherwise $(NODE_EXE) being a .PHONY target means it is always re-run.
# Without the check there is a race condition between the link being deleted
# and recreated which can break the addons build when running test-ci
# See comments on the build-addons target for some more info
$(NODE_EXE): config.gypi out/Makefile
$(MAKE) -C out BUILDTYPE=Release V=$(V)
if [ ! -r $@ -o ! -L $@ ]; then ln -fs out/Release/$(NODE_EXE) $@; fi
Expand Down Expand Up @@ -193,7 +192,6 @@ v8:

test: all
$(MAKE) build-addons
$(MAKE) build-addons-napi
$(MAKE) cctest
$(PYTHON) tools/test.py --mode=release -J \
doctool inspector known_issues message pseudo-tty parallel sequential $(CI_NATIVE_SUITES)
Expand All @@ -205,98 +203,9 @@ test-parallel: all
test-valgrind: all
$(PYTHON) tools/test.py --mode=release --valgrind sequential parallel message

# Implicitly depends on $(NODE_EXE). We don't depend on it explicitly because
# it always triggers a rebuild due to it being a .PHONY rule. See the comment
# near the build-addons rule for more background.
test/gc/build/Release/binding.node: test/gc/binding.cc test/gc/binding.gyp
$(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp rebuild \
--python="$(PYTHON)" \
--directory="$(shell pwd)/test/gc" \
--nodedir="$(shell pwd)"

# Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale.
DOCBUILDSTAMP_PREREQS = tools/doc/addon-verify.js doc/api/addons.md

ifeq ($(OSTYPE),aix)
DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
endif

test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
$(RM) -r test/addons/??_*/
$(NODE) $<
touch $@

ADDONS_BINDING_GYPS := \
$(filter-out test/addons/??_*/binding.gyp, \
$(wildcard test/addons/*/binding.gyp))

ADDONS_BINDING_SOURCES := \
$(filter-out test/addons/??_*/*.cc, $(wildcard test/addons/*/*.cc)) \
$(filter-out test/addons/??_*/*.h, $(wildcard test/addons/*/*.h))

# Implicitly depends on $(NODE_EXE), see the build-addons rule for rationale.
# Depends on node-gyp package.json so that build-addons is (re)executed when
# node-gyp is updated as part of an npm update.
test/addons/.buildstamp: config.gypi \
deps/npm/node_modules/node-gyp/package.json \
$(ADDONS_BINDING_GYPS) $(ADDONS_BINDING_SOURCES) \
deps/uv/include/*.h deps/v8/include/*.h \
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
test/addons/.docbuildstamp
# Cannot use $(wildcard test/addons/*/) here, it's evaluated before
# embedded addons have been generated from the documentation.
@for dirname in test/addons/*/; do \
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
--loglevel=$(LOGLEVEL) rebuild \
--python="$(PYTHON)" \
--directory="$$PWD/$$dirname" \
--nodedir="$$PWD" || exit 1 ; \
done
touch $@

# .buildstamp and .docbuildstamp need $(NODE_EXE) but cannot depend on it
# directly because it calls make recursively. The parent make cannot know
# if the subprocess touched anything so it pessimistically assumes that
# .buildstamp and .docbuildstamp are out of date and need a rebuild.
# Just goes to show that recursive make really is harmful...
# TODO(bnoordhuis) Force rebuild after gyp update.
build-addons: $(NODE_EXE) test/addons/.buildstamp

ADDONS_NAPI_BINDING_GYPS := \
$(filter-out test/addons-napi/??_*/binding.gyp, \
$(wildcard test/addons-napi/*/binding.gyp))

ADDONS_NAPI_BINDING_SOURCES := \
$(filter-out test/addons-napi/??_*/*.cc, $(wildcard test/addons-napi/*/*.cc)) \
$(filter-out test/addons-napi/??_*/*.h, $(wildcard test/addons-napi/*/*.h))

# Implicitly depends on $(NODE_EXE), see the build-addons-napi rule for rationale.
test/addons-napi/.buildstamp: config.gypi \
deps/npm/node_modules/node-gyp/package.json \
$(ADDONS_NAPI_BINDING_GYPS) $(ADDONS_NAPI_BINDING_SOURCES) \
deps/uv/include/*.h deps/v8/include/*.h \
src/node.h src/node_buffer.h src/node_object_wrap.h src/node_version.h \
src/node_api.h src/node_api_types.h
# Cannot use $(wildcard test/addons-napi/*/) here, it's evaluated before
# embedded addons have been generated from the documentation.
@for dirname in test/addons-napi/*/; do \
printf "\nBuilding addon $$PWD/$$dirname\n" ; \
env MAKEFLAGS="-j1" $(NODE) deps/npm/node_modules/node-gyp/bin/node-gyp \
--loglevel=$(LOGLEVEL) rebuild \
--python="$(PYTHON)" \
--directory="$$PWD/$$dirname" \
--nodedir="$$PWD" || exit 1 ; \
done
touch $@

# .buildstamp and .docbuildstamp need $(NODE_EXE) but cannot depend on it
# directly because it calls make recursively. The parent make cannot know
# if the subprocess touched anything so it pessimistically assumes that
# .buildstamp and .docbuildstamp are out of date and need a rebuild.
# Just goes to show that recursive make really is harmful...
# TODO(bnoordhuis) Force rebuild after gyp or node-gyp update.
build-addons-napi: $(NODE_EXE) test/addons-napi/.buildstamp
# Builds test/addons, test/addons-napi and test/gc.
build-addons: $(NODE_EXE)
./$< tools/build-addons.js

clear-stalled:
# Clean up any leftover processes but don't error if found.
Expand All @@ -306,17 +215,15 @@ clear-stalled:
echo $${PS_OUT} | xargs kill; \
fi

test-gc: all test/gc/build/Release/binding.node
test-gc: test-build
$(PYTHON) tools/test.py --mode=release gc

test-gc-clean:
$(RM) -r test/gc/build

test-build: | all build-addons build-addons-napi
test-build: | all build-addons

test-build-addons-napi: all build-addons-napi

test-all: test-build test/gc/build/Release/binding.node
test-all: test-build
$(PYTHON) tools/test.py --mode=debug,release

test-all-valgrind: test-build
Expand All @@ -327,7 +234,7 @@ CI_JS_SUITES := doctool inspector known_issues message parallel pseudo-tty seque

# Build and test addons without building anything else
test-ci-native: LOGLEVEL := info
test-ci-native: | test/addons/.buildstamp test/addons-napi/.buildstamp
test-ci-native: | build-addons
$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
--mode=release --flaky-tests=$(FLAKY_TESTS) \
$(TEST_CI_ARGS) $(CI_NATIVE_SUITES)
Expand All @@ -345,7 +252,7 @@ test-ci-js: | clear-stalled
fi

test-ci: LOGLEVEL := info
test-ci: | clear-stalled build-addons build-addons-napi
test-ci: | clear-stalled build-addons
out/Release/cctest --gtest_output=tap:cctest.tap
$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
--mode=release --flaky-tests=$(FLAKY_TESTS) \
Expand Down Expand Up @@ -397,16 +304,17 @@ test-npm: $(NODE_EXE)
test-npm-publish: $(NODE_EXE)
npm_package_config_publishtest=true $(NODE) deps/npm/test/run.js

test-addons-napi: test-build-addons-napi
$(PYTHON) tools/test.py --mode=release addons-napi

test-addons: test-build test-addons-napi
test-addons: test-build
$(PYTHON) tools/test.py --mode=release addons

test-addons-napi: test-build
$(PYTHON) tools/test.py --mode=release addons-napi

test-addons-clean:
$(RM) -rf test/addons/??_*/
$(RM) -rf test/addons/*/build
$(RM) test/addons/.buildstamp test/addons/.docbuildstamp
$(RM) -rf test/addons/Release/
$(RM) -rf test/addons/include/

test-timers:
$(MAKE) --directory=tools faketime
Expand Down Expand Up @@ -934,7 +842,6 @@ endif
blog \
blogclean \
build-addons \
build-addons-napi \
build-ci \
cctest \
check \
Expand Down
2 changes: 0 additions & 2 deletions test/addons-napi/.gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
.buildstamp
.docbuildstamp
Makefile
*.Makefile
*.mk
Expand Down
4 changes: 2 additions & 2 deletions test/addons/.gitignore
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.buildstamp
.docbuildstamp
Makefile
*.Makefile
*.mk
gyp-mac-tool
/*/build
/Release/
/include/
1 change: 0 additions & 1 deletion test/addons/openssl-binding/binding.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
'conditions': [
['node_use_openssl=="true"', {
'sources': ['binding.cc'],
'include_dirs': ['../../../deps/openssl/openssl/include'],
}]
]
},
Expand Down
1 change: 0 additions & 1 deletion test/addons/zlib-binding/binding.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
{
'target_name': 'binding',
'sources': ['binding.cc'],
'include_dirs': ['../../../deps/zlib'],
},
]
}
92 changes: 92 additions & 0 deletions tools/build-addons.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
'use strict';

const fs = require('fs');
const os = require('os');
const { spawn, spawnSync } = require('child_process');
const { resolve } = require('path');

const kTopLevelDirectory = resolve(__dirname, '..');
const kAddonsDirectory = resolve(kTopLevelDirectory, 'test/addons');
const kNapiAddonsDirectory = resolve(kTopLevelDirectory, 'test/addons-napi');

// Location where the headers are installed to.
const kIncludeDirectory = kAddonsDirectory;

const kPython = process.env.PYTHON || 'python';
const kNodeGyp =
resolve(kTopLevelDirectory, 'deps/npm/node_modules/node-gyp/bin/node-gyp');

process.chdir(kTopLevelDirectory);

// Copy headers to `${kIncludeDirectory}/include`. install.py preserves
// timestamps so this won't cause unnecessary rebuilds.
{
const args = [ 'tools/install.py', 'install', kIncludeDirectory, '/' ];
const env = Object.assign({}, process.env);
env.HEADERS_ONLY = 'yes, please'; // Ask nicely.
env.LOGLEVEL = 'WARNING';

const options = { env, stdio: 'inherit' };
const result = spawnSync(kPython, args, options);
if (result.status !== 0) process.exit(1);
}

// Scrape embedded add-ons from doc/api/addons.md.
require('./doc/addon-verify.js');

// Regenerate build files and rebuild if necessary.
let failures = 0;
process.on('exit', () => process.exit(failures > 0));

const jobs = [];

// test/gc contains a single add-on to build.
{
const path = resolve(kTopLevelDirectory, 'test/gc');
exec(path, 'configure', () => exec(path, 'build'));
}

for (const directory of [kAddonsDirectory, kNapiAddonsDirectory]) {
for (const basedir of fs.readdirSync(directory)) {
const path = resolve(directory, basedir);
const gypfile = resolve(path, 'binding.gyp');
if (!fs.existsSync(gypfile)) continue;
exec(path, 'configure', () => exec(path, 'build'));
}
}

// FIXME(bnoordhuis) I would have liked to derive the desired level of
// parallelism from MAKEFLAGS but it's missing the actual -j<jobs> flag.
for (const _ of os.cpus()) next(); // eslint-disable-line no-unused-vars

function next() {
const job = jobs.shift();
if (job) job();
}

function exec(path, command, done) {
jobs.push(() => {
if (failures > 0) return;

const args = [kNodeGyp,
'--loglevel=silent',
'--directory=' + path,
'--nodedir=' + kIncludeDirectory,
'--python=' + kPython,
command];

const env = Object.assign({}, process.env);
env.MAKEFLAGS = '-j1';
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps --silent could be added to this to avoid extra output?


const options = { env, stdio: 'inherit' };
const proc = spawn(process.execPath, args, options);

proc.on('exit', (exitCode) => {
if (exitCode !== 0) ++failures;
if (done) done();
next();
});

console.log(command, path);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could console.log call perhaps be removed in favour of just printing console.log(`building addon ${path}`); before 9864090#diff-31601f8dbf082ea61af036efe1e9c00dR54?

});
}
54 changes: 21 additions & 33 deletions tools/doc/addon-verify.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
'use strict';

const assert = require('assert');
const fs = require('fs');
const path = require('path');
const marked = require('marked');
Expand All @@ -20,13 +21,8 @@ tokens.push({ type: 'heading' });
for (var i = 0; i < tokens.length; i++) {
var token = tokens[i];
if (token.type === 'heading' && token.text) {
const blockName = token.text;
if (files && Object.keys(files).length !== 0) {
verifyFiles(files,
blockName,
console.log.bind(null, 'wrote'),
function(err) { if (err) throw err; });
}
if (files && Object.keys(files).length !== 0)
verifyFiles(files, token.text);
files = {};
} else if (token.type === 'code') {
var match = token.text.match(/^\/\/\s+(.*\.(?:cc|h|js))[\r\n]/);
Expand All @@ -36,17 +32,7 @@ for (var i = 0; i < tokens.length; i++) {
}
}

function once(fn) {
var once = false;
return function() {
if (once)
return;
once = true;
fn.apply(this, arguments);
};
}

function verifyFiles(files, blockName, onprogress, ondone) {
function verifyFiles(files, blockName) {
// must have a .cc and a .js to be a valid test
if (!Object.keys(files).some((name) => /\.cc$/.test(name)) ||
!Object.keys(files).some((name) => /\.js$/.test(name))) {
Expand Down Expand Up @@ -91,22 +77,24 @@ ${files[name].replace('Release', "' + common.buildType + '")}
})
});

fs.mkdir(dir, function() {
// Ignore errors
try {
fs.mkdirSync(dir);
} catch (e) {
assert.strictEqual(e.code, 'EEXIST');
}

var done = once(ondone);
var waiting = files.length;
files.forEach(function(file) {
fs.writeFile(file.path, file.content, function(err) {
if (err)
return done(err);
for (const file of files) {
try {
var content = fs.readFileSync(file.path);
} catch (e) {
assert.strictEqual(e.code, 'ENOENT');
}

if (onprogress)
onprogress(file.path);
// Only update when file content has changed to prevent unneeded rebuilds.
if (content && content.toString() === file.content)
continue;

if (--waiting === 0)
done();
});
});
});
fs.writeFileSync(file.path, file.content);
console.log('wrote', file.path);
}
}
Loading