Skip to content

Commit 0c377ea

Browse files
MoLowMoLow
committed
test_runner: flatten TAP output when running using --test 
1 parent 806a516 commit 0c377ea

11 files changed

+707
-703
lines changed

‎lib/internal/test_runner/runner.js

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ const {
88
ArrayPrototypeSlice,
99
ArrayPrototypeSome,
1010
ArrayPrototypeSort,
11+
Number,
1112
ObjectAssign,
1213
PromisePrototypeThen,
13-
SafePromiseAll,
1414
SafePromiseAllReturnVoid,
1515
SafePromiseAllSettledReturnVoid,
1616
SafeMap,
@@ -35,7 +35,7 @@ const { validateArray, validateBoolean } = require('internal/validators');
3535
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
3636
const { kEmptyObject } = require('internal/util');
3737
const { createTestTree } = require('internal/test_runner/harness');
38-
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
38+
const { kSubtestsFailed, kTestCodeFailure, kCancelledByParent, Test } = require('internal/test_runner/test');
3939
const { TapParser } = require('internal/test_runner/tap_parser');
4040
const { YAMLToJs } = require('internal/test_runner/yaml_to_js');
4141
const { TokenKind } = require('internal/test_runner/tap_lexer');
@@ -133,6 +133,10 @@ function getRunArgs({ path, inspectPort }) {
133133

134134
class FileTest extends Test {
135135
#buffer = [];
136+
#hasSubtests = false;
137+
get skipReporting() {
138+
return this.#hasSubtests && (!this.error || this.error.failureType === kSubtestsFailed);
139+
}
136140
#checkNestedComment({ comment }) {
137141
const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
138142
if (firstSpaceIndex === -1) return false;
@@ -141,8 +145,6 @@ class FileTest extends Test {
141145
ArrayPrototypeIncludes(kDiagnosticsFilterArgs, StringPrototypeSlice(comment, 0, firstSpaceIndex));
142146
}
143147
#handleReportItem({ kind, node, comments, nesting = 0 }) {
144-
nesting += 1;
145-
146148
if (comments) {
147149
ArrayPrototypeForEach(comments, (comment) => this.reporter.diagnostic(nesting, this.name, comment));
148150
}
@@ -153,17 +155,20 @@ class FileTest extends Test {
153155
break;
154156

155157
case TokenKind.TAP_PLAN:
158+
if (nesting === 0 && this.skipReporting) {
159+
break;
160+
}
156161
this.reporter.plan(nesting, this.name, node.end - node.start + 1);
157162
break;
158163

159164
case TokenKind.TAP_SUBTEST_POINT:
160165
this.reporter.start(nesting, this.name, node.name);
161166
break;
162167

163-
case TokenKind.TAP_TEST_POINT:
164-
// eslint-disable-next-line no-case-declarations
168+
case TokenKind.TAP_TEST_POINT: {
169+
165170
const { todo, skip, pass } = node.status;
166-
// eslint-disable-next-line no-case-declarations
171+
167172
let directive;
168173

169174
if (skip) {
@@ -174,29 +179,21 @@ class FileTest extends Test {
174179
directive = kEmptyObject;
175180
}
176181

177-
if (pass) {
178-
this.reporter.ok(
179-
nesting,
180-
this.name,
181-
node.id,
182-
node.description,
183-
YAMLToJs(node.diagnostics),
184-
directive
185-
);
186-
} else {
187-
this.reporter.fail(
188-
nesting,
189-
this.name,
190-
node.id,
191-
node.description,
192-
YAMLToJs(node.diagnostics),
193-
directive
194-
);
182+
this.#hasSubtests = true;
183+
const diagnostics = YAMLToJs(node.diagnostics);
184+
const cancelled = diagnostics.error?.failureType === kCancelledByParent;
185+
const testNumber = nesting === 0 ? (Number(node.id) + this.testNumber - 1) : node.id;
186+
const method = pass ? 'ok' : 'fail';
187+
this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
188+
if (nesting === 0) {
189+
this.parent.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
190+
this.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
195191
}
196192
break;
197193

194+
}
198195
case TokenKind.COMMENT:
199-
if (nesting === 1 && this.#checkNestedComment(node)) {
196+
if (nesting === 0 && this.#checkNestedComment(node)) {
200197
// Ignore file top level diagnostics
201198
break;
202199
}
@@ -216,10 +213,15 @@ class FileTest extends Test {
216213
this.reportStarted();
217214
this.#handleReportItem(ast);
218215
}
216+
reportStarted() {}
219217
report() {
220-
this.reportStarted();
218+
if (!this.skipReporting) {
219+
super.reportStarted();
220+
}
221221
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
222-
super.report();
222+
if (!this.skipReporting) {
223+
super.report();
224+
}
223225
}
224226
}
225227

@@ -274,16 +276,14 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
274276
subtest.addToReport(ast);
275277
});
276278

277-
const { 0: { 0: code, 1: signal } } = await SafePromiseAll([
278-
once(child, 'exit', { signal: t.signal }),
279-
child.stdout.toArray({ signal: t.signal }),
280-
]);
279+
const { 0: code, 1: signal } = await once(child, 'exit', { signal: t.signal });
281280

282281
runningProcesses.delete(path);
283282
runningSubtests.delete(path);
284283
if (code !== 0 || signal !== null) {
285284
if (!err) {
286-
err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), {
285+
const failureType = subtest.completedCounters.totalFailed > 0 ? kSubtestsFailed : kTestCodeFailure;
286+
err = ObjectAssign(new ERR_TEST_FAILURE('test failed', failureType), {
287287
__proto__: null,
288288
exitCode: code,
289289
signal: signal,

‎lib/internal/test_runner/test.js

Lines changed: 41 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,10 @@ class Test extends AsyncResource {
286286
beforeEach: [],
287287
afterEach: [],
288288
};
289+
this.completedCounters = {
290+
__proto__: null,
291+
all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0
292+
};
289293
this.waitingOn = 0;
290294
this.finished = false;
291295
}
@@ -404,6 +408,7 @@ class Test extends AsyncResource {
404408
kCancelledByParent
405409
)
406410
);
411+
this.error.failureType = kCancelledByParent;
407412
this.startTime = this.startTime || this.endTime; // If a test was canceled before it was started, e.g inside a hook
408413
this.cancelled = true;
409414
this.#abortController.abort();
@@ -577,9 +582,32 @@ class Test extends AsyncResource {
577582
this.postRun();
578583
}
579584

580-
postRun(pendingSubtestsError) {
581-
const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
585+
countSubtest(subtest) {
586+
if (subtest.skipReporting || subtest.counted) {
587+
return;
588+
}
589+
subtest.counted = true;
590+
// Check SKIP and TODO tests first, as those should not be counted as
591+
// failures.
592+
if (subtest.skipped) {
593+
this.completedCounters.skipped++;
594+
} else if (subtest.isTodo) {
595+
this.completedCounters.todo++;
596+
} else if (subtest.cancelled) {
597+
this.completedCounters.cancelled++;
598+
} else if (!subtest.passed) {
599+
this.completedCounters.failed++;
600+
} else {
601+
this.completedCounters.passed++;
602+
}
603+
604+
if (!subtest.passed) {
605+
this.completedCounters.totalFailed++;
606+
}
607+
this.completedCounters.all++;
608+
}
582609

610+
postRun(pendingSubtestsError) {
583611
// If the test was failed before it even started, then the end time will
584612
// be earlier than the start time. Correct that here.
585613
if (this.endTime < this.startTime) {
@@ -592,34 +620,15 @@ class Test extends AsyncResource {
592620
this.pendingSubtests = [];
593621
for (let i = 0; i < this.subtests.length; i++) {
594622
const subtest = this.subtests[i];
595-
596623
if (!subtest.finished) {
597624
subtest.cancel(pendingSubtestsError);
598625
subtest.postRun(pendingSubtestsError);
599626
}
600-
601-
// Check SKIP and TODO tests first, as those should not be counted as
602-
// failures.
603-
if (subtest.skipped) {
604-
counters.skipped++;
605-
} else if (subtest.isTodo) {
606-
counters.todo++;
607-
} else if (subtest.cancelled) {
608-
counters.cancelled++;
609-
} else if (!subtest.passed) {
610-
counters.failed++;
611-
} else {
612-
counters.passed++;
613-
}
614-
615-
if (!subtest.passed) {
616-
counters.totalFailed++;
617-
}
618627
}
619628

620-
if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
621-
const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
622-
const msg = `${counters.totalFailed} ${subtestString} failed`;
629+
if ((this.passed || this.parent === null) && this.completedCounters.totalFailed > 0) {
630+
const subtestString = `subtest${this.completedCounters.totalFailed > 1 ? 's' : ''}`;
631+
const msg = `${this.completedCounters.totalFailed} ${subtestString} failed`;
623632

624633
this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
625634
}
@@ -632,20 +641,21 @@ class Test extends AsyncResource {
632641
this.parent.addReadySubtest(this);
633642
this.parent.processReadySubtestRange(false);
634643
this.parent.processPendingSubtests();
644+
this.parent.countSubtest(this);
635645
} else if (!this.reported) {
636646
this.reported = true;
637-
this.reporter.plan(this.nesting, kFilename, this.subtests.length);
647+
this.reporter.plan(this.nesting, kFilename, this.completedCounters.all);
638648

639649
for (let i = 0; i < this.diagnostics.length; i++) {
640650
this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
641651
}
642652

643-
this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.subtests.length}`);
644-
this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`);
645-
this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`);
646-
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`);
647-
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${counters.skipped}`);
648-
this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`);
653+
this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.completedCounters.all}`);
654+
this.reporter.diagnostic(this.nesting, kFilename, `pass ${this.completedCounters.passed}`);
655+
this.reporter.diagnostic(this.nesting, kFilename, `fail ${this.completedCounters.failed}`);
656+
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${this.completedCounters.cancelled}`);
657+
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${this.completedCounters.skipped}`);
658+
this.reporter.diagnostic(this.nesting, kFilename, `todo ${this.completedCounters.todo}`);
649659
this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`);
650660

651661
if (this.coverage) {

‎test/message/test_runner_abort.out

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ TAP version 13
4040
not ok 7 - not ok 3
4141
---
4242
duration_ms: *
43+
failureType: 'cancelledByParent'
4344
error: 'This operation was aborted'
4445
code: 20
4546
stack: |-
@@ -58,6 +59,7 @@ TAP version 13
5859
not ok 8 - not ok 4
5960
---
6061
duration_ms: *
62+
failureType: 'cancelledByParent'
6163
error: 'This operation was aborted'
6264
code: 20
6365
stack: |-
@@ -76,6 +78,7 @@ TAP version 13
7678
not ok 9 - not ok 5
7779
---
7880
duration_ms: *
81+
failureType: 'cancelledByParent'
7982
error: 'This operation was aborted'
8083
code: 20
8184
stack: |-
@@ -94,6 +97,7 @@ TAP version 13
9497
not ok 1 - promise timeout signal
9598
---
9699
duration_ms: *
100+
failureType: 'cancelledByParent'
97101
error: 'The operation was aborted due to timeout'
98102
code: 23
99103
stack: |-
@@ -106,6 +110,7 @@ not ok 1 - promise timeout signal
106110
not ok 2 - promise abort signal
107111
---
108112
duration_ms: *
113+
failureType: 'cancelledByParent'
109114
error: 'This operation was aborted'
110115
code: 20
111116
stack: |-
@@ -160,6 +165,7 @@ not ok 2 - promise abort signal
160165
not ok 7 - not ok 3
161166
---
162167
duration_ms: *
168+
failureType: 'cancelledByParent'
163169
error: 'This operation was aborted'
164170
code: 20
165171
stack: |-
@@ -178,6 +184,7 @@ not ok 2 - promise abort signal
178184
not ok 8 - not ok 4
179185
---
180186
duration_ms: *
187+
failureType: 'cancelledByParent'
181188
error: 'This operation was aborted'
182189
code: 20
183190
stack: |-
@@ -196,6 +203,7 @@ not ok 2 - promise abort signal
196203
not ok 9 - not ok 5
197204
---
198205
duration_ms: *
206+
failureType: 'cancelledByParent'
199207
error: 'This operation was aborted'
200208
code: 20
201209
stack: |-
@@ -214,6 +222,7 @@ not ok 2 - promise abort signal
214222
not ok 3 - callback timeout signal
215223
---
216224
duration_ms: *
225+
failureType: 'cancelledByParent'
217226
error: 'The operation was aborted due to timeout'
218227
code: 23
219228
stack: |-
@@ -226,6 +235,7 @@ not ok 3 - callback timeout signal
226235
not ok 4 - callback abort signal
227236
---
228237
duration_ms: *
238+
failureType: 'cancelledByParent'
229239
error: 'This operation was aborted'
230240
code: 20
231241
stack: |-

‎test/message/test_runner_abort_suite.out

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ TAP version 13
6464
not ok 1 - describe timeout signal
6565
---
6666
duration_ms: *
67+
failureType: 'cancelledByParent'
6768
error: 'The operation was aborted due to timeout'
6869
code: 23
6970
stack: |-
@@ -76,6 +77,7 @@ not ok 1 - describe timeout signal
7677
not ok 2 - describe abort signal
7778
---
7879
duration_ms: *
80+
failureType: 'cancelledByParent'
7981
error: 'This operation was aborted'
8082
code: 20
8183
stack: |-

‎test/message/test_runner_describe_it.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -548,7 +548,7 @@ not ok 55 - describe async throw fails
548548
not ok 1 - timed out async test
549549
---
550550
duration_ms: *
551-
failureType: 'testTimeoutFailure'
551+
failureType: 'cancelledByParent'
552552
error: 'test timed out after 5ms'
553553
code: 'ERR_TEST_FAILURE'
554554
stack: |-
@@ -558,7 +558,7 @@ not ok 55 - describe async throw fails
558558
not ok 2 - timed out callback test
559559
---
560560
duration_ms: *
561-
failureType: 'testTimeoutFailure'
561+
failureType: 'cancelledByParent'
562562
error: 'test timed out after 5ms'
563563
code: 'ERR_TEST_FAILURE'
564564
...

‎test/message/test_runner_output.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -561,15 +561,15 @@ not ok 56 - subtest sync throw fails
561561
not ok 57 - timed out async test
562562
---
563563
duration_ms: *
564-
failureType: 'testTimeoutFailure'
564+
failureType: 'cancelledByParent'
565565
error: 'test timed out after 5ms'
566566
code: 'ERR_TEST_FAILURE'
567567
...
568568
# Subtest: timed out callback test
569569
not ok 58 - timed out callback test
570570
---
571571
duration_ms: *
572-
failureType: 'testTimeoutFailure'
572+
failureType: 'cancelledByParent'
573573
error: 'test timed out after 5ms'
574574
code: 'ERR_TEST_FAILURE'
575575
...

0 commit comments

Comments
 (0)