Skip to content

Commit 19c252a

Browse files
GrosSacASacCharlike Mike Reagent
authored andcommitted
feat: use modern Streams API (#531)
* start with streams * fix JSON decoder * save * Update multipart_parser.js * use streams API in the multipart example * use sreaing api end does not return errors, use 'error' event instead write does not return length anymore * use the new multipartparser which is a stream * revert, add todo * emit error via recommended stream way * octet parser is passtrough * querystringparser is a stream * Update incoming_form.js * Dummy parser is a stream * formatting * listen for errors earlier * decouple, don't self check internals * use existing var * display name of failing test * chore: tweaks Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: merge with latest master Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: switch lib/ to src/ in tests Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: comment failing test, fix ti soon Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: make tests passing; ignore workarounds fixture Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: proper ignore; use tap reporter; Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: some formatting happens... Signed-off-by: Charlike Mike Reagent <[email protected]> * chore: add to changelog Signed-off-by: Charlike Mike Reagent <[email protected]> Co-authored-by: Charlike Mike Reagent <[email protected]>
1 parent a26cdac commit 19c252a

23 files changed

+516
-463
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
* Improve examples and tests ([#523](https://github.com/node-formidable/node-formidable/pull/523))
1212
* First step of Code quality improvements ([#525](https://github.com/node-formidable/node-formidable/pull/525))
1313
* chore(funding): remove patreon & add npm funding field ([#525](https://github.com/node-formidable/node-formidable/pull/532)
14+
* Modern Streams API ([#531](https://github.com/node-formidable/node-formidable/pull/531))
1415

1516
### v1.2.1 (2018-03-20)
1617

example/json.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ server = http.createServer(function(req, res) {
1818
form
1919
.on('error', function(err) {
2020
res.writeHead(500, {'content-type': 'text/plain'});
21-
res.end('error:\n\n'+util.inspect(err));
21+
res.end('error:\n\n' + util.inspect(err));
2222
console.error(err);
2323
})
2424
.on('field', function(field, value) {

example/multipartParser.js

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
const { MultipartParser } = require('../lib/multipart_parser.js');
22

33

4-
const multipartParser = new MultipartParser();
5-
64
// hand crafted multipart
75
const boundary = '--abcxyz';
86
const next = '\r\n';
@@ -11,25 +9,19 @@ const buffer = Buffer.from(
119
`${boundary}${next}${formData}name="text"${next}${next}text ...${next}${next}${boundary}${next}${formData}name="z"${next}${next}text inside z${next}${next}${boundary}${next}${formData}name="file1"; filename="a.txt"${next}Content-Type: text/plain${next}${next}Content of a.txt.${next}${next}${boundary}${next}${formData}name="file2"; filename="a.html"${next}Content-Type: text/html${next}${next}<!DOCTYPE html><title>Content of a.html.</title>${next}${next}${boundary}--`
1210
);
1311

14-
const logAnalyzed = (buffer, start, end) => {
12+
const multipartParser = new MultipartParser();
13+
multipartParser.on('data', ({name, buffer, start, end}) => {
14+
console.log(`${name}:`);
1515
if (buffer && start && end) {
16-
console.log(String(buffer.slice(start, end)))
16+
console.log(String(buffer.slice(start, end)));
1717
}
18-
};
19-
20-
// multipartParser.onPartBegin
21-
// multipartParser.onPartEnd
22-
23-
// multipartParser.on('partData', logAnalyzed) // non supported syntax
24-
multipartParser.onPartData = logAnalyzed;
25-
multipartParser.onHeaderField = logAnalyzed;
26-
multipartParser.onHeaderValue = logAnalyzed;
27-
multipartParser.initWithBoundary(boundary.substring(2));
28-
29-
30-
const bytesParsed = multipartParser.write(buffer);
31-
const error = multipartParser.end();
32-
33-
if (error) {
18+
console.log();
19+
});
20+
multipartParser.on('error', (error) => {
3421
console.error(error);
35-
}
22+
});
23+
24+
multipartParser.initWithBoundary(boundary.substring(2)); // todo make better error message when it is forgotten
25+
// const shouldWait = !multipartParser.write(buffer);
26+
multipartParser.end();
27+
// multipartParser.destroy();

src/default_options.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@ const defaultOptions = {
77
hash: false,
88
multiples: false,
99
};
10-
10+
1111
exports.defaultOptions = defaultOptions;

src/dummy_parser.js

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
const { Transform } = require('stream');
2+
3+
4+
class DummyParser extends Transform {
5+
constructor(incomingForm) {
6+
super();
7+
this.incomingForm = incomingForm;
8+
}
9+
10+
_flush(callback) {
11+
this.incomingForm.ended = true;
12+
this.incomingForm._maybeEnd();
13+
callback();
14+
}
15+
}
16+
17+
18+
exports.DummyParser = DummyParser;

src/file.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ function File(properties) {
1414
this.lastModifiedDate = null;
1515

1616
this._writeStream = null;
17-
17+
1818
for (var key in properties) {
1919
this[key] = properties[key];
2020
}

src/incoming_form.js

Lines changed: 113 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ var util = require('util'),
44
path = require('path'),
55
File = require('./file'),
66
defaultOptions = require('./default_options').defaultOptions,
7+
DummyParser = require('./dummy_parser').DummyParser,
78
MultipartParser = require('./multipart_parser').MultipartParser,
89
QuerystringParser = require('./querystring_parser').QuerystringParser,
910
OctetParser = require('./octet_parser').OctetParser,
@@ -140,10 +141,7 @@ IncomingForm.prototype.parse = function(req, cb) {
140141
return;
141142
}
142143

143-
var err = this._parser.end();
144-
if (err) {
145-
this._error(err);
146-
}
144+
this._parser.end();
147145
});
148146

149147
return this;
@@ -153,6 +151,9 @@ IncomingForm.prototype.writeHeaders = function(headers) {
153151
this.headers = headers;
154152
this._parseContentLength();
155153
this._parseContentType();
154+
this._parser.once('error', (error) => {
155+
this._error(error);
156+
});
156157
};
157158

158159
IncomingForm.prototype.write = function(buffer) {
@@ -167,12 +168,9 @@ IncomingForm.prototype.write = function(buffer) {
167168
this.bytesReceived += buffer.length;
168169
this.emit('progress', this.bytesReceived, this.bytesExpected);
169170

170-
var bytesParsed = this._parser.write(buffer);
171-
if (bytesParsed !== buffer.length) {
172-
this._error(new Error(`parser error,${bytesParsed} of ${buffer.length} bytes parsed`));
173-
}
171+
this._parser.write(buffer);
174172

175-
return bytesParsed;
173+
return this.bytesReceived;
176174
};
177175

178176
IncomingForm.prototype.pause = function() {
@@ -249,19 +247,10 @@ IncomingForm.prototype.handlePart = function(part) {
249247
});
250248
};
251249

252-
function dummyParser(incomingForm) {
253-
return {
254-
end: function () {
255-
incomingForm.ended = true;
256-
incomingForm._maybeEnd();
257-
return null;
258-
}
259-
};
260-
}
261250

262251
IncomingForm.prototype._parseContentType = function() {
263252
if (this.bytesExpected === 0) {
264-
this._parser = dummyParser(this);
253+
this._parser = new DummyParser(this);
265254
return;
266255
}
267256

@@ -341,98 +330,103 @@ IncomingForm.prototype._initMultipart = function(boundary) {
341330

342331
parser.initWithBoundary(boundary);
343332

344-
parser.onPartBegin = function() {
345-
part = new Stream();
346-
part.readable = true;
347-
part.headers = {};
348-
part.name = null;
349-
part.filename = null;
350-
part.mime = null;
351-
352-
part.transferEncoding = 'binary';
353-
part.transferBuffer = '';
354-
355-
headerField = '';
356-
headerValue = '';
357-
};
358-
359-
parser.onHeaderField = (b, start, end) => {
360-
headerField += b.toString(this.encoding, start, end);
361-
};
362-
363-
parser.onHeaderValue = (b, start, end) => {
364-
headerValue += b.toString(this.encoding, start, end);
365-
};
366-
367-
parser.onHeaderEnd = () => {
368-
headerField = headerField.toLowerCase();
369-
part.headers[headerField] = headerValue;
370-
371-
// matches either a quoted-string or a token (RFC 2616 section 19.5.1)
372-
var m = headerValue.match(/\bname=("([^"]*)"|([^\(\)<>@,;:\\"\/\[\]\?=\{\}\s\t/]+))/i);
373-
if (headerField == 'content-disposition') {
374-
if (m) {
375-
part.name = m[2] || m[3] || '';
376-
}
333+
parser.on('data', ({name, buffer, start, end}) => {
334+
if (name === 'partBegin') {
335+
part = new Stream();
336+
part.readable = true;
337+
part.headers = {};
338+
part.name = null;
339+
part.filename = null;
340+
part.mime = null;
341+
342+
part.transferEncoding = 'binary';
343+
part.transferBuffer = '';
344+
345+
headerField = '';
346+
headerValue = '';
347+
} else if (name === 'headerField') {
348+
headerField += buffer.toString(this.encoding, start, end);
349+
} else if (name === 'headerValue') {
350+
headerValue += buffer.toString(this.encoding, start, end);
351+
} else if (name === 'headerEnd') {
352+
headerField = headerField.toLowerCase();
353+
part.headers[headerField] = headerValue;
354+
355+
// matches either a quoted-string or a token (RFC 2616 section 19.5.1)
356+
var m = headerValue.match(/\bname=("([^"]*)"|([^\(\)<>@,;:\\"\/\[\]\?=\{\}\s\t/]+))/i);
357+
if (headerField == 'content-disposition') {
358+
if (m) {
359+
part.name = m[2] || m[3] || '';
360+
}
377361

378-
part.filename = this._fileName(headerValue);
379-
} else if (headerField == 'content-type') {
380-
part.mime = headerValue;
381-
} else if (headerField == 'content-transfer-encoding') {
382-
part.transferEncoding = headerValue.toLowerCase();
383-
}
362+
part.filename = this._fileName(headerValue);
363+
} else if (headerField == 'content-type') {
364+
part.mime = headerValue;
365+
} else if (headerField == 'content-transfer-encoding') {
366+
part.transferEncoding = headerValue.toLowerCase();
367+
}
384368

385-
headerField = '';
386-
headerValue = '';
387-
};
369+
headerField = '';
370+
headerValue = '';
371+
} else if (name === 'headersEnd') {
372+
373+
switch(part.transferEncoding){
374+
case 'binary':
375+
case '7bit':
376+
case '8bit': {
377+
const dataPropagation = ({name, buffer, start, end}) => {
378+
if (name === 'partData') {
379+
part.emit('data', buffer.slice(start, end));
380+
}
381+
};
382+
const dataStopPropagation = ({name}) => {
383+
if (name === 'partEnd') {
384+
part.emit('end');
385+
parser.off('data', dataPropagation);
386+
parser.off('data', dataStopPropagation);
387+
}
388+
};
389+
parser.on('data', dataPropagation);
390+
parser.on('data', dataStopPropagation);
391+
break;
392+
} case 'base64': {
393+
const dataPropagation = ({name, buffer, start, end}) => {
394+
if (name === 'partData') {
395+
part.transferBuffer += buffer.slice(start, end).toString('ascii');
396+
397+
/*
398+
four bytes (chars) in base64 converts to three bytes in binary
399+
encoding. So we should always work with a number of bytes that
400+
can be divided by 4, it will result in a number of buytes that
401+
can be divided vy 3.
402+
*/
403+
var offset = parseInt(part.transferBuffer.length / 4, 10) * 4;
404+
part.emit('data', Buffer.from(part.transferBuffer.substring(0, offset), 'base64'));
405+
part.transferBuffer = part.transferBuffer.substring(offset);
406+
}
407+
};
408+
const dataStopPropagation = ({name}) => {
409+
if (name === 'partEnd') {
410+
part.emit('data', Buffer.from(part.transferBuffer, 'base64'));
411+
part.emit('end');
412+
parser.off('data', dataPropagation);
413+
parser.off('data', dataStopPropagation);
414+
}
415+
};
416+
parser.on('data', dataPropagation);
417+
parser.on('data', dataStopPropagation);
418+
break;
419+
420+
} default:
421+
return this._error(new Error('unknown transfer-encoding'));
422+
}
388423

389-
parser.onHeadersEnd = () => {
390-
switch(part.transferEncoding){
391-
case 'binary':
392-
case '7bit':
393-
case '8bit':
394-
parser.onPartData = function(b, start, end) {
395-
part.emit('data', b.slice(start, end));
396-
};
397-
398-
parser.onPartEnd = function() {
399-
part.emit('end');
400-
};
401-
break;
402-
403-
case 'base64':
404-
parser.onPartData = function(b, start, end) {
405-
part.transferBuffer += b.slice(start, end).toString('ascii');
406-
407-
/*
408-
four bytes (chars) in base64 converts to three bytes in binary
409-
encoding. So we should always work with a number of bytes that
410-
can be divided by 4, it will result in a number of buytes that
411-
can be divided vy 3.
412-
*/
413-
var offset = parseInt(part.transferBuffer.length / 4, 10) * 4;
414-
part.emit('data', Buffer.from(part.transferBuffer.substring(0, offset), 'base64'));
415-
part.transferBuffer = part.transferBuffer.substring(offset);
416-
};
417-
418-
parser.onPartEnd = function() {
419-
part.emit('data', Buffer.from(part.transferBuffer, 'base64'));
420-
part.emit('end');
421-
};
422-
break;
423-
424-
default:
425-
return this._error(new Error('unknown transfer-encoding'));
424+
this.onPart(part);
425+
} else if (name === 'end') {
426+
this.ended = true;
427+
this._maybeEnd();
426428
}
427-
428-
this.onPart(part);
429-
};
430-
431-
432-
parser.onEnd = () => {
433-
this.ended = true;
434-
this._maybeEnd();
435-
};
429+
});
436430

437431
this._parser = parser;
438432
};
@@ -456,9 +450,9 @@ IncomingForm.prototype._initUrlencoded = function() {
456450

457451
var parser = new QuerystringParser(this.maxFields);
458452

459-
parser.onField = (key, val) => {
460-
this.emit('field', key, val);
461-
};
453+
parser.on('data', ({key, value}) => {
454+
this.emit('field', key, value);
455+
});
462456

463457
parser.onEnd = () => {
464458
this.ended = true;
@@ -525,16 +519,19 @@ IncomingForm.prototype._initOctetStream = function() {
525519
IncomingForm.prototype._initJSONencoded = function() {
526520
this.type = 'json';
527521

528-
var parser = new JSONParser(this);
522+
var parser = new JSONParser();
529523

530-
parser.onField = (key, val) => {
531-
this.emit('field', key, val);
532-
};
524+
parser.on('data', ({ key, value }) => {
525+
this.emit('field', key, value);
526+
});
527+
// parser.on('data', (key) => {
528+
// this.emit('field', key);
529+
// });
533530

534-
parser.onEnd = () => {
531+
parser.once('end', () => {
535532
this.ended = true;
536533
this._maybeEnd();
537-
};
534+
});
538535

539536
this._parser = parser;
540537
};

0 commit comments

Comments
 (0)