From cb4a7eba5a6e1cd2f6f14220f2fb3f1fd9d9dd80 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 21 Jul 2020 23:10:00 -0700 Subject: [PATCH 1/4] feat: accept Uint8Arrays in place of Buffers --- src/base.js | 23 ++++++++++++++++++++ src/constants.js | 14 ++++++++---- src/index.js | 49 ++++++++++++++++++++++++------------------ src/rfc4648.js | 27 +++++++++++++++++++++++ src/util.js | 33 ++++++++++++++++++++++++++++ test/multibase.spec.js | 29 +++++++++++++------------ 6 files changed, 136 insertions(+), 39 deletions(-) create mode 100644 src/util.js diff --git a/src/base.js b/src/base.js index fd37ca9..28d6f27 100644 --- a/src/base.js +++ b/src/base.js @@ -1,7 +1,22 @@ +// @ts-check 'use strict' const { Buffer } = require('buffer') +/** + * @typedef {Object} Codec + * @property {function(Uint8Array):string} encode + * @property {function(string):Uint8Array} decode + * + * @typedef {function(string):Codec} CodecFactory + */ + class Base { + /** + * @param {string} name + * @param {string} code + * @param {CodecFactory} implementation + * @param {string} alphabet + */ constructor (name, code, implementation, alphabet) { this.name = name this.code = code @@ -10,10 +25,18 @@ class Base { this.engine = implementation(alphabet) } + /** + * @param {Uint8Array} buf + * @returns {string} + */ encode (buf) { return this.engine.encode(buf) } + /** + * @param {string} string + * @returns {Uint8Array} + */ decode (string) { for (const char of string) { if (this.alphabet && this.alphabet.indexOf(char) < 0) { diff --git a/src/constants.js b/src/constants.js index fa9b2b1..0435aee 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,18 +1,24 @@ +// @ts-check 'use strict' const baseX = require('base-x') -const { Buffer } = require('buffer') const Base = require('./base.js') const rfc4648 = require('./rfc4648') +const { decodeText, encodeText } = require('./util') const identity = () => { return { - encode: (data) => Buffer.from(data).toString(), - decode: (string) => Buffer.from(string) + encode: decodeText, + decode: encodeText } } -// name, code, implementation, alphabet +/** + * @typedef {import('./base').CodecFactory} CodecFactory + * + * name, code, implementation, alphabet + * @type {Array<[string, string, CodecFactory, string]>} + */ const constants = [ ['identity', '\x00', identity, ''], ['base2', '0', rfc4648(1), '01'], diff --git a/src/index.js b/src/index.js index a9915c9..4852ba2 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,4 @@ +// @ts-check /** * Implementation of the [multibase](https://github.com/multiformats/multibase) specification. * @@ -7,6 +8,7 @@ const { Buffer } = require('buffer') const constants = require('./constants') +const { decodeText, asBuffer } = require('./util') /** @typedef {import("./base")} Base */ @@ -14,7 +16,7 @@ const constants = require('./constants') * Create a new buffer with the multibase varint+code. * * @param {string|number} nameOrCode - The multibase name or code number. - * @param {Buffer} buf - The data to be prefixed with multibase. + * @param {Uint8Array} buf - The data to be prefixed with multibase. * @returns {Buffer} * @throws {Error} Will throw if the encoding is not supported */ @@ -22,16 +24,21 @@ function multibase (nameOrCode, buf) { if (!buf) { throw new Error('requires an encoded buffer') } - const enc = encoding(nameOrCode) - validEncode(enc.name, buf) - return Buffer.concat([enc.codeBuf, buf]) + const { name, codeBuf } = encoding(nameOrCode) + validEncode(name, buf) + + const buffer = Buffer.alloc(codeBuf.length + buf.length) + buffer.set(codeBuf, 0) + buffer.set(buf, codeBuf.length) + + return buffer } /** * Encode data with the specified base and add the multibase prefix. * * @param {string|number} nameOrCode - The multibase name or code number. - * @param {Buffer} buf - The data to be encoded. + * @param {Uint8Array} buf - The data to be encoded. * @returns {Buffer} * @throws {Error} Will throw if the encoding is not supported * @@ -43,17 +50,17 @@ function encode (nameOrCode, buf) { } /** - * Takes a buffer or string encoded with multibase header, decodes it and + * Takes a Uint8Array or string encoded with multibase header, decodes it and * returns the decoded buffer * - * @param {Buffer|string} data + * @param {Uint8Array|string} data * @returns {Buffer} * @throws {Error} Will throw if the encoding is not supported * */ function decode (data) { - if (Buffer.isBuffer(data)) { - data = data.toString() + if (ArrayBuffer.isView(data)) { + data = decodeText(data) } const prefix = data[0] @@ -62,18 +69,18 @@ function decode (data) { data = data.toLowerCase() } const enc = encoding(data[0]) - return Buffer.from(enc.decode(data.substring(1))) + return asBuffer(enc.decode(data.substring(1))) } /** * Is the given data multibase encoded? * - * @param {Buffer|string} data - * @returns {boolean} + * @param {Uint8Array|string} data + * @returns {false|string} */ function isEncoded (data) { - if (Buffer.isBuffer(data)) { - data = data.toString() + if (data instanceof Uint8Array) { + data = decodeText(data) } // Ensure bufOrString is a string @@ -93,19 +100,19 @@ function isEncoded (data) { * Validate encoded data * * @param {string} name - * @param {Buffer} buf - * @returns {undefined} + * @param {Uint8Array} buf + * @returns {void} * @throws {Error} Will throw if the encoding is not supported */ function validEncode (name, buf) { const enc = encoding(name) - enc.decode(buf.toString()) + enc.decode(decodeText(buf)) } /** * Get the encoding by name or code * - * @param {string} nameOrCode + * @param {string|number} nameOrCode * @returns {Base} * @throws {Error} Will throw if the encoding is not supported */ @@ -122,13 +129,13 @@ function encoding (nameOrCode) { /** * Get encoding from data * - * @param {string|Buffer} data + * @param {string|Uint8Array} data * @returns {Base} * @throws {Error} Will throw if the encoding is not supported */ function encodingFromData (data) { - if (Buffer.isBuffer(data)) { - data = data.toString() + if (data instanceof Uint8Array) { + data = decodeText(data) } return encoding(data[0]) diff --git a/src/rfc4648.js b/src/rfc4648.js index b020a5a..e1e4ba3 100644 --- a/src/rfc4648.js +++ b/src/rfc4648.js @@ -1,5 +1,14 @@ +// @ts-check 'use strict' +/** @typedef {import('./base').CodecFactory} CodecFactory */ + +/** + * @param {string} string + * @param {string} alphabet + * @param {number} bitsPerChar + * @returns {Uint8Array} + */ const decode = (string, alphabet, bitsPerChar) => { // Build the character lookup table: const codes = {} @@ -46,6 +55,12 @@ const decode = (string, alphabet, bitsPerChar) => { return out } +/** + * @param {Uint8Array} data + * @param {string} alphabet + * @param {number} bitsPerChar + * @returns {string} + */ const encode = (data, alphabet, bitsPerChar) => { const pad = alphabet[alphabet.length - 1] === '=' const mask = (1 << bitsPerChar) - 1 @@ -80,11 +95,23 @@ const encode = (data, alphabet, bitsPerChar) => { return out } +/** + * @param {number} bitsPerChar + * @returns {CodecFactory} + */ module.exports = (bitsPerChar) => (alphabet) => { return { + /** + * @param {Uint8Array} input + * @returns {string} + */ encode (input) { return encode(input, alphabet, bitsPerChar) }, + /** + * @param {string} input + * @returns {Uint8Array} + */ decode (input) { return decode(input, alphabet, bitsPerChar) } diff --git a/src/util.js b/src/util.js new file mode 100644 index 0000000..fd85d5b --- /dev/null +++ b/src/util.js @@ -0,0 +1,33 @@ +// @ts-check +'use strict' + +const { Buffer } = require('buffer') + +const textDecoder = typeof TextDecoder !== 'undefined' + ? new TextDecoder() + : new (require('util').TextDecoder)() + +const textEncoder = typeof TextEncoder !== 'undefined' + ? new TextEncoder() + : new (require('util').TextEncoder)() + +/** + * @param {ArrayBufferView|ArrayBuffer} bytes + * @returns {string} + */ +const decodeText = (bytes) => textDecoder.decode(bytes) + +/** + * @param {string} text + * @returns {Uint8Array} + */ +const encodeText = (text) => textEncoder.encode(text) + +/** + * @param {ArrayBufferView} bytes + * @returns {Buffer} + */ +const asBuffer = ({ buffer, byteLength, byteOffset }) => + Buffer.from(buffer, byteOffset, byteLength) + +module.exports = { decodeText, encodeText, asBuffer } diff --git a/test/multibase.spec.js b/test/multibase.spec.js index cd72fb1..0a39a70 100644 --- a/test/multibase.spec.js +++ b/test/multibase.spec.js @@ -3,6 +3,7 @@ const { expect } = require('aegir/utils/chai') const { Buffer } = require('buffer') +const { encodeText } = require('../src/util') const multibase = require('../src') const constants = require('../src/constants.js') @@ -10,8 +11,8 @@ const unsupportedBases = [] const supportedBases = [ - ['base16', Buffer.from([0x01]), 'f01'], - ['base16', Buffer.from([15]), 'f0f'], + ['base16', Buffer.from([0x01]).toString(), 'f01'], + ['base16', Buffer.from([15]).toString(), 'f0f'], ['base16', 'f', 'f66'], ['base16', 'fo', 'f666f'], ['base16', 'foo', 'f666f6f'], @@ -97,13 +98,13 @@ describe('multibase', () => { it('fails on non supported name', () => { expect(() => { - multibase('base1001', Buffer.from('meh')) + multibase('base1001', encodeText('meh')) }).to.throw(Error) }) it('fails on non supported code', () => { expect(() => { - multibase('6', Buffer.from('meh')) + multibase('6', encodeText('meh')) }).to.throw(Error) }) }) @@ -116,26 +117,26 @@ describe('multibase', () => { describe(name, () => { it('adds multibase code to valid encoded buffer, by name', () => { if (typeof input === 'string') { - const buf = Buffer.from(input) - const encodedBuf = Buffer.from(base.encode(buf)) + const buf = encodeText(input) + const encodedBuf = encodeText(base.encode(buf)) const multibasedBuf = multibase(base.name, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) } else { - const encodedBuf = Buffer.from(base.encode(input)) + const encodedBuf = encodeText(base.encode(input)) const multibasedBuf = multibase(base.name, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) } }) it('adds multibase code to valid encoded buffer, by code', () => { - const buf = Buffer.from(input) - const encodedBuf = Buffer.from(base.encode(buf)) + const buf = encodeText(input) + const encodedBuf = encodeText(base.encode(buf)) const multibasedBuf = multibase(base.code, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) }) it('fails to add multibase code to invalid encoded buffer', () => { - const nonEncodedBuf = Buffer.from('^!@$%!#$%@#y') + const nonEncodedBuf = encodeText('^!@$%!#$%@#y') expect(() => { multibase(base.name, nonEncodedBuf) }).to.throw(Error) @@ -147,7 +148,7 @@ describe('multibase', () => { }) it('isEncoded buffer', () => { - const multibasedStr = Buffer.from(output) + const multibasedStr = encodeText(output) const name = multibase.isEncoded(multibasedStr) expect(name).to.equal(base.name) }) @@ -162,7 +163,7 @@ describe('multibase.encode ', () => { const output = elements[2] describe(name, () => { it('encodes a buffer', () => { - const buf = Buffer.from(input) + const buf = encodeText(input) const multibasedBuf = multibase.encode(name, buf) expect(multibasedBuf.toString()).to.equal(output) }) @@ -191,7 +192,7 @@ describe('multibase.decode', () => { }) it('decodes a buffer', () => { - const multibasedBuf = Buffer.from(output) + const multibasedBuf = encodeText(output) const buf = multibase.decode(multibasedBuf) expect(buf).to.eql(Buffer.from(input)) }) @@ -204,7 +205,7 @@ for (const elements of unsupportedBases) { describe(name, () => { it('fails on non implemented name', () => { expect(() => { - multibase(name, Buffer.from('meh')) + multibase(name, encodeText('meh')) }).to.throw(Error) }) }) From babf53f344039be12831942221c004b0c6d8b8c7 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 22 Jul 2020 15:28:01 -0700 Subject: [PATCH 2/4] chore: remove dependecy on node util --- package.json | 3 ++- src/util.js | 11 +++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 281562b..2274cdc 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ }, "dependencies": { "base-x": "^3.0.8", - "buffer": "^5.5.0" + "buffer": "^5.5.0", + "web-encoding": "^1.0.1" }, "devDependencies": { "aegir": "^25.0.0", diff --git a/src/util.js b/src/util.js index fd85d5b..b391830 100644 --- a/src/util.js +++ b/src/util.js @@ -2,21 +2,16 @@ 'use strict' const { Buffer } = require('buffer') +const { TextEncoder, TextDecoder } = require('web-encoding') -const textDecoder = typeof TextDecoder !== 'undefined' - ? new TextDecoder() - : new (require('util').TextDecoder)() - -const textEncoder = typeof TextEncoder !== 'undefined' - ? new TextEncoder() - : new (require('util').TextEncoder)() - +const textDecoder = new TextDecoder() /** * @param {ArrayBufferView|ArrayBuffer} bytes * @returns {string} */ const decodeText = (bytes) => textDecoder.decode(bytes) +const textEncoder = new TextEncoder() /** * @param {string} text * @returns {Uint8Array} From 2226e2db782260409de04eb6877b72af084b9e67 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 22 Jul 2020 15:57:30 -0700 Subject: [PATCH 3/4] chore: test with both Uint8Array & Buffer --- test/multibase.spec.js | 47 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/test/multibase.spec.js b/test/multibase.spec.js index 0a39a70..381bd97 100644 --- a/test/multibase.spec.js +++ b/test/multibase.spec.js @@ -84,6 +84,11 @@ const supportedBases = [ ['base64urlpad', '÷ïÿ🥰÷ïÿ😎🥶🤯', 'Uw7fDr8O_8J-lsMO3w6_Dv_CfmI7wn6W28J-krw=='] ] +const they = (label, def) => { + it(`${label} (Buffer)`, def.bind(null, Buffer.from)) + it(`${label} (Uint8Array)`, def.bind(null, encodeText)) +} + describe('multibase', () => { describe('generic', () => { it('fails on no args', () => { @@ -96,15 +101,15 @@ describe('multibase', () => { }).to.throw(Error) }) - it('fails on non supported name', () => { + they('fails on non supported name', (encode) => { expect(() => { - multibase('base1001', encodeText('meh')) + multibase('base1001', encode('meh')) }).to.throw(Error) }) - it('fails on non supported code', () => { + they('fails on non supported code', (encode) => { expect(() => { - multibase('6', encodeText('meh')) + multibase('6', encode('meh')) }).to.throw(Error) }) }) @@ -115,28 +120,28 @@ describe('multibase', () => { const output = elements[2] const base = constants.names[name] describe(name, () => { - it('adds multibase code to valid encoded buffer, by name', () => { + they('adds multibase code to valid encoded buffer, by name', (encode) => { if (typeof input === 'string') { - const buf = encodeText(input) - const encodedBuf = encodeText(base.encode(buf)) + const buf = encode(input) + const encodedBuf = encode(base.encode(buf)) const multibasedBuf = multibase(base.name, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) } else { - const encodedBuf = encodeText(base.encode(input)) + const encodedBuf = encode(base.encode(input)) const multibasedBuf = multibase(base.name, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) } }) - it('adds multibase code to valid encoded buffer, by code', () => { - const buf = encodeText(input) - const encodedBuf = encodeText(base.encode(buf)) + they('adds multibase code to valid encoded buffer, by code', (encode) => { + const buf = encode(input) + const encodedBuf = encode(base.encode(buf)) const multibasedBuf = multibase(base.code, encodedBuf) expect(multibasedBuf.toString()).to.equal(output) }) - it('fails to add multibase code to invalid encoded buffer', () => { - const nonEncodedBuf = encodeText('^!@$%!#$%@#y') + they('fails to add multibase code to invalid encoded buffer', (encode) => { + const nonEncodedBuf = encode('^!@$%!#$%@#y') expect(() => { multibase(base.name, nonEncodedBuf) }).to.throw(Error) @@ -147,8 +152,8 @@ describe('multibase', () => { expect(name).to.equal(base.name) }) - it('isEncoded buffer', () => { - const multibasedStr = encodeText(output) + they('isEncoded buffer', (encode) => { + const multibasedStr = encode(output) const name = multibase.isEncoded(multibasedStr) expect(name).to.equal(base.name) }) @@ -162,8 +167,8 @@ describe('multibase.encode ', () => { const input = elements[1] const output = elements[2] describe(name, () => { - it('encodes a buffer', () => { - const buf = encodeText(input) + they('encodes a buffer', (encode) => { + const buf = encode(input) const multibasedBuf = multibase.encode(name, buf) expect(multibasedBuf.toString()).to.equal(output) }) @@ -191,8 +196,8 @@ describe('multibase.decode', () => { expect(buf).to.eql(Buffer.from(input)) }) - it('decodes a buffer', () => { - const multibasedBuf = encodeText(output) + they('decodes a buffer', (encode) => { + const multibasedBuf = encode(output) const buf = multibase.decode(multibasedBuf) expect(buf).to.eql(Buffer.from(input)) }) @@ -203,9 +208,9 @@ describe('multibase.decode', () => { for (const elements of unsupportedBases) { const name = elements[0] describe(name, () => { - it('fails on non implemented name', () => { + they('fails on non implemented name', (encode) => { expect(() => { - multibase(name, encodeText('meh')) + multibase(name, encode('meh')) }).to.throw(Error) }) }) From 51cef5d690d29c81a8d7809c940401925a4619ee Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 23 Jul 2020 11:04:07 -0700 Subject: [PATCH 4/4] chore: bump web-encoding version Addresses https://github.com/multiformats/js-multibase/pull/61#pullrequestreview-453849068 --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2274cdc..349ca15 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "dependencies": { "base-x": "^3.0.8", "buffer": "^5.5.0", - "web-encoding": "^1.0.1" + "web-encoding": "^1.0.2" }, "devDependencies": { "aegir": "^25.0.0", @@ -63,4 +63,4 @@ "theobat ", "Henrique Dias " ] -} +} \ No newline at end of file