diff --git a/.aegir.js b/.aegir.js index d9a727161..20aa93b74 100644 --- a/.aegir.js +++ b/.aegir.js @@ -5,7 +5,7 @@ const createServer = require('ipfsd-ctl').createServer const server = createServer() module.exports = { - bundlesize: { maxSize: '237kB' }, + bundlesize: { maxSize: '240kB' }, webpack: { resolve: { mainFields: ['browser', 'main'] diff --git a/package.json b/package.json index cc37fe15e..fd4d82c69 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "glob": false, "fs": false, "stream": "readable-stream", - "ky-universal": "ky/umd" + "ky-universal": "ky/umd", + "./src/add/form-data.js": "./src/add/form-data.browser.js", + "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js" }, "repository": "github:ipfs/js-ipfs-http-client", "scripts": { @@ -40,6 +42,8 @@ "dependencies": { "abort-controller": "^3.0.0", "async": "^2.6.1", + "async-iterator-all": "^1.0.0", + "async-iterator-to-pull-stream": "^1.3.0", "bignumber.js": "^9.0.0", "bl": "^3.0.0", "bs58": "^4.0.1", @@ -52,9 +56,10 @@ "err-code": "^2.0.0", "explain-error": "^1.0.4", "flatmap": "0.0.3", + "fs-extra": "^8.1.0", "glob": "^7.1.3", "ipfs-block": "~0.8.1", - "ipfs-utils": "~0.0.3", + "ipfs-utils": "^0.1.0", "ipld-dag-cbor": "~0.15.0", "ipld-dag-pb": "~0.17.3", "ipld-raw": "^4.0.0", @@ -63,6 +68,8 @@ "is-stream": "^2.0.0", "iso-stream-http": "~0.1.2", "iso-url": "~0.4.6", + "it-glob": "0.0.4", + "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "just-kebab-case": "^1.1.0", "just-map-keys": "^1.1.0", @@ -82,6 +89,7 @@ "promisify-es6": "^1.0.3", "pull-defer": "~0.2.3", "pull-stream": "^3.6.9", + "pull-stream-to-async-iterator": "^1.0.2", "pull-to-stream": "~0.1.1", "pump": "^3.0.0", "qs": "^6.5.2", @@ -98,7 +106,7 @@ "cross-env": "^5.2.0", "dirty-chai": "^2.0.1", "go-ipfs-dep": "^0.4.22", - "interface-ipfs-core": "^0.111.0", + "interface-ipfs-core": "^0.112.0", "ipfsd-ctl": "~0.45.0", "nock": "^10.0.2", "stream-equal": "^1.1.1" diff --git a/src/add-from-fs/index.browser.js b/src/add-from-fs/index.browser.js new file mode 100644 index 000000000..81d551294 --- /dev/null +++ b/src/add-from-fs/index.browser.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = () => () => { throw new Error('unavailable in the browser') } diff --git a/src/add-from-fs/index.js b/src/add-from-fs/index.js new file mode 100644 index 000000000..01a82e166 --- /dev/null +++ b/src/add-from-fs/index.js @@ -0,0 +1,9 @@ +'use strict' + +const configure = require('../lib/configure') +const globSource = require('ipfs-utils/src/files/glob-source') + +module.exports = configure(({ ky }) => { + const add = require('../add')({ ky }) + return (path, options) => add(globSource(path, options), options) +}) diff --git a/src/add-from-url.js b/src/add-from-url.js new file mode 100644 index 000000000..336906ea1 --- /dev/null +++ b/src/add-from-url.js @@ -0,0 +1,22 @@ +'use strict' + +const kyDefault = require('ky-universal').default +const configure = require('./lib/configure') +const toIterable = require('./lib/stream-to-iterable') + +module.exports = configure(({ ky }) => { + const add = require('./add')({ ky }) + + return (url, options) => (async function * () { + options = options || {} + + const { body } = await kyDefault.get(url) + + const input = { + path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''), + content: toIterable(body) + } + + yield * add(input, options) + })() +}) diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js new file mode 100644 index 000000000..8e228c164 --- /dev/null +++ b/src/add/form-data.browser.js @@ -0,0 +1,31 @@ +'use strict' +/* eslint-env browser */ + +const { Buffer } = require('buffer') +const normaliseInput = require('ipfs-utils/src/files/normalise-input') + +exports.toFormData = async input => { + const files = normaliseInput(input) + const formData = new FormData() + let i = 0 + + for await (const file of files) { + if (file.content) { + // In the browser there's _currently_ no streaming upload, buffer up our + // async iterator chunks and append a big Blob :( + // One day, this will be browser streams + const bufs = [] + for await (const chunk of file.content) { + bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk) + } + + formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), file.path) + } else { + formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), file.path) + } + + i++ + } + + return formData +} diff --git a/src/add/form-data.js b/src/add/form-data.js new file mode 100644 index 000000000..2465ce2e3 --- /dev/null +++ b/src/add/form-data.js @@ -0,0 +1,42 @@ +'use strict' + +const FormData = require('form-data') +const { Buffer } = require('buffer') +const toStream = require('it-to-stream') +const normaliseInput = require('ipfs-utils/src/files/normalise-input') + +exports.toFormData = async input => { + const files = normaliseInput(input) + const formData = new FormData() + let i = 0 + + for await (const file of files) { + if (file.content) { + // In Node.js, FormData can be passed a stream so no need to buffer + formData.append( + `file-${i}`, + // FIXME: add a `path` property to the stream so `form-data` doesn't set + // a Content-Length header that is only the sum of the size of the + // header/footer when knownLength option (below) is null. + Object.assign( + toStream.readable(file.content), + { path: file.path || `file-${i}` } + ), + { + filepath: encodeURIComponent(file.path), + contentType: 'application/octet-stream', + knownLength: file.content.length // Send Content-Length header if known + } + ) + } else { + formData.append(`dir-${i}`, Buffer.alloc(0), { + filepath: encodeURIComponent(file.path), + contentType: 'application/x-directory' + }) + } + + i++ + } + + return formData +} diff --git a/src/add/index.js b/src/add/index.js new file mode 100644 index 000000000..34a5d33bd --- /dev/null +++ b/src/add/index.js @@ -0,0 +1,54 @@ +'use strict' + +const ndjson = require('iterable-ndjson') +const configure = require('../lib/configure') +const toIterable = require('../lib/stream-to-iterable') +const { toFormData } = require('./form-data') +const toCamel = require('../lib/object-to-camel') + +module.exports = configure(({ ky }) => { + return (input, options) => (async function * () { + options = options || {} + + const searchParams = new URLSearchParams(options.searchParams) + + searchParams.set('stream-channels', true) + if (options.chunker) searchParams.set('chunker', options.chunker) + if (options.cidVersion) searchParams.set('cid-version', options.cidVersion) + if (options.cidBase) searchParams.set('cid-base', options.cidBase) + if (options.enableShardingExperiment != null) searchParams.set('enable-sharding-experiment', options.enableShardingExperiment) + if (options.hashAlg) searchParams.set('hash', options.hashAlg) + if (options.onlyHash != null) searchParams.set('only-hash', options.onlyHash) + if (options.pin != null) searchParams.set('pin', options.pin) + if (options.progress) searchParams.set('progress', true) + if (options.quiet != null) searchParams.set('quiet', options.quiet) + if (options.quieter != null) searchParams.set('quieter', options.quieter) + if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves) + if (options.shardSplitThreshold) searchParams.set('shard-split-threshold', options.shardSplitThreshold) + if (options.silent) searchParams.set('silent', options.silent) + if (options.trickle != null) searchParams.set('trickle', options.trickle) + if (options.wrapWithDirectory != null) searchParams.set('wrap-with-directory', options.wrapWithDirectory) + + const res = await ky.post('add', { + timeout: options.timeout, + signal: options.signal, + headers: options.headers, + searchParams, + body: await toFormData(input) + }) + + for await (let file of ndjson(toIterable(res.body))) { + file = toCamel(file) + // console.log(file) + if (options.progress && file.bytes) { + options.progress(file.bytes) + } else { + yield toCoreInterface(file) + } + } + })() +}) + +function toCoreInterface ({ name, hash, size }) { + return { path: name, hash, size: parseInt(size) } +} diff --git a/src/files-mfs/write.js b/src/files-mfs/write.js index 0485406bd..33f1ec973 100644 --- a/src/files-mfs/write.js +++ b/src/files-mfs/write.js @@ -3,7 +3,6 @@ const promisify = require('promisify-es6') const concatStream = require('concat-stream') const once = require('once') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') const SendFilesStream = require('../utils/send-files-stream') module.exports = (send) => { @@ -29,8 +28,7 @@ module.exports = (send) => { const options = { args: pathDst, - qs: opts, - converter: FileResultStreamConverter + qs: opts } const stream = sendFilesStream({ qs: options }) diff --git a/src/files-regular/add-async-iterator.js b/src/files-regular/add-async-iterator.js deleted file mode 100644 index 3fa2b23ed..000000000 --- a/src/files-regular/add-async-iterator.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - return async function * (source, options) { - options = options || {} - options.converter = FileResultStreamConverter - - const stream = SendFilesStream(send, 'add')(options) - - for await (const entry of source) { - stream.write(entry) - } - - stream.end() - - for await (const entry of stream) { - yield entry - } - } -} diff --git a/src/files-regular/add-from-fs.js b/src/files-regular/add-from-fs.js deleted file mode 100644 index 2320fc537..000000000 --- a/src/files-regular/add-from-fs.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict' - -const isNode = require('detect-node') -const promisify = require('promisify-es6') -const SendOneFile = require('../utils/send-one-file-multiple-results') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - const sendOneFile = SendOneFile(send, 'add') - - return promisify((path, opts, callback) => { - if (typeof opts === 'function' && - callback === undefined) { - callback = opts - opts = {} - } - - // opts is the real callback -- - // 'callback' is being injected by promisify - if (typeof opts === 'function' && - typeof callback === 'function') { - callback = opts - opts = {} - } - - if (!isNode) { - return callback(new Error('fsAdd does not work in the browser')) - } - - if (typeof path !== 'string') { - return callback(new Error('"path" must be a string')) - } - - const requestOpts = { - qs: opts, - converter: FileResultStreamConverter - } - sendOneFile(path, requestOpts, callback) - }) -} diff --git a/src/files-regular/add-from-url.js b/src/files-regular/add-from-url.js deleted file mode 100644 index f1065f2de..000000000 --- a/src/files-regular/add-from-url.js +++ /dev/null @@ -1,70 +0,0 @@ -'use strict' - -const promisify = require('promisify-es6') -const { URL } = require('iso-url') -const { getRequest } = require('iso-stream-http') -const SendOneFile = require('../utils/send-one-file-multiple-results') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - const sendOneFile = SendOneFile(send, 'add') - - return promisify((url, opts, callback) => { - if (typeof (opts) === 'function' && - callback === undefined) { - callback = opts - opts = {} - } - - // opts is the real callback -- - // 'callback' is being injected by promisify - if (typeof opts === 'function' && - typeof callback === 'function') { - callback = opts - opts = {} - } - - if (!validUrl(url)) { - return callback(new Error('"url" param must be an http(s) url')) - } - - requestWithRedirect(url, opts, sendOneFile, callback) - }) -} - -const validUrl = (url) => typeof url === 'string' && url.startsWith('http') - -const requestWithRedirect = (url, opts, sendOneFile, callback) => { - const parsedUrl = new URL(url) - - const req = getRequest(parsedUrl, (res) => { - if (res.statusCode >= 400) { - return callback(new Error(`Failed to download with ${res.statusCode}`)) - } - - const redirection = res.headers.location - - if (res.statusCode >= 300 && res.statusCode < 400 && redirection) { - if (!validUrl(redirection)) { - return callback(new Error('redirection url must be an http(s) url')) - } - - requestWithRedirect(redirection, opts, sendOneFile, callback) - } else { - const requestOpts = { - qs: opts, - converter: FileResultStreamConverter - } - const fileName = decodeURIComponent(parsedUrl.pathname.split('/').pop()) - - sendOneFile({ - content: res, - path: fileName - }, requestOpts, callback) - } - }) - - req.once('error', callback) - - req.end() -} diff --git a/src/files-regular/add-pull-stream.js b/src/files-regular/add-pull-stream.js deleted file mode 100644 index 2076ffa8d..000000000 --- a/src/files-regular/add-pull-stream.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') -const toPull = require('stream-to-pull-stream') - -module.exports = (send) => { - return (options) => { - options = options || {} - options.converter = FileResultStreamConverter - return toPull(SendFilesStream(send, 'add')({ qs: options })) - } -} diff --git a/src/files-regular/add-readable-stream.js b/src/files-regular/add-readable-stream.js deleted file mode 100644 index 320abe692..000000000 --- a/src/files-regular/add-readable-stream.js +++ /dev/null @@ -1,12 +0,0 @@ -'use strict' - -const SendFilesStream = require('../utils/send-files-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') - -module.exports = (send) => { - return (options) => { - options = options || {} - options.converter = FileResultStreamConverter - return SendFilesStream(send, 'add')(options) - } -} diff --git a/src/files-regular/add.js b/src/files-regular/add.js deleted file mode 100644 index cb5898265..000000000 --- a/src/files-regular/add.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict' - -const promisify = require('promisify-es6') -const ConcatStream = require('concat-stream') -const once = require('once') -const { isSource } = require('is-pull-stream') -const FileResultStreamConverter = require('../utils/file-result-stream-converter') -const SendFilesStream = require('../utils/send-files-stream') -const validateAddInput = require('ipfs-utils/src/files/add-input-validation') - -module.exports = (send) => { - const createAddStream = SendFilesStream(send, 'add') - - const add = promisify((_files, options, _callback) => { - if (typeof options === 'function') { - _callback = options - options = null - } - const callback = once(_callback) - - if (!options) { - options = {} - } - options.converter = FileResultStreamConverter - - try { - validateAddInput(_files) - } catch (err) { - return callback(err) - } - - const files = [].concat(_files) - - const stream = createAddStream({ qs: options }) - const concat = ConcatStream((result) => callback(null, result)) - stream.once('error', callback) - stream.pipe(concat) - - files.forEach((file) => stream.write(file)) - stream.end() - }) - - return function () { - const args = Array.from(arguments) - - // If we files.add(), then promisify thinks the pull stream is - // a callback! Add an empty options object in this case so that a promise - // is returned. - if (args.length === 1 && isSource(args[0])) { - args.push({}) - } - - return add.apply(null, args) - } -} diff --git a/src/files-regular/index.js b/src/files-regular/index.js index e5b49e495..d098516f1 100644 --- a/src/files-regular/index.js +++ b/src/files-regular/index.js @@ -1,18 +1,47 @@ 'use strict' +const nodeify = require('promise-nodeify') const moduleConfig = require('../utils/module-config') +const { collectify, pullify, streamify } = require('../lib/converters') module.exports = (arg) => { const send = moduleConfig(arg) + const add = require('../add')(arg) + const addFromFs = require('../add-from-fs')(arg) + const addFromURL = require('../add-from-url')(arg) return { - add: require('../files-regular/add')(send), - addReadableStream: require('../files-regular/add-readable-stream')(send), - addPullStream: require('../files-regular/add-pull-stream')(send), - addFromFs: require('../files-regular/add-from-fs')(send), - addFromURL: require('../files-regular/add-from-url')(send), - addFromStream: require('../files-regular/add')(send), - _addAsyncIterator: require('../files-regular/add-async-iterator')(send), + add: (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(collectify(add)(input, options), callback) + }, + addReadableStream: streamify.transform(add), + addPullStream: pullify.transform(add), + addFromFs: (path, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(collectify(addFromFs)(path, options), callback) + }, + addFromURL: (url, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(collectify(addFromURL)(url, options), callback) + }, + addFromStream: (input, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} + } + return nodeify(collectify(add)(input, options), callback) + }, + _addAsyncIterator: add, cat: require('../files-regular/cat')(send), catReadableStream: require('../files-regular/cat-readable-stream')(send), catPullStream: require('../files-regular/cat-pull-stream')(send), diff --git a/src/lib/converters.js b/src/lib/converters.js new file mode 100644 index 000000000..7c5965feb --- /dev/null +++ b/src/lib/converters.js @@ -0,0 +1,16 @@ +'use strict' + +const toPull = require('async-iterator-to-pull-stream') +const all = require('async-iterator-all') +const toStream = require('it-to-stream') + +exports.collectify = fn => (...args) => all(fn(...args)) + +exports.pullify = { + source: fn => (...args) => toPull(fn(...args)), + transform: fn => (...args) => toPull.transform(source => fn(source, ...args)) +} + +exports.streamify = { + transform: fn => (...args) => toStream.transform(source => fn(source, ...args), { objectMode: true }) +} diff --git a/src/lib/object-to-camel.js b/src/lib/object-to-camel.js new file mode 100644 index 000000000..f13b2b6a1 --- /dev/null +++ b/src/lib/object-to-camel.js @@ -0,0 +1,21 @@ +'use strict' + +// Convert object properties to camel case. +// NOT recursive! +// e.g. +// AgentVersion => agentVersion +// ID => id +module.exports = obj => { + if (obj == null) return obj + const caps = /^[A-Z]+$/ + return Object.keys(obj).reduce((camelObj, k) => { + if (caps.test(k)) { // all caps + camelObj[k.toLowerCase()] = obj[k] + } else if (caps.test(k[0])) { // pascal + camelObj[k[0].toLowerCase() + k.slice(1)] = obj[k] + } else { + camelObj[k] = obj[k] + } + return camelObj + }, {}) +} diff --git a/src/utils/file-result-stream-converter.js b/src/utils/file-result-stream-converter.js deleted file mode 100644 index 7f5b19aeb..000000000 --- a/src/utils/file-result-stream-converter.js +++ /dev/null @@ -1,44 +0,0 @@ -'use strict' - -const TransformStream = require('readable-stream').Transform - -/* - Transforms a stream of {Name, Hash} objects to include size - of the DAG object. - - Usage: inputStream.pipe(new FileResultStreamConverter()) - - Input object format: - { - Name: '/path/to/file/foo.txt', - Hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP' - Size: '20' - } - - Output object format: - { - path: '/path/to/file/foo.txt', - hash: 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', - size: 20 - } -*/ -class FileResultStreamConverter extends TransformStream { - constructor (options) { - const opts = Object.assign({}, options || {}, { objectMode: true }) - super(opts) - } - - _transform (obj, enc, callback) { - if (!obj.Hash) { - return callback() - } - - callback(null, { - path: obj.Name, - hash: obj.Hash, - size: parseInt(obj.Size, 10) - }) - } -} - -module.exports = FileResultStreamConverter diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index 467af3cea..710b8396f 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -1,28 +1,12 @@ 'use strict' -function requireCommands () { - return { - // Files Regular (not MFS) - add: require('../files-regular/add'), - addReadableStream: require('../files-regular/add-readable-stream'), - addPullStream: require('../files-regular/add-pull-stream'), - addFromFs: require('../files-regular/add-from-fs'), - addFromURL: require('../files-regular/add-from-url'), - addFromStream: require('../files-regular/add'), - _addAsyncIterator: require('../files-regular/add-async-iterator'), - cat: require('../files-regular/cat'), - catReadableStream: require('../files-regular/cat-readable-stream'), - catPullStream: require('../files-regular/cat-pull-stream'), - get: require('../files-regular/get'), - getReadableStream: require('../files-regular/get-readable-stream'), - getPullStream: require('../files-regular/get-pull-stream'), - ls: require('../files-regular/ls'), - lsReadableStream: require('../files-regular/ls-readable-stream'), - lsPullStream: require('../files-regular/ls-pull-stream'), - refs: require('../files-regular/refs'), - refsReadableStream: require('../files-regular/refs-readable-stream'), - refsPullStream: require('../files-regular/refs-pull-stream'), +function requireCommands (send, config) { + const cmds = { + ...require('../files-regular')(config), + getEndpointConfig: require('../get-endpoint-config')(config) + } + const subCmds = { // Files MFS (Mutable Filesystem) files: require('../files-mfs'), @@ -60,21 +44,14 @@ function requireCommands () { stats: require('../stats'), update: require('../update'), version: require('../version'), - resolve: require('../resolve'), - // ipfs-http-client instance - getEndpointConfig: (send, config) => require('../get-endpoint-config')(config) + resolve: require('../resolve') } -} - -function loadCommands (send, config) { - const files = requireCommands() - const cmds = {} - Object.keys(files).forEach((file) => { - cmds[file] = files[file](send, config) + Object.keys(subCmds).forEach((file) => { + cmds[file] = subCmds[file](send, config) }) return cmds } -module.exports = loadCommands +module.exports = requireCommands diff --git a/test/files-mfs.spec.js b/test/files-mfs.spec.js index 311659200..d1c2e0900 100644 --- a/test/files-mfs.spec.js +++ b/test/files-mfs.spec.js @@ -91,7 +91,7 @@ describe('.files (the MFS API part)', function () { it('.add with cid-version=1 and raw-leaves=false', async () => { const expectedCid = 'bafybeifogzovjqrcxvgt7g36y7g63hvwvoakledwk4b2fr2dl4wzawpnny' - const options = { 'cid-version': 1, 'raw-leaves': false } + const options = { cidVersion: 1, rawLeaves: false } const res = await ipfs.add(testfile, options) @@ -149,7 +149,7 @@ describe('.files (the MFS API part)', function () { path: content + '.txt', content: Buffer.from(content) } - const options = { hash: name, 'raw-leaves': false } + const options = { hashAlg: name, rawLeaves: false } const res = await ipfs.add([file], options) @@ -222,7 +222,7 @@ describe('.files (the MFS API part)', function () { path: content + '.txt', content: Buffer.from(content) } - const options = { hash: name, 'raw-leaves': false } + const options = { hashAlg: name, rawLeaves: false } const res = await ipfs.add([file], options) diff --git a/test/interface.spec.js b/test/interface.spec.js index fb54c422b..32c6f6749 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -126,11 +126,6 @@ describe('interface-ipfs-core tests', () => { name: 'should add readable stream of valid files and dirs', reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339' }, - // .addFromStream - isNode ? null : { - name: 'addFromStream', - reason: 'Not designed to run in the browser' - }, // .addFromFs isNode ? null : { name: 'addFromFs',