From bf7e38d365864d0b07d0e09521a61c8899011d12 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Tue, 5 Sep 2017 15:25:46 +0100 Subject: [PATCH 01/11] Progress bar for cli --- package.json | 4 ++ src/cli/commands/files/add.js | 87 ++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index dcd9edfceb..fa551da129 100644 --- a/package.json +++ b/package.json @@ -93,11 +93,14 @@ "async": "^2.5.0", "bl": "^1.2.1", "boom": "^5.2.0", + "byteman": "^1.3.5", + "cids": "~0.5.1", "debug": "^3.0.1", "cids": "^0.5.1", "file-type": "^6.1.0", "filesize": "^3.5.10", "fsm-event": "^2.1.0", + "get-folder-size": "^1.0.0", "glob": "^7.1.2", "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", @@ -138,6 +141,7 @@ "peer-book": "~0.5.1", "peer-id": "~0.10.1", "peer-info": "~0.11.0", + "progress": "^2.0.0", "promisify-es6": "^1.0.3", "pull-file": "^1.0.0", "pull-paramap": "^1.2.2", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 82fe7ecb18..25c757673c 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,6 +8,10 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const Progress = require('progress') +const getFolderSize = require('get-folder-size') +const byteman = require('byteman') +const async = require('async') const utils = require('../../utils') const print = require('../../utils').print @@ -40,6 +44,27 @@ function checkPath (inPath, recursive) { return inPath } +function getTotalBytes (path, recursive, cb) { + if (recursive) { + getFolderSize(path, cb) + } else { + fs.stat(path, (err, stat) => cb(err, stat.size)) + } +} + +function createProgressBar (totalBytes) { + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` + + // 16 MB / 34 MB [=========== ] 48% 5.8s // + return new Progress(barFormat, { + incomplete: ' ', + clear: true, + stream: process.stdout, + total: totalBytes + }) +} + function addPipeline (index, addStream, list, argv) { const { wrapWithDirectory, @@ -47,7 +72,6 @@ function addPipeline (index, addStream, list, argv) { quieter, silent } = argv - pull( zip( pull.values(list), @@ -102,6 +126,12 @@ module.exports = { describe: 'Add a file to IPFS using the UnixFS data format', builder: { + progress: { + alias: 'p', + type: 'boolean', + default: true, + describe: 'Stream progress data' + }, recursive: { alias: 'r', type: 'boolean', @@ -185,34 +215,45 @@ module.exports = { } const ipfs = argv.ipfs - // TODO: revist when interface-ipfs-core exposes pull-streams - let createAddStream = (cb) => { - ipfs.files.createAddStream(options, (err, stream) => { - cb(err, err ? null : toPull.transform(stream)) - }) - } + let list = [] + let currentBytes = 0 - if (typeof ipfs.files.createAddPullStream === 'function') { - createAddStream = (cb) => { - cb(null, ipfs.files.createAddPullStream(options)) - } - } + async.waterfall([ + (next) => glob(path.join(inPath, '/**/*'), next), + (globResult, next) => { + list = globResult.length === 0 ? [inPath] : globResult - createAddStream((err, addStream) => { - if (err) { - throw err - } + getTotalBytes(inPath, argv.recursive, next) + }, + (totalBytes, next) => { + if (argv.progress) { + const bar = createProgressBar(totalBytes) + options.progress = function (byteLength) { + currentBytes += byteLength + bar.tick(byteLength, {progress: byteman(currentBytes, 2, 'MB')}) + } + } + + // TODO: revist when interface-ipfs-core exposes pull-streams - glob(path.join(inPath, '/**/*'), (err, list) => { - if (err) { - throw err + let createAddStream = (cb) => { + ipfs.files.createAddStream(options, (err, stream) => { + cb(err, err ? null : toPull.transform(stream)) + }) } - if (list.length === 0) { - list = [inPath] + + if (typeof ipfs.files.createAddPullStream === 'function') { + createAddStream = (cb) => { + cb(null, ipfs.files.createAddPullStream(options)) + } } - addPipeline(index, addStream, list, argv) - }) + createAddStream(next) + } + ], (err, addStream) => { + if (err) throw err + + addPipeline(index, addStream, list, argv) }) } } From cb428447b1bdac7a726a2c3bfdbf6a4deef02593 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Tue, 5 Sep 2017 16:00:55 +0100 Subject: [PATCH 02/11] requires async/waterfall in place of async --- src/cli/commands/files/add.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 25c757673c..3be4319d92 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -11,7 +11,7 @@ const toPull = require('stream-to-pull-stream') const Progress = require('progress') const getFolderSize = require('get-folder-size') const byteman = require('byteman') -const async = require('async') +const waterfall = require('async/waterfall') const utils = require('../../utils') const print = require('../../utils').print @@ -218,7 +218,7 @@ module.exports = { let list = [] let currentBytes = 0 - async.waterfall([ + waterfall([ (next) => glob(path.join(inPath, '/**/*'), next), (globResult, next) => { list = globResult.length === 0 ? [inPath] : globResult From d311dc597fabeda28ae5b34a25366c251749e4ff Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Thu, 7 Sep 2017 16:43:52 +0100 Subject: [PATCH 03/11] adds a test for createProgressBar --- src/cli/commands/files/add.js | 15 +-------------- src/cli/utils.js | 15 +++++++++++++++ test/cli/progress-bar.js | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 test/cli/progress-bar.js diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 3be4319d92..805f80217f 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,12 +8,12 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') -const Progress = require('progress') const getFolderSize = require('get-folder-size') const byteman = require('byteman') const waterfall = require('async/waterfall') const utils = require('../../utils') const print = require('../../utils').print +const createProgressBar = require('../../utils').createProgressBar const WRAPPER = 'wrapper/' @@ -52,19 +52,6 @@ function getTotalBytes (path, recursive, cb) { } } -function createProgressBar (totalBytes) { - const total = byteman(totalBytes, 2, 'MB') - const barFormat = `:progress / ${total} [:bar] :percent :etas` - - // 16 MB / 34 MB [=========== ] 48% 5.8s // - return new Progress(barFormat, { - incomplete: ' ', - clear: true, - stream: process.stdout, - total: totalBytes - }) -} - function addPipeline (index, addStream, list, argv) { const { wrapWithDirectory, diff --git a/src/cli/utils.js b/src/cli/utils.js index 76ab67d969..4565fc316e 100644 --- a/src/cli/utils.js +++ b/src/cli/utils.js @@ -9,6 +9,8 @@ const path = require('path') const debug = require('debug') const log = debug('cli') log.error = debug('cli:error') +const Progress = require('progress') +const byteman = require('byteman') exports = module.exports @@ -85,3 +87,16 @@ exports.print = (msg, newline) => { process.stdout.write(msg) } } + +exports.createProgressBar = (totalBytes) => { + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` + + // 16 MB / 34 MB [=========== ] 48% 5.8s // + return new Progress(barFormat, { + incomplete: ' ', + clear: true, + stream: process.stdout, + total: totalBytes + }) +} diff --git a/test/cli/progress-bar.js b/test/cli/progress-bar.js new file mode 100644 index 0000000000..21708a59f8 --- /dev/null +++ b/test/cli/progress-bar.js @@ -0,0 +1,15 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const createProgressBar = require('../../src/cli/utils').createProgressBar + +describe('progress bar', () => { + it('created with the correct properties', () => { + const total = 1000 + + const bar = createProgressBar(total) + expect(bar.total).to.eql(total) + expect(typeof bar.tick).to.eql('function') + }) +}) From 8ea66f9aac72abe2829d3905a8b5b51a50ae7be4 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 10 Oct 2017 00:06:59 -0600 Subject: [PATCH 04/11] feat: add missing progress bar support for http api --- package.json | 6 +++--- src/http/api/resources/files.js | 36 ++++++++++++++++++++++++++------- test/cli/files.js | 8 ++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index fa551da129..8802bcfc7a 100644 --- a/package.json +++ b/package.json @@ -94,9 +94,8 @@ "bl": "^1.2.1", "boom": "^5.2.0", "byteman": "^1.3.5", - "cids": "~0.5.1", - "debug": "^3.0.1", "cids": "^0.5.1", + "debug": "^3.0.1", "file-type": "^6.1.0", "filesize": "^3.5.10", "fsm-event": "^2.1.0", @@ -144,6 +143,7 @@ "progress": "^2.0.0", "promisify-es6": "^1.0.3", "pull-file": "^1.0.0", + "pull-ndjson": "^0.1.1", "pull-paramap": "^1.2.2", "pull-pushable": "^2.1.1", "pull-sort": "^1.0.1", @@ -220,4 +220,4 @@ "Łukasz Magiera ", "ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ " ] -} \ No newline at end of file +} diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 35c53a32e6..e6ba96184a 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -9,9 +9,9 @@ log.error = debug('jsipfs:http-api:files:error') const pull = require('pull-stream') const toPull = require('stream-to-pull-stream') const pushable = require('pull-pushable') -const EOL = require('os').EOL const toStream = require('pull-stream-to-stream') const Joi = require('joi') +const ndjson = require('pull-ndjson') exports = module.exports @@ -104,7 +104,7 @@ exports.get = { pull( stream, pull.asyncMap((file, cb) => { - const header = {name: file.path} + const header = { name: file.path } if (!file.content) { header.type = 'directory' pack.entry(header) @@ -207,10 +207,34 @@ exports.add = { fileAdder.end() }) + const replyStream = pushable() + const progressHandler = (bytes) => { + replyStream.push({ Bytes: bytes }) + } + const options = { 'cid-version': request.query['cid-version'], - 'raw-leaves': request.query['raw-leaves'] + 'raw-leaves': request.query['raw-leaves'], + progress: request.query['progress'] ? progressHandler : null + } + + const stream = toStream.source(pull( + replyStream, + ndjson.serialize() + )) + + // const stream = toStream.source(replyStream.source) + // hapi is not very clever and throws if no + // - _read method + // - _readableState object + // are there :( + if (!stream._read) { + stream._read = () => {} + stream._readableState = {} } + reply(stream) + .header('x-chunked-output', '1') + .header('content-type', 'application/json') pull( fileAdder, @@ -221,7 +245,6 @@ exports.add = { Hash: file.hash } }), - pull.map((file) => JSON.stringify(file) + EOL), pull.collect((err, files) => { if (err) { return reply({ @@ -237,9 +260,8 @@ exports.add = { }).code(500) } - reply(files.join('\n')) - .header('x-chunked-output', '1') - .header('content-type', 'application/json') + files.forEach((f) => replyStream.push(f)) + replyStream.end() }) ) } diff --git a/test/cli/files.js b/test/cli/files.js index afcf581112..93da6eb59f 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -106,6 +106,14 @@ describe('files', () => runOnAndOff((thing) => { ipfs = thing.ipfs }) + it('add with progress', () => { + return ipfs('files add -p src/init-files/init-docs/readme') + .then((out) => { + expect(out) + .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') + }) + }) + it('add', () => { return ipfs('files add src/init-files/init-docs/readme') .then((out) => { From 480896ab1ddada67e239d078ef064d20a321e4f2 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Tue, 10 Oct 2017 02:06:59 -0600 Subject: [PATCH 05/11] feat: add error propagation --- package.json | 1 + src/http/api/resources/files.js | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8802bcfc7a..b0663ba2ac 100644 --- a/package.json +++ b/package.json @@ -142,6 +142,7 @@ "peer-info": "~0.11.0", "progress": "^2.0.0", "promisify-es6": "^1.0.3", + "pull-abortable": "^4.1.1", "pull-file": "^1.0.0", "pull-ndjson": "^0.1.1", "pull-paramap": "^1.2.2", diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index e6ba96184a..28ab40fa65 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -10,6 +10,7 @@ const pull = require('pull-stream') const toPull = require('stream-to-pull-stream') const pushable = require('pull-pushable') const toStream = require('pull-stream-to-stream') +const abortable = require('pull-abortable') const Joi = require('joi') const ndjson = require('pull-ndjson') @@ -218,8 +219,10 @@ exports.add = { progress: request.query['progress'] ? progressHandler : null } + const aborter = abortable() const stream = toStream.source(pull( replyStream, + aborter, ndjson.serialize() )) @@ -231,6 +234,7 @@ exports.add = { if (!stream._read) { stream._read = () => {} stream._readableState = {} + stream.unpipe = () => {} } reply(stream) .header('x-chunked-output', '1') @@ -247,17 +251,19 @@ exports.add = { }), pull.collect((err, files) => { if (err) { - return reply({ + replyStream.push({ Message: err, Code: 0 - }).code(500) + }) + return aborter.abort() } if (files.length === 0 && filesParsed) { - return reply({ + replyStream.push({ Message: 'Failed to add files.', Code: 0 - }).code(500) + }) + return aborter.abort() } files.forEach((f) => replyStream.push(f)) From 0a7092df7a0f9130a56e8becdfd951f9c3e14e32 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 11 Oct 2017 22:55:22 -0700 Subject: [PATCH 06/11] feat: handle errors as trailer headers --- src/http/api/resources/files.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index 28ab40fa65..b03753d1a8 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -239,6 +239,15 @@ exports.add = { reply(stream) .header('x-chunked-output', '1') .header('content-type', 'application/json') + .header('Trailer', 'X-Stream-Error') + + function _writeErr (msg, code) { + const err = JSON.stringify({ Message: msg, Code: code }) + request.raw.res.addTrailers({ + 'X-Stream-Error': err + }) + return aborter.abort() + } pull( fileAdder, @@ -251,19 +260,11 @@ exports.add = { }), pull.collect((err, files) => { if (err) { - replyStream.push({ - Message: err, - Code: 0 - }) - return aborter.abort() + return _writeErr(err, 0) } if (files.length === 0 && filesParsed) { - replyStream.push({ - Message: 'Failed to add files.', - Code: 0 - }) - return aborter.abort() + return _writeErr('Failed to add files.', 0) } files.forEach((f) => replyStream.push(f)) From 50d1d2b2bcdf7a2b4430b673704a041bdcbd6d95 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 18 Oct 2017 06:18:47 -0700 Subject: [PATCH 07/11] feat: acumulate progress bytes sent for consistency with go --- src/core/components/files.js | 10 +++++++++- src/http/api/resources/files.js | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/components/files.js b/src/core/components/files.js index 6a9f5fbced..7dc619bf9c 100644 --- a/src/core/components/files.js +++ b/src/core/components/files.js @@ -54,7 +54,7 @@ module.exports = function files (self) { add: promisify((data, options, callback) => { if (typeof options === 'function') { callback = options - options = undefined + options = {} } else if (!callback || typeof callback !== 'function') { callback = noop } @@ -65,6 +65,14 @@ module.exports = function files (self) { return callback(new Error('Invalid arguments, data must be an object, Buffer or readable stream')) } + let total = 0 + let prog = options.progress || (() => {}) + const progress = (bytes) => { + total += bytes + prog(total) + } + + options.progress = progress pull( pull.values(normalizeContent(data)), importer(self._ipldResolver, options), diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index b03753d1a8..f1e6c61377 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -209,8 +209,10 @@ exports.add = { }) const replyStream = pushable() + let total = 0 const progressHandler = (bytes) => { - replyStream.push({ Bytes: bytes }) + total += bytes + replyStream.push({ Bytes: total }) } const options = { From 6086a722be09f245e052a901fd1e605e6eb360b5 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 18 Oct 2017 06:44:18 -0700 Subject: [PATCH 08/11] chore: upgrade to latest interface-ipfs-core --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b0663ba2ac..5560be6a72 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "expose-loader": "^0.7.3", "form-data": "^2.3.1", "gulp": "^3.9.1", - "interface-ipfs-core": "~0.31.19", + "interface-ipfs-core": "^0.32.1", "ipfsd-ctl": "~0.23.0", "left-pad": "^1.1.3", "lodash": "^4.17.4", From f953cdb486f54315a79c71d178e2607a0507b6de Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 18 Oct 2017 12:02:44 -0700 Subject: [PATCH 09/11] chore: upgrade ipfs-api to latest --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5560be6a72..eb53c4615c 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", "hoek": "^4.2.0", - "ipfs-api": "^14.3.5", + "ipfs-api": "^14.3.7", "ipfs-bitswap": "~0.17.2", "ipfs-block": "~0.6.0", "ipfs-block-service": "~0.12.0", From bd70fbd92c8c977f1bd5a212fcc5c4a741908f18 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Wed, 18 Oct 2017 15:04:41 -0700 Subject: [PATCH 10/11] fix: send file size along Name and Hash --- src/http/api/resources/files.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js index f1e6c61377..64466b1622 100644 --- a/src/http/api/resources/files.js +++ b/src/http/api/resources/files.js @@ -257,7 +257,8 @@ exports.add = { pull.map((file) => { return { Name: file.path ? file.path : file.hash, - Hash: file.hash + Hash: file.hash, + Size: file.size } }), pull.collect((err, files) => { From d880de5a3139eef5fbb418b23cf860cef7e47ccd Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 19 Oct 2017 10:46:25 +0100 Subject: [PATCH 11/11] fix semver --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index eb53c4615c..e3d3898482 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "expose-loader": "^0.7.3", "form-data": "^2.3.1", "gulp": "^3.9.1", - "interface-ipfs-core": "^0.32.1", + "interface-ipfs-core": "~0.32.1", "ipfsd-ctl": "~0.23.0", "left-pad": "^1.1.3", "lodash": "^4.17.4",