From 0f62d8ec368cc0cfd41f55471eac142d7f4c42aa Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Tue, 5 Sep 2017 15:25:46 +0100 Subject: [PATCH 1/5] Progress bar for cli --- package.json | 3 ++ src/cli/commands/files/add.js | 88 ++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 5e0c8f8669..0a3d4bfdef 100644 --- a/package.json +++ b/package.json @@ -92,9 +92,11 @@ "async": "^2.5.0", "bl": "^1.2.1", "boom": "^5.2.0", + "byteman": "^1.3.5", "cids": "~0.5.1", "debug": "^3.0.1", "fsm-event": "^2.1.0", + "get-folder-size": "^1.0.0", "glob": "^7.1.2", "hapi": "^16.5.2", "hapi-set-header": "^1.0.2", @@ -134,6 +136,7 @@ "peer-book": "~0.5.0", "peer-id": "~0.10.0", "peer-info": "~0.11.0", + "progress": "^2.0.0", "promisify-es6": "^1.0.3", "pull-file": "^1.0.0", "pull-paramap": "^1.2.2", diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index a28c9c1fb3..3c58b63e32 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,6 +8,10 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') +const Progress = require('progress') +const getFolderSize = require('get-folder-size') +const byteman = require('byteman') +const async = require('async') const utils = require('../../utils') const print = require('../../utils').print @@ -40,6 +44,27 @@ function checkPath (inPath, recursive) { return inPath } +function getTotalBytes (path, recursive, cb) { + if (recursive) { + getFolderSize(path, cb) + } else { + fs.stat(path, (err, stat) => cb(err, stat.size)) + } +} + +function createProgressBar (totalBytes) { + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` + + // 16 MB / 34 MB [=========== ] 48% 5.8s // + return new Progress(barFormat, { + incomplete: ' ', + clear: true, + stream: process.stdout, + total: totalBytes + }) +} + function addPipeline (index, addStream, list, wrapWithDirectory) { pull( zip( @@ -92,6 +117,12 @@ module.exports = { describe: 'Add a file to IPFS using the UnixFS data format', builder: { + progress: { + alias: 'p', + type: 'boolean', + default: true, + describe: 'Stream progress data' + }, recursive: { alias: 'r', type: 'boolean', @@ -158,34 +189,45 @@ module.exports = { } const ipfs = argv.ipfs - // TODO: revist when interface-ipfs-core exposes pull-streams - let createAddStream = (cb) => { - ipfs.files.createAddStream(options, (err, stream) => { - cb(err, err ? null : toPull.transform(stream)) - }) - } - - if (typeof ipfs.files.createAddPullStream === 'function') { - createAddStream = (cb) => { - cb(null, ipfs.files.createAddPullStream(options)) - } - } + let list = [] + let currentBytes = 0 + + async.waterfall([ + (next) => glob(path.join(inPath, '/**/*'), next), + (globResult, next) => { + list = globResult.length === 0 ? [inPath] : globResult + + getTotalBytes(inPath, argv.recursive, next) + }, + (totalBytes, next) => { + if (argv.progress) { + const bar = createProgressBar(totalBytes) + options.progress = function (byteLength) { + currentBytes += byteLength + bar.tick(byteLength, {progress: byteman(currentBytes, 2, 'MB')}) + } + } - createAddStream((err, addStream) => { - if (err) { - throw err - } + // TODO: revist when interface-ipfs-core exposes pull-streams - glob(path.join(inPath, '/**/*'), (err, list) => { - if (err) { - throw err + let createAddStream = (cb) => { + ipfs.files.createAddStream(options, (err, stream) => { + cb(err, err ? null : toPull.transform(stream)) + }) } - if (list.length === 0) { - list = [inPath] + + if (typeof ipfs.files.createAddPullStream === 'function') { + createAddStream = (cb) => { + cb(null, ipfs.files.createAddPullStream(options)) + } } - addPipeline(index, addStream, list, argv.wrapWithDirectory) - }) + createAddStream(next) + } + ], (err, addStream) => { + if (err) throw err + + addPipeline(index, addStream, list, argv.wrapWithDirectory) }) } } From e01bbc7a8083323d59198c01d956b4fc0326e8db Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Tue, 5 Sep 2017 16:00:55 +0100 Subject: [PATCH 2/5] requires async/waterfall in place of async --- src/cli/commands/files/add.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index 3c58b63e32..a441f255c4 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -11,7 +11,7 @@ const toPull = require('stream-to-pull-stream') const Progress = require('progress') const getFolderSize = require('get-folder-size') const byteman = require('byteman') -const async = require('async') +const waterfall = require('async/waterfall') const utils = require('../../utils') const print = require('../../utils').print @@ -192,7 +192,7 @@ module.exports = { let list = [] let currentBytes = 0 - async.waterfall([ + waterfall([ (next) => glob(path.join(inPath, '/**/*'), next), (globResult, next) => { list = globResult.length === 0 ? [inPath] : globResult From de122ba0af15c2e6edf5e0050d8cd9cd34dccb48 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Tue, 5 Sep 2017 19:28:22 +0100 Subject: [PATCH 3/5] Adds and corrects examples/traverse-ipld-graphs/git.js My PR failed to pass the Travis build because of examples/traverse-ipld-graphs/git.js I have had to add this file and correct the code style errors. This should make the tests pass for this PR. --- examples/traverse-ipld-graphs/git.js | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 examples/traverse-ipld-graphs/git.js diff --git a/examples/traverse-ipld-graphs/git.js b/examples/traverse-ipld-graphs/git.js new file mode 100644 index 0000000000..d72bcbe67b --- /dev/null +++ b/examples/traverse-ipld-graphs/git.js @@ -0,0 +1,73 @@ +'use strict' + +const createNode = require('./create-node.js') +const asyncEach = require('async/each') +const path = require('path') +const multihashing = require('multihashing-async') +const Block = require('ipfs-block') +const CID = require('cids') +const fs = require('fs') + +createNode((err, ipfs) => { + if (err) { + throw err + } + + console.log('\nStart of the example:') + + const gitObjects = [ + path.join(__dirname, '/git-objects/0f328c91df28c5c01b9e9f9f7e663191fa156593'), + path.join(__dirname, '/git-objects/177bf18bc707d82b21cdefd0b43b38fc8c5c13fe'), + path.join(__dirname, '/git-objects/23cc25f631cb076d5de5036c87678ea713cbaa6a'), + path.join(__dirname, '/git-objects/4e425dba7745a781f0712c9a01455899e8c0c249'), + path.join(__dirname, '/git-objects/6850c7be7136e6be00976ddbae80671b945c3e9d'), + path.join(__dirname, '/git-objects/a5095353cd62a178663dd26efc2d61f4f61bccbe'), + path.join(__dirname, '/git-objects/dc9bd15e8b81b6565d3736f9c308bd1bba60f33a'), + path.join(__dirname, '/git-objects/e68e6f6e31857877a79fd6b3956898436bb5a76f'), + path.join(__dirname, '/git-objects/ee62b3d206cb23f939208898f32d8708c0e3fa3c'), + path.join(__dirname, '/git-objects/ee71cef5001b84b0314438f76cf0acd338a2fd21') + ] + + asyncEach(gitObjects, (gitObjectsPath, cb) => { + const data = fs.readFileSync(gitObjectsPath) + + multihashing(data, 'sha1', (err, multihash) => { + if (err) { + return cb(err) + } + const cid = new CID(1, 'git-raw', multihash) + console.log(cid.toBaseEncodedString()) + + ipfs.block.put(new Block(data, cid), cb) + }) + }, (err) => { + if (err) { + throw err + } + + const v1tag = 'z8mWaGfwSWLMPJ6Q2JdsAjGiXTf61Nbue' + + function errOrLog (comment) { + return (err, result) => { + if (err) { + throw err + } + + if (Buffer.isBuffer(result.value)) { // Blobs (files) are returned as buffer instance + result.value = result.value.toString() + } + + console.log('-'.repeat(80)) + console.log(comment) + console.log(result.value) + } + } + + ipfs.dag.get(v1tag + '/', errOrLog('Tag object:')) + ipfs.dag.get(v1tag + '/object/message', errOrLog('Tagged commit message:')) + ipfs.dag.get(v1tag + '/object/parents/0/message', errOrLog('Parent of tagged commit:')) + + ipfs.dag.get(v1tag + '/object/tree/src/hash/hello/hash', errOrLog('/src/hello file:')) + ipfs.dag.get(v1tag + '/object/parents/0/tree/src/hash/hello/hash', errOrLog('previous version of /src/hello file:')) + }) +}) From bf05b66b88c440a594ca106249093180a607a622 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Thu, 7 Sep 2017 16:43:52 +0100 Subject: [PATCH 4/5] adds a test for createProgressBar --- src/cli/commands/files/add.js | 15 +-------------- src/cli/utils.js | 15 +++++++++++++++ test/cli/progress-bar.js | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 test/cli/progress-bar.js diff --git a/src/cli/commands/files/add.js b/src/cli/commands/files/add.js index a441f255c4..6126a131dd 100644 --- a/src/cli/commands/files/add.js +++ b/src/cli/commands/files/add.js @@ -8,12 +8,12 @@ const pull = require('pull-stream') const paramap = require('pull-paramap') const zip = require('pull-zip') const toPull = require('stream-to-pull-stream') -const Progress = require('progress') const getFolderSize = require('get-folder-size') const byteman = require('byteman') const waterfall = require('async/waterfall') const utils = require('../../utils') const print = require('../../utils').print +const createProgressBar = require('../../utils').createProgressBar const WRAPPER = 'wrapper/' @@ -52,19 +52,6 @@ function getTotalBytes (path, recursive, cb) { } } -function createProgressBar (totalBytes) { - const total = byteman(totalBytes, 2, 'MB') - const barFormat = `:progress / ${total} [:bar] :percent :etas` - - // 16 MB / 34 MB [=========== ] 48% 5.8s // - return new Progress(barFormat, { - incomplete: ' ', - clear: true, - stream: process.stdout, - total: totalBytes - }) -} - function addPipeline (index, addStream, list, wrapWithDirectory) { pull( zip( diff --git a/src/cli/utils.js b/src/cli/utils.js index 76ab67d969..4565fc316e 100644 --- a/src/cli/utils.js +++ b/src/cli/utils.js @@ -9,6 +9,8 @@ const path = require('path') const debug = require('debug') const log = debug('cli') log.error = debug('cli:error') +const Progress = require('progress') +const byteman = require('byteman') exports = module.exports @@ -85,3 +87,16 @@ exports.print = (msg, newline) => { process.stdout.write(msg) } } + +exports.createProgressBar = (totalBytes) => { + const total = byteman(totalBytes, 2, 'MB') + const barFormat = `:progress / ${total} [:bar] :percent :etas` + + // 16 MB / 34 MB [=========== ] 48% 5.8s // + return new Progress(barFormat, { + incomplete: ' ', + clear: true, + stream: process.stdout, + total: totalBytes + }) +} diff --git a/test/cli/progress-bar.js b/test/cli/progress-bar.js new file mode 100644 index 0000000000..21708a59f8 --- /dev/null +++ b/test/cli/progress-bar.js @@ -0,0 +1,15 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const createProgressBar = require('../../src/cli/utils').createProgressBar + +describe('progress bar', () => { + it('created with the correct properties', () => { + const total = 1000 + + const bar = createProgressBar(total) + expect(bar.total).to.eql(total) + expect(typeof bar.tick).to.eql('function') + }) +}) From 7513f6ef2aee696f480748564975b3add7686a72 Mon Sep 17 00:00:00 2001 From: Bernard Mordan Date: Fri, 8 Sep 2017 10:07:03 +0100 Subject: [PATCH 5/5] Update files.js --- test/cli/files.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cli/files.js b/test/cli/files.js index 85b7b38df3..85f7fbd9b4 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -8,7 +8,7 @@ const compareDir = require('dir-compare').compareSync const rimraf = require('rimraf').sync const runOnAndOff = require('../utils/on-and-off') -describe.only('files', () => runOnAndOff((thing) => { +describe('files', () => runOnAndOff((thing) => { let ipfs const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme')) .toString('utf-8')