From 3771a195c311650be4dc89d290ce1b46b2827b25 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 8 Oct 2019 08:40:47 +0100 Subject: [PATCH 1/4] feat: add mssing `dag put` and `dag resolve` cli commands Also allows passing input to `ipfs-exec` to simulate piping in cli tests. --- src/cli/commands/dag/put.js | 127 +++++++++++++++++++++++++++++++ src/cli/commands/dag/resolve.js | 45 +++++++++++ src/core/components/dag.js | 106 +++++++++++++------------- test/cli/commands.js | 2 +- test/cli/daemon.js | 12 ++- test/cli/dag.js | 130 +++++++++++++++++++++++++++++--- test/cli/files.js | 2 +- test/cli/name-pubsub.js | 2 +- test/cli/swarm.js | 4 +- test/utils/ipfs-exec.js | 51 +++++++------ 10 files changed, 388 insertions(+), 93 deletions(-) create mode 100644 src/cli/commands/dag/put.js create mode 100644 src/cli/commands/dag/resolve.js diff --git a/src/cli/commands/dag/put.js b/src/cli/commands/dag/put.js new file mode 100644 index 0000000000..9a722e8386 --- /dev/null +++ b/src/cli/commands/dag/put.js @@ -0,0 +1,127 @@ +'use strict' + +const mh = require('multihashes') +const multibase = require('multibase') +const dagCBOR = require('ipld-dag-cbor') +const dagPB = require('ipld-dag-pb') +const { cidToString } = require('../../../utils/cid') + +const inputDecoders = { + json: (buf) => JSON.parse(buf.toString()), + cbor: (buf) => dagCBOR.util.deserialize(buf), + protobuf: (buf) => dagPB.util.deserialize(buf), + raw: (buf) => buf +} + +const formats = { + cbor: 'dag-cbor', + raw: 'raw', + protobuf: 'dag-pb', + 'dag-cbor': 'dag-cbor', + 'dag-pb': 'dag-pb' +} + +module.exports = { + command: 'put [data]', + + describe: 'accepts input from a file or stdin and parses it into an object of the specified format', + + builder: { + data: { + type: 'string' + }, + format: { + type: 'string', + alias: 'f', + default: 'cbor', + describe: 'Format that the object will be added as', + choices: ['dag-cbor', 'dag-pb', 'raw', 'cbor', 'protobuf'] + }, + 'input-encoding': { + type: 'string', + alias: 'input-enc', + default: 'json', + describe: 'Format that the input object will be', + choices: ['json', 'cbor', 'raw', 'protobuf'] + }, + pin: { + type: 'boolean', + default: true, + describe: 'Pin this object when adding' + }, + 'hash-alg': { + type: 'string', + alias: 'hash', + default: 'sha2-256', + describe: 'Hash function to use', + choices: Object.keys(mh.names) + }, + 'cid-version': { + type: 'integer', + describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed', + default: 0 + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + }, + preload: { + type: 'boolean', + default: true, + describe: 'Preload this object when adding' + }, + 'only-hash': { + type: 'boolean', + default: false, + describe: 'Only hash the content, do not write to the underlying block store' + } + }, + + handler ({ data, format, inputEncoding, pin, hashAlg, cidVersion, cidBase, preload, onlyHash, getIpfs, print, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + + if (inputEncoding === 'cbor') { + format = 'dag-cbor' + } else if (inputEncoding === 'protobuf') { + format = 'dag-pb' + } + + format = formats[format] + + if (format !== 'dag-pb') { + cidVersion = 1 + } + + let source = data + + if (!source) { + // pipe from stdin + source = Buffer.alloc(0) + + for await (const buf of process.stdin) { + source = Buffer.concat([source, buf]) + } + } else { + source = Buffer.from(source) + } + + source = inputDecoders[inputEncoding](source) + + const cid = await ipfs.dag.put(source, { + format, + hashAlg, + version: cidVersion, + onlyHash, + preload + }) + + if (pin) { + await ipfs.pin.add(cid) + } + + print(cidToString(cid, { base: cidBase })) + })()) + } +} diff --git a/src/cli/commands/dag/resolve.js b/src/cli/commands/dag/resolve.js new file mode 100644 index 0000000000..bba7886034 --- /dev/null +++ b/src/cli/commands/dag/resolve.js @@ -0,0 +1,45 @@ +'use strict' + +const CID = require('cids') + +module.exports = { + command: 'resolve ', + + describe: 'fetches a dag node from ipfs, prints its address and remaining path', + + builder: { + ref: { + type: 'string' + } + }, + + handler ({ ref, getIpfs, print, resolve }) { + resolve((async () => { + const ipfs = await getIpfs() + const options = {} + + try { + const result = await ipfs.dag.resolve(ref, options) + let lastCid + + for (const res of result) { + if (CID.isCID(res.value)) { + lastCid = res.value + } + } + + if (!lastCid) { + if (ref.startsWith('/ipfs/')) { + ref = ref.substring(6) + } + + lastCid = ref.split('/').shift() + } + + print(lastCid.toString()) + } catch (err) { + return print(`dag get resolve: ${err}`) + } + })()) + } +} diff --git a/src/core/components/dag.js b/src/core/components/dag.js index af0d745e2c..42c99e75d5 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -6,6 +6,50 @@ const all = require('async-iterator-all') const errCode = require('err-code') const multicodec = require('multicodec') +function parseArgs (cid, path, options) { + options = options || {} + + // Allow options in path position + if (path !== undefined && typeof path !== 'string') { + options = path + path = undefined + } + + if (typeof cid === 'string') { + if (cid.startsWith('/ipfs/')) { + cid = cid.substring(6) + } + + const split = cid.split('/') + + try { + cid = new CID(split[0]) + } catch (err) { + throw errCode(err, 'ERR_INVALID_CID') + } + + split.shift() + + if (split.length > 0) { + path = split.join('/') + } else { + path = path || '/' + } + } else if (Buffer.isBuffer(cid)) { + try { + cid = new CID(cid) + } catch (err) { + throw errCode(err, 'ERR_INVALID_CID') + } + } + + return [ + cid, + path, + options + ] +} + module.exports = function dag (self) { return { put: callbackify.variadic(async (dagNode, options) => { @@ -57,37 +101,7 @@ module.exports = function dag (self) { }), get: callbackify.variadic(async (cid, path, options) => { - options = options || {} - - // Allow options in path position - if (path !== undefined && typeof path !== 'string') { - options = path - path = undefined - } - - if (typeof cid === 'string') { - const split = cid.split('/') - - try { - cid = new CID(split[0]) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } - - split.shift() - - if (split.length > 0) { - path = split.join('/') - } else { - path = path || '/' - } - } else if (Buffer.isBuffer(cid)) { - try { - cid = new CID(cid) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } - } + [cid, path, options] = parseArgs(cid, path, options) if (options.preload !== false) { self._preload(cid) @@ -116,37 +130,23 @@ module.exports = function dag (self) { }), tree: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await - options = options || {} + [cid, path, options] = parseArgs(cid, path, options) - // Allow options in path position - if (path !== undefined && typeof path !== 'string') { - options = path - path = undefined + if (options.preload !== false) { + self._preload(cid) } - if (typeof cid === 'string') { - const split = cid.split('/') - - try { - cid = new CID(split[0]) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } + return all(self._ipld.tree(cid, path, options)) + }), - split.shift() - - if (split.length > 0) { - path = split.join('/') - } else { - path = undefined - } - } + resolve: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await + [cid, path, options] = parseArgs(cid, path, options) if (options.preload !== false) { self._preload(cid) } - return all(self._ipld.tree(cid, path, options)) + return all(self._ipld.resolve(cid, path)) }) } } diff --git a/test/cli/commands.js b/test/cli/commands.js index 23cc091bd2..e2390ef250 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const runOnAndOff = require('../utils/on-and-off') -const commandCount = 98 +const commandCount = 100 describe('commands', () => runOnAndOff((thing) => { let ipfs diff --git a/test/cli/daemon.js b/test/cli/daemon.js index 0bee3e507e..005bf4fb17 100644 --- a/test/cli/daemon.js +++ b/test/cli/daemon.js @@ -27,6 +27,10 @@ const daemonReady = (daemon) => { reject(new Error('Daemon didn\'t start ' + data.toString('utf8'))) } }) + + daemon.catch(err => { + reject(err) + }) }) } const checkLock = (repo) => { @@ -128,8 +132,8 @@ describe('daemon', () => { ] await ipfs('init') - await ipfs('config', 'Addresses.API', JSON.stringify(apiAddrs), '--json') - await ipfs('config', 'Addresses.Gateway', JSON.stringify(gatewayAddrs), '--json') + await ipfs(`config Addresses.API ${JSON.stringify(apiAddrs)} --json`) + await ipfs(`config Addresses.Gateway ${JSON.stringify(gatewayAddrs)} --json`) const daemon = ipfs('daemon') let stdout = '' @@ -157,8 +161,8 @@ describe('daemon', () => { this.timeout(100 * 1000) await ipfs('init') - await ipfs('config', 'Addresses.API', '[]', '--json') - await ipfs('config', 'Addresses.Gateway', '[]', '--json') + await ipfs('config Addresses.API [] --json') + await ipfs('config Addresses.Gateway [] --json') const daemon = ipfs('daemon') let stdout = '' diff --git a/test/cli/dag.js b/test/cli/dag.js index 8fbad2e458..cab2460864 100644 --- a/test/cli/dag.js +++ b/test/cli/dag.js @@ -4,6 +4,8 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const runOnAndOff = require('../utils/on-and-off') const path = require('path') +const dagCBOR = require('ipld-dag-cbor') +const dagPB = require('ipld-dag-pb') describe('dag', () => runOnAndOff.off((thing) => { let ipfs @@ -12,16 +14,126 @@ describe('dag', () => runOnAndOff.off((thing) => { ipfs = thing.ipfs }) - it('get', async function () { - this.timeout(20 * 1000) + before(async function () { + this.timeout(50 * 1000) + ipfs = thing.ipfs + await ipfs('add -r test/fixtures/test-data/recursive-get-dir') + }) + + describe('get', () => { + it('get', async function () { + this.timeout(20 * 1000) + + // put test eth-block + const out = await ipfs(`block put --format eth-block --mhtype keccak-256 ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/eth-block`) + expect(out).to.eql('bagiacgzarkhijr4xmbp345ovwwxra7kcecrnwcwtl7lg3g7d2ogyprdswjwq\n') + + // lookup path on eth-block + const out2 = await ipfs('dag get bagiacgzarkhijr4xmbp345ovwwxra7kcecrnwcwtl7lg3g7d2ogyprdswjwq/parentHash') + const expectHash = Buffer.from('c8c0a17305adea9bbb4b98a52d44f0c1478f5c48fc4b64739ee805242501b256', 'hex') + expect(out2).to.be.eql('0x' + expectHash.toString('hex') + '\n') + }) + }) + + describe('resolve', () => { + it('resolve cid', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag resolve Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z') + expect(out).to.equal('Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z\n') + }) + + it('resolve sub directory', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag resolve Qmaj2NmcyAXT8dFmZRRytE12wpcaHADzbChKToMEjBsj5Z/init-docs/tour/0.0-intro') + expect(out).to.equal('QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs\n') + }) + }) + + describe('put', () => { + it('puts json string', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put "{}"') + expect(out).to.equal('bafyreigbtj4x7ip5legnfznufuopl4sg4knzc2cof6duas4b3q2fy6swua\n') + }) + + it('puts piped json string', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put', { + input: Buffer.from('{}') + }) + expect(out).to.equal('bafyreigbtj4x7ip5legnfznufuopl4sg4knzc2cof6duas4b3q2fy6swua\n') + }) + + it('puts piped cbor node', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --input-encoding cbor', { + input: dagCBOR.util.serialize({}) + }) + expect(out).to.equal('bafyreigbtj4x7ip5legnfznufuopl4sg4knzc2cof6duas4b3q2fy6swua\n') + }) + + it('puts piped raw node', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --input-encoding raw --format raw', { + input: Buffer.alloc(10) + }) + expect(out).to.equal('bafkreiab2rek7wjiazkfrt3hbnqpljmu24226alszdlh6ivic2abgjubzi\n') + }) + + it('puts piped protobuf node', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --input-encoding protobuf --format protobuf', { + input: dagPB.util.serialize({}) + }) + expect(out).to.equal('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n\n') + }) + + it('puts protobuf node as json', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --format protobuf "{"Links":[]}"', { + input: dagPB.util.serialize({}) + }) + expect(out).to.equal('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n\n') + }) + + it('puts piped protobuf node with cid-v1', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --input-encoding protobuf --format protobuf --cid-version=1', { + input: dagPB.util.serialize({}) + }) + expect(out).to.equal('bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku\n') + }) + + it('puts json string with esoteric hashing algorithm', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --hash-alg blake2s-40 "{}"') + expect(out).to.equal('bafy4lzacausjadzcia\n') + }) + + it('puts json string with cid base', async function () { + this.timeout(20 * 1000) + + const out = await ipfs('dag put --cid-base base64 "{}"') + expect(out).to.equal('mAXESIMGaeX+h/VkM0uW0LRz18kbim5FoTi+HQEuB3DRcelag\n') + }) + + it('pins node after putting', async function () { + this.timeout(20 * 1000) - // put test eth-block - const out = await ipfs(`block put --format eth-block --mhtype keccak-256 ${path.resolve(path.join(__dirname, '..'))}/fixtures/test-data/eth-block`) - expect(out).to.eql('bagiacgzarkhijr4xmbp345ovwwxra7kcecrnwcwtl7lg3g7d2ogyprdswjwq\n') + const cid = (await ipfs('dag put --pin "{"hello":"world"}"')).trim() - // lookup path on eth-block - const out2 = await ipfs('dag get bagiacgzarkhijr4xmbp345ovwwxra7kcecrnwcwtl7lg3g7d2ogyprdswjwq/parentHash') - const expectHash = Buffer.from('c8c0a17305adea9bbb4b98a52d44f0c1478f5c48fc4b64739ee805242501b256', 'hex') - expect(out2).to.be.eql('0x' + expectHash.toString('hex') + '\n') + const out = await ipfs('pin ls') + expect(out).to.include(cid) + }) }) })) diff --git a/test/cli/files.js b/test/cli/files.js index 868fdd1f36..aef70f4a8b 100644 --- a/test/cli/files.js +++ b/test/cli/files.js @@ -143,7 +143,7 @@ describe('files', () => runOnAndOff((thing) => { it('add multiple', async function () { this.timeout(30 * 1000) - const out = await ipfs('add', 'src/init-files/init-docs/readme', 'test/fixtures/odd-name-[v0]/odd name [v1]/hello', '--wrap-with-directory') + const out = await ipfs('add src/init-files/init-docs/readme test/fixtures/odd-name-[v0]/odd\\ name\\ [v1]/hello --wrap-with-directory') expect(out) .to.include('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n') expect(out) diff --git a/test/cli/name-pubsub.js b/test/cli/name-pubsub.js index af899501d2..fb7d02a645 100644 --- a/test/cli/name-pubsub.js +++ b/test/cli/name-pubsub.js @@ -67,7 +67,7 @@ describe('name-pubsub', () => { // Connect before(async function () { - const out = await ipfsA('swarm', 'connect', bMultiaddr) + const out = await ipfsA(`swarm connect ${bMultiaddr}`) expect(out).to.eql(`connect ${bMultiaddr} success\n`) }) diff --git a/test/cli/swarm.js b/test/cli/swarm.js index 0ec1b1b00b..2b3b08f017 100644 --- a/test/cli/swarm.js +++ b/test/cli/swarm.js @@ -66,7 +66,7 @@ describe('swarm', () => { after(() => Promise.all(nodes.map((node) => node.stop()))) it('connect', async () => { - const out = await ipfsA('swarm', 'connect', bMultiaddr) + const out = await ipfsA(`swarm connect ${bMultiaddr}`) expect(out).to.eql(`connect ${bMultiaddr} success\n`) }) @@ -86,7 +86,7 @@ describe('swarm', () => { }) it('disconnect', async () => { - const out = await ipfsA('swarm', 'disconnect', bMultiaddr) + const out = await ipfsA(`swarm disconnect ${bMultiaddr}`) expect(out).to.eql( `disconnect ${bMultiaddr} success\n` ) diff --git a/test/utils/ipfs-exec.js b/test/utils/ipfs-exec.js index 3e002722b3..41f11a6fe2 100644 --- a/test/utils/ipfs-exec.js +++ b/test/utils/ipfs-exec.js @@ -23,22 +23,33 @@ module.exports = (repoPath, opts) => { env: env, timeout: 60 * 1000 }, opts) - const exec = (args) => execa(path.resolve(`${__dirname}/../../src/cli/bin.js`), args, config) - const execRaw = (args) => execa(path.resolve(`${__dirname}/../../src/cli/bin.js`), args, Object.assign({}, config, { - encoding: null - })) + const exec = (args, options) => { + const opts = Object.assign({}, config, options) - const execute = (exec, args) => { - if (args.length === 1) { - args = args[0].split(' ') - } + return execa.command(`${path.resolve(`${__dirname}/../../src/cli/bin.js`)} ${args}`, opts) + } + const execRaw = (args, options) => { + const opts = Object.assign({}, config, options, { + encoding: null + }) + + return execa.command(`${path.resolve(`${__dirname}/../../src/cli/bin.js`)} ${args}`, opts) + } - const cp = exec(args) + const execute = (exec, args, options) => { + const cp = exec(args, options) const res = cp.then((res) => { // We can't escape the os.tmpdir warning due to: // https://github.com/shelljs/shelljs/blob/master/src/tempdir.js#L43 // expect(res.stderr).to.be.eql('') return res.stdout + }, (err) => { + if (process.env.DEBUG) { + // print the error output if we are debugging + console.error(err.stderr) // eslint-disable-line no-console + } + + throw err }) res.cancel = cp.cancel.bind(cp) @@ -50,30 +61,26 @@ module.exports = (repoPath, opts) => { return res } - function ipfs () { - return execute(exec, Array.from(arguments)) + function ipfs (command, options) { + return execute(exec, command, options) } // Will return buffers instead of strings - ipfs.raw = function () { - return execute(execRaw, Array.from(arguments)) + ipfs.raw = function (command, options) { + return execute(execRaw, command, options) } /** * Expect the command passed as @param arguments to fail. + * @param {String} command String command to run, e.g. `'pin ls'` + * @param {Object} options Options to pass to `execa` * @return {Promise} Resolves if the command passed as @param arguments fails, * rejects if it was successful. */ - ipfs.fail = function ipfsFail () { - let args = Array.from(arguments) - - if (args.length === 1) { - args = args[0].split(' ') - } - - return exec(args) + ipfs.fail = function ipfsFail (command, options) { + return ipfs(command, options) .then(() => { - throw new Error(`jsipfs expected to fail during command: jsipfs ${args.join(' ')}`) + throw new Error(`jsipfs expected to fail during command: jsipfs ${command}`) }, (err) => { return err }) From 61bd4e5972f648a1b7c1af83a52925a1e65aca0d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 8 Oct 2019 09:18:49 +0100 Subject: [PATCH 2/4] chore: update interop dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a32cd13d41..ceae6d6f73 100644 --- a/package.json +++ b/package.json @@ -204,7 +204,7 @@ "form-data": "^2.5.1", "hat": "0.0.3", "interface-ipfs-core": "^0.117.2", - "ipfs-interop": "~0.1.0", + "ipfs-interop": "^0.1.1", "ipfsd-ctl": "^0.47.2", "libp2p-websocket-star": "~0.10.2", "ncp": "^2.0.0", From f330f0f64ab4e184ebfce2b9fc88f3324ea657d5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 17 Oct 2019 15:01:58 +0100 Subject: [PATCH 3/4] fix: do pinning inside core with gc lock --- src/cli/commands/dag/put.js | 7 ++----- src/core/components/dag.js | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cli/commands/dag/put.js b/src/cli/commands/dag/put.js index 9a722e8386..45ebd0345c 100644 --- a/src/cli/commands/dag/put.js +++ b/src/cli/commands/dag/put.js @@ -114,13 +114,10 @@ module.exports = { hashAlg, version: cidVersion, onlyHash, - preload + preload, + pin }) - if (pin) { - await ipfs.pin.add(cid) - } - print(cidToString(cid, { base: cidBase })) })()) } diff --git a/src/core/components/dag.js b/src/core/components/dag.js index 42c99e75d5..a539840e67 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -97,6 +97,12 @@ module.exports = function dag (self) { self._preload(cid) } + if (pin) { + await ipfs.pin.add(cid, { + lock: true + }) + } + return cid }), From 3d8a038d1281909169b76bfe40b1e9afda7b03c1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 17 Oct 2019 15:43:46 +0100 Subject: [PATCH 4/4] fix: wrap put in gc lock --- src/core/components/dag.js | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/core/components/dag.js b/src/core/components/dag.js index a539840e67..fd704e8139 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -88,22 +88,34 @@ module.exports = function dag (self) { } } - const cid = await self._ipld.put(dagNode, options.format, { - hashAlg: options.hashAlg, - cidVersion: options.version - }) + let release - if (options.preload !== false) { - self._preload(cid) + if (options.pin) { + release = await self._gcLock.readLock() } - if (pin) { - await ipfs.pin.add(cid, { - lock: true + try { + const cid = await self._ipld.put(dagNode, options.format, { + hashAlg: options.hashAlg, + cidVersion: options.version }) - } - return cid + if (options.pin) { + await self.pin.add(cid, { + lock: false + }) + } + + if (options.preload !== false) { + self._preload(cid) + } + + return cid + } finally { + if (release) { + release() + } + } }), get: callbackify.variadic(async (cid, path, options) => {