From d2459b495fe84c3d178c004a80a8a4e9debbe59e Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 01:00:57 +1300 Subject: [PATCH 01/10] feat: cli for "file ls" --- src/cli/commands/file.js | 15 +++++++++++++++ src/cli/commands/file/ls.js | 27 +++++++++++++++++++++++++++ test/cli/file.js | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 src/cli/commands/file.js create mode 100644 src/cli/commands/file/ls.js create mode 100644 test/cli/file.js diff --git a/src/cli/commands/file.js b/src/cli/commands/file.js new file mode 100644 index 0000000000..b8a0d73224 --- /dev/null +++ b/src/cli/commands/file.js @@ -0,0 +1,15 @@ +'use strict' + +module.exports = { + command: 'file', + + description: ' Interact with IPFS objects representing Unix filesystems.', + + builder (yargs) { + return yargs + .commandDir('file') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js new file mode 100644 index 0000000000..58a1e72e0d --- /dev/null +++ b/src/cli/commands/file/ls.js @@ -0,0 +1,27 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'ls ', + + describe: 'List directory contents for Unix filesystem objects.', + + builder: {}, + + handler (argv) { + let path = argv.key + argv.ipfs.ls(path, (err, links) => { + if (err) { + throw err + } + + // Single file? Then print its hash + if (links.length === 0) { + links = [{hash: path}] + } + + links.forEach((file) => print(file.hash)) + }) + } +} diff --git a/test/cli/file.js b/test/cli/file.js new file mode 100644 index 0000000000..cfd683fedd --- /dev/null +++ b/test/cli/file.js @@ -0,0 +1,34 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const runOnAndOff = require('../utils/on-and-off') +const file = 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV' +const dir = 'QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2' + +describe('file ls', () => runOnAndOff((thing) => { + let ipfs + + before(function () { + this.timeout(50 * 1000) + ipfs = thing.ipfs + return ipfs('files add -r test/fixtures/test-data/recursive-get-dir') + }) + + it('prints a filename', () => { + return ipfs(`file ls ${file}`) + .then((out) => expect(out).to.eql(`${file}\n`)) + }) + + it('prints the filenames in a directory', () => { + return ipfs(`file ls ${dir}`) + .then((out) => expect(out).to.eql( + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV\n' + )) + }) + +})) From 37a9054a726b65f60bbeba84d4930f2e3cc41603 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 01:09:55 +1300 Subject: [PATCH 02/10] fix: typo --- src/cli/commands/file.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/commands/file.js b/src/cli/commands/file.js index b8a0d73224..261e50db00 100644 --- a/src/cli/commands/file.js +++ b/src/cli/commands/file.js @@ -3,7 +3,7 @@ module.exports = { command: 'file', - description: ' Interact with IPFS objects representing Unix filesystems.', + description: 'Interact with IPFS objects representing Unix filesystems.', builder (yargs) { return yargs From 8204e7c60b37617b132b32ab72fa14174c4bf451 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 01:30:05 +1300 Subject: [PATCH 03/10] chore: remove crlfs --- src/cli/commands/file.js | 28 +++++++-------- src/cli/commands/file/ls.js | 54 ++++++++++++++--------------- test/cli/file.js | 68 ++++++++++++++++++------------------- 3 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/cli/commands/file.js b/src/cli/commands/file.js index 261e50db00..5bc4dcf93f 100644 --- a/src/cli/commands/file.js +++ b/src/cli/commands/file.js @@ -1,15 +1,15 @@ -'use strict' - -module.exports = { - command: 'file', - +'use strict' + +module.exports = { + command: 'file', + description: 'Interact with IPFS objects representing Unix filesystems.', - - builder (yargs) { - return yargs - .commandDir('file') - }, - - handler (argv) { - } -} + + builder (yargs) { + return yargs + .commandDir('file') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js index 58a1e72e0d..e226046c5e 100644 --- a/src/cli/commands/file/ls.js +++ b/src/cli/commands/file/ls.js @@ -1,27 +1,27 @@ -'use strict' - -const print = require('../../utils').print - -module.exports = { - command: 'ls ', - - describe: 'List directory contents for Unix filesystem objects.', - - builder: {}, - - handler (argv) { - let path = argv.key - argv.ipfs.ls(path, (err, links) => { - if (err) { - throw err - } - - // Single file? Then print its hash - if (links.length === 0) { - links = [{hash: path}] - } - - links.forEach((file) => print(file.hash)) - }) - } -} +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'ls ', + + describe: 'List directory contents for Unix filesystem objects.', + + builder: {}, + + handler (argv) { + let path = argv.key + argv.ipfs.ls(path, (err, links) => { + if (err) { + throw err + } + + // Single file? Then print its hash + if (links.length === 0) { + links = [{hash: path}] + } + + links.forEach((file) => print(file.hash)) + }) + } +} diff --git a/test/cli/file.js b/test/cli/file.js index cfd683fedd..27861d0b45 100644 --- a/test/cli/file.js +++ b/test/cli/file.js @@ -1,34 +1,34 @@ -/* eslint-env mocha */ -'use strict' - -const expect = require('chai').expect -const runOnAndOff = require('../utils/on-and-off') -const file = 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV' -const dir = 'QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2' - -describe('file ls', () => runOnAndOff((thing) => { - let ipfs - - before(function () { - this.timeout(50 * 1000) - ipfs = thing.ipfs - return ipfs('files add -r test/fixtures/test-data/recursive-get-dir') - }) - - it('prints a filename', () => { - return ipfs(`file ls ${file}`) - .then((out) => expect(out).to.eql(`${file}\n`)) - }) - - it('prints the filenames in a directory', () => { - return ipfs(`file ls ${dir}`) - .then((out) => expect(out).to.eql( - 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' + - 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' + - 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' + - 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU\n' + - 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV\n' - )) - }) - -})) +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect +const runOnAndOff = require('../utils/on-and-off') +const file = 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV' +const dir = 'QmYmW4HiZhotsoSqnv2o1oUusvkRM8b9RweBoH7ao5nki2' + +describe('file ls', () => runOnAndOff((thing) => { + let ipfs + + before(function () { + this.timeout(50 * 1000) + ipfs = thing.ipfs + return ipfs('files add -r test/fixtures/test-data/recursive-get-dir') + }) + + it('prints a filename', () => { + return ipfs(`file ls ${file}`) + .then((out) => expect(out).to.eql(`${file}\n`)) + }) + + it('prints the filenames in a directory', () => { + return ipfs(`file ls ${dir}`) + .then((out) => expect(out).to.eql( + 'QmQQHYDwAQms78fPcvx1uFFsfho23YJNoewfLbi9AtdyJ9\n' + + 'QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN\n' + + 'Qma13ZrhKG52MWnwtZ6fMD8jGj8d4Q9sJgn5xtKgeZw5uz\n' + + 'QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU\n' + + 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV\n' + )) + }) + +})) From f75b36d9ddc7245613cab064d9598828541e85ab Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 01:36:54 +1300 Subject: [PATCH 04/10] chore: linting --- src/cli/commands/file/ls.js | 4 ++-- test/cli/file.js | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js index e226046c5e..f9a9736491 100644 --- a/src/cli/commands/file/ls.js +++ b/src/cli/commands/file/ls.js @@ -15,12 +15,12 @@ module.exports = { if (err) { throw err } - + // Single file? Then print its hash if (links.length === 0) { links = [{hash: path}] } - + links.forEach((file) => print(file.hash)) }) } diff --git a/test/cli/file.js b/test/cli/file.js index 27861d0b45..59ab531f6d 100644 --- a/test/cli/file.js +++ b/test/cli/file.js @@ -30,5 +30,4 @@ describe('file ls', () => runOnAndOff((thing) => { 'QmR56UJmAaZLXLdTT1ALrE9vVqV8soUEekm9BMd4FnuYqV\n' )) }) - })) From bbc3100dc45f425693f0714ec202291a95117ec8 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 01:58:16 +1300 Subject: [PATCH 05/10] chore: linting --- src/cli/commands/file/ls.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/commands/file/ls.js b/src/cli/commands/file/ls.js index f9a9736491..c952087b98 100644 --- a/src/cli/commands/file/ls.js +++ b/src/cli/commands/file/ls.js @@ -20,7 +20,7 @@ module.exports = { if (links.length === 0) { links = [{hash: path}] } - + links.forEach((file) => print(file.hash)) }) } From e0e4efb7fbacd88cc40493f462640587b3b49f89 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 14:07:41 +1300 Subject: [PATCH 06/10] feat: http-api for 'file ls' --- src/http/api/resources/file.js | 95 +++++++++++++++++++++++++++++++++ src/http/api/resources/index.js | 1 + src/http/api/routes/file.js | 20 +++++++ src/http/api/routes/index.js | 1 + 4 files changed, 117 insertions(+) create mode 100644 src/http/api/resources/file.js create mode 100644 src/http/api/routes/file.js diff --git a/src/http/api/resources/file.js b/src/http/api/resources/file.js new file mode 100644 index 0000000000..dec19ea98b --- /dev/null +++ b/src/http/api/resources/file.js @@ -0,0 +1,95 @@ +'use strict' + +const mh = require('multihashes') +const multipart = require('ipfs-multipart') +const debug = require('debug') +const tar = require('tar-stream') +const log = debug('jsipfs:http-api:files') +log.error = debug('jsipfs:http-api:files:error') +const pull = require('pull-stream') +const toPull = require('stream-to-pull-stream') +const pushable = require('pull-pushable') +const toStream = require('pull-stream-to-stream') +const abortable = require('pull-abortable') +const Joi = require('joi') +const ndjson = require('pull-ndjson') + +exports = module.exports + +// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` +exports.parseKey = (request, reply) => { + if (!request.query.arg) { + return reply({ + Message: "Argument 'key' is required", + Code: 0 + }).code(400).takeover() + } + + let key = request.query.arg + if (key.indexOf('/ipfs/') === 0) { + key = key.substring(6) + } + + let hash = key + const slashIndex = hash.indexOf('/') + if (slashIndex > 0) { + hash = hash.substring(0, slashIndex) + } + + try { + mh.fromB58String(hash) + } catch (err) { + log.error(err) + return reply({ + Message: 'invalid ipfs ref path', + Code: 0 + }).code(500).takeover() + } + + reply({ + path: request.query.arg, + key: key, + hash: hash + }) +} + +exports.ls = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + const path = request.pre.args.path + const hash = request.pre.args.hash + const ipfs = request.server.app.ipfs + + ipfs.ls(key, (err, files) => { + if (err) { + return reply({ + Message: 'Failed to list dir: ' + err.message, + Code: 0 + }).code(500) + } + + let res = { + Arguments: {}, + Objects: {} + } + res.Arguments[path] = key + res.Objects[key] = { + Hash: hash, + Size: 0, + Type: 'Directory', + Links: files.map((file) => ({ + Name: file.name, + Hash: file.hash, + Size: file.size, + Type: file.type + })) + } + reply(res) + + }) + } +} diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js index 671a349952..37bb316cab 100644 --- a/src/http/api/resources/index.js +++ b/src/http/api/resources/index.js @@ -9,5 +9,6 @@ exports.config = require('./config') exports.block = require('./block') exports.swarm = require('./swarm') exports.bitswap = require('./bitswap') +exports.file = require('./file') exports.files = require('./files') exports.pubsub = require('./pubsub') diff --git a/src/http/api/routes/file.js b/src/http/api/routes/file.js new file mode 100644 index 0000000000..6928783de4 --- /dev/null +++ b/src/http/api/routes/file.js @@ -0,0 +1,20 @@ +'use strict' + +const resources = require('./../resources') + +module.exports = (server) => { + const api = server.select('API') + + + api.route({ + // TODO fix method + method: '*', + path: '/api/v0/file/ls', + config: { + pre: [ + { method: resources.file.ls.parseArgs, assign: 'args' } + ], + handler: resources.file.ls.handler + } + }) +} diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js index c317db6de7..3fa1d75e68 100644 --- a/src/http/api/routes/index.js +++ b/src/http/api/routes/index.js @@ -10,6 +10,7 @@ module.exports = (server) => { require('./config')(server) require('./swarm')(server) require('./bitswap')(server) + require('./file')(server) require('./files')(server) require('./pubsub')(server) require('./debug')(server) From f04fa9956a4e1109a2889cfcbba2ddb4ab013187 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 14:29:38 +1300 Subject: [PATCH 07/10] fix: lint errors --- src/http/api/resources/file.js | 10 ---------- src/http/api/routes/file.js | 1 - test/cli/commands.js | 2 +- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/http/api/resources/file.js b/src/http/api/resources/file.js index dec19ea98b..58648baf1d 100644 --- a/src/http/api/resources/file.js +++ b/src/http/api/resources/file.js @@ -1,18 +1,9 @@ 'use strict' const mh = require('multihashes') -const multipart = require('ipfs-multipart') const debug = require('debug') -const tar = require('tar-stream') const log = debug('jsipfs:http-api:files') log.error = debug('jsipfs:http-api:files:error') -const pull = require('pull-stream') -const toPull = require('stream-to-pull-stream') -const pushable = require('pull-pushable') -const toStream = require('pull-stream-to-stream') -const abortable = require('pull-abortable') -const Joi = require('joi') -const ndjson = require('pull-ndjson') exports = module.exports @@ -89,7 +80,6 @@ exports.ls = { })) } reply(res) - }) } } diff --git a/src/http/api/routes/file.js b/src/http/api/routes/file.js index 6928783de4..1abf7456b2 100644 --- a/src/http/api/routes/file.js +++ b/src/http/api/routes/file.js @@ -5,7 +5,6 @@ const resources = require('./../resources') module.exports = (server) => { const api = server.select('API') - api.route({ // TODO fix method method: '*', diff --git a/test/cli/commands.js b/test/cli/commands.js index e6d9059d36..dc681ecebd 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') -const commandCount = 57 +const commandCount = 58 describe('commands', () => runOnAndOff((thing) => { let ipfs From 5c2231e652cb2b8083b9a2743d7ad27657440c87 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 14:59:57 +1300 Subject: [PATCH 08/10] fix: file type codes --- src/http/api/resources/file.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/http/api/resources/file.js b/src/http/api/resources/file.js index 58648baf1d..719733578a 100644 --- a/src/http/api/resources/file.js +++ b/src/http/api/resources/file.js @@ -7,6 +7,11 @@ log.error = debug('jsipfs:http-api:files:error') exports = module.exports +const fileTypeMap = { + file: 'File', + dir: 'Directory' +} + // common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` exports.parseKey = (request, reply) => { if (!request.query.arg) { @@ -76,7 +81,7 @@ exports.ls = { Name: file.name, Hash: file.hash, Size: file.size, - Type: file.type + Type: fileTypeMap[file.type] || file.type })) } reply(res) From bdc0729d600c45cca58f077357a7845094622898 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 20:28:01 +1300 Subject: [PATCH 09/10] refactor: talk directly to ipfs-unixfs-engine ipfs.ls does not provide enough information --- src/http/api/resources/file.js | 79 +++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/src/http/api/resources/file.js b/src/http/api/resources/file.js index 719733578a..cad0379fa4 100644 --- a/src/http/api/resources/file.js +++ b/src/http/api/resources/file.js @@ -2,8 +2,12 @@ const mh = require('multihashes') const debug = require('debug') -const log = debug('jsipfs:http-api:files') -log.error = debug('jsipfs:http-api:files:error') +const log = debug('jsipfs:http-api:file') +log.error = debug('jsipfs:http-api:file:error') +const unixfsEngine = require('ipfs-unixfs-engine') +const exporter = unixfsEngine.exporter +const pull = require('pull-stream') +const toB58String = require('multihashes').toB58String exports = module.exports @@ -12,6 +16,18 @@ const fileTypeMap = { dir: 'Directory' } +function toFileObject(file) { + const fo = { + Hash: toB58String(file.hash), + Size: file.size, + Type: fileTypeMap[file.type] || file.type + } + if (fo.Hash !== file.name) { + fo.Name = file.name + } + return fo; +} + // common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` exports.parseKey = (request, reply) => { if (!request.query.arg) { @@ -42,8 +58,11 @@ exports.parseKey = (request, reply) => { }).code(500).takeover() } + const subpaths = key.split('/') + subpaths.shift() reply({ path: request.query.arg, + subpaths: subpaths, key: key, hash: hash }) @@ -55,36 +74,38 @@ exports.ls = { // main route handler which is called after the above `parseArgs`, but only if the args were valid handler: (request, reply) => { - const key = request.pre.args.key const path = request.pre.args.path - const hash = request.pre.args.hash const ipfs = request.server.app.ipfs + const subpaths = request.pre.args.subpaths + const rootDepth = subpaths.length - ipfs.ls(key, (err, files) => { - if (err) { - return reply({ - Message: 'Failed to list dir: ' + err.message, - Code: 0 - }).code(500) - } + pull( + exporter(path, ipfs._ipldResolver, { maxDepth: rootDepth + 1 }), + pull.collect((err, files) => { + if (err) { + return reply({ + Message: 'Failed to list dir: ' + err.message, + Code: 0 + }).code(500) + } - let res = { - Arguments: {}, - Objects: {} - } - res.Arguments[path] = key - res.Objects[key] = { - Hash: hash, - Size: 0, - Type: 'Directory', - Links: files.map((file) => ({ - Name: file.name, - Hash: file.hash, - Size: file.size, - Type: fileTypeMap[file.type] || file.type - })) - } - reply(res) - }) + let res = { + Arguments: {}, + Objects: {} + } + const links = [] + files.forEach((file) => { + if (file.depth === rootDepth) { + let id = toB58String(file.hash) + res.Arguments[path] = id + res.Objects[id] = toFileObject(file) + res.Objects[id].Links = file.type === 'file' ? null : links; + } else { + links.push(toFileObject(file)) + } + }) + return reply(res) + }) + ) } } From eefad2da3e1be6cbc9eeb0fb0ec73ee2378cc329 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Thu, 16 Nov 2017 21:00:52 +1300 Subject: [PATCH 10/10] fix: lint errors --- src/http/api/resources/file.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http/api/resources/file.js b/src/http/api/resources/file.js index cad0379fa4..6c0b6b4c4c 100644 --- a/src/http/api/resources/file.js +++ b/src/http/api/resources/file.js @@ -16,7 +16,7 @@ const fileTypeMap = { dir: 'Directory' } -function toFileObject(file) { +function toFileObject (file) { const fo = { Hash: toB58String(file.hash), Size: file.size, @@ -25,7 +25,7 @@ function toFileObject(file) { if (fo.Hash !== file.name) { fo.Name = file.name } - return fo; + return fo } // common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` @@ -99,7 +99,7 @@ exports.ls = { let id = toB58String(file.hash) res.Arguments[path] = id res.Objects[id] = toFileObject(file) - res.Objects[id].Links = file.type === 'file' ? null : links; + res.Objects[id].Links = file.type === 'file' ? null : links } else { links.push(toFileObject(file)) }