From 43503d4321c710ffc42c9ea1eabfb25795865790 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Fri, 16 Sep 2016 18:28:45 +0200 Subject: [PATCH 1/2] fix(exporter): add some parallel fetching of blocks where possible --- package.json | 3 ++- src/exporters/dir.js | 15 +++++++++------ src/exporters/file.js | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index a2e0b519..4b33957f 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "is-ipfs": "^0.2.0", "multihashes": "^0.2.2", "pull-block": "^1.0.2", + "pull-paramap": "^1.1.6", "pull-pushable": "^2.0.1", "pull-stream": "^3.4.5", "pull-traverse": "^1.0.3", @@ -70,4 +71,4 @@ "jbenet ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/exporters/dir.js b/src/exporters/dir.js index 98e01417..779d2d8b 100644 --- a/src/exporters/dir.js +++ b/src/exporters/dir.js @@ -2,6 +2,7 @@ const path = require('path') const pull = require('pull-stream') +const paramap = require('pull-paramap') const fileExporter = require('./file') const switchType = require('../util').switchType @@ -14,15 +15,17 @@ module.exports = (node, name, dagService) => { path: path.join(name, link.name), hash: link.hash })), - pull.map((item) => pull( - dagService.getStream(item.hash), - pull.map((n) => switchType( + paramap((item, cb) => dagService.get(item.hash, (err, n) => { + if (err) { + return cb(err) + } + + cb(null, switchType( n, () => pull.values([item]), () => fileExporter(n, item.path, dagService) - )), - pull.flatten() - )), + )) + })), pull.flatten() ) } diff --git a/src/exporters/file.js b/src/exporters/file.js index a468047c..49deb561 100644 --- a/src/exporters/file.js +++ b/src/exporters/file.js @@ -3,6 +3,7 @@ const traverse = require('pull-traverse') const UnixFS = require('ipfs-unixfs') const pull = require('pull-stream') +const paramap = require('pull-paramap') // Logic to export a single (possibly chunked) unixfs file. module.exports = (node, name, ds) => { @@ -18,8 +19,7 @@ module.exports = (node, name, ds) => { function visitor (node) { return pull( pull.values(node.links), - pull.map((link) => ds.getStream(link.hash)), - pull.flatten() + paramap((link, cb) => ds.get(link.hash, cb)) ) } From a9d303cc3619014f676213dcd5ad634514b0017a Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Sun, 25 Sep 2016 19:31:36 +0200 Subject: [PATCH 2/2] docs(exporter): describe directory exporter --- src/exporters/dir.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/exporters/dir.js b/src/exporters/dir.js index 779d2d8b..75aa08f6 100644 --- a/src/exporters/dir.js +++ b/src/exporters/dir.js @@ -9,6 +9,16 @@ const switchType = require('../util').switchType // Logic to export a unixfs directory. module.exports = (node, name, dagService) => { + // The algorithm below is as follows + // + // 1. Take all links from a given directory node + // 2. Map each link to their full name (parent + link name) + hash + // 3. Parallel map to + // 3.1. Resolve the hash against the dagService + // 3.2. Switch on the node type + // - `directory`: return node + // - `file`: use the fileExporter to load and return the file + // 4. Flatten return pull( pull.values(node.links), pull.map((link) => ({