Merge pull request #73 from ipfs/fix-blocks

fix(exporter): add some parallel fetching of blocks where possible

Author: daviddias

package.json

@@ -54,6 +54,7 @@
     "is-ipfs": "^0.2.0",
     "multihashes": "^0.2.2",
     "pull-block": "^1.0.2",
+    "pull-paramap": "^1.1.6",
     "pull-pushable": "^2.0.1",
     "pull-stream": "^3.4.5",
     "pull-traverse": "^1.0.3",
@@ -70,4 +71,4 @@
     "jbenet <[email protected]>",
     "nginnever <[email protected]>"
   ]
-}
+}

src/exporters/dir.js

@@ -2,27 +2,40 @@
 
 const path = require('path')
 const pull = require('pull-stream')
+const paramap = require('pull-paramap')
 
 const fileExporter = require('./file')
 const switchType = require('../util').switchType
 
 // Logic to export a unixfs directory.
 module.exports = (node, name, dagService) => {
+  // The algorithm below is as follows
+  //
+  // 1. Take all links from a given directory node
+  // 2. Map each link to their full name (parent + link name) + hash
+  // 3. Parallel map to
+  // 3.1. Resolve the hash against the dagService
+  // 3.2. Switch on the node type
+  //      - `directory`: return node
+  //      - `file`: use the fileExporter to load and return the file
+  // 4. Flatten
   return pull(
     pull.values(node.links),
     pull.map((link) => ({
       path: path.join(name, link.name),
       hash: link.hash
     })),
-    pull.map((item) => pull(
-      dagService.getStream(item.hash),
-      pull.map((n) => switchType(
+    paramap((item, cb) => dagService.get(item.hash, (err, n) => {
+      if (err) {
+        return cb(err)
+      }
+
+      cb(null, switchType(
         n,
         () => pull.values([item]),
         () => fileExporter(n, item.path, dagService)
-      )),
-      pull.flatten()
-    )),
+      ))
+    })),
     pull.flatten()
   )
 }

src/exporters/file.js

@@ -3,6 +3,7 @@
 const traverse = require('pull-traverse')
 const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
+const paramap = require('pull-paramap')
 
 // Logic to export a single (possibly chunked) unixfs file.
 module.exports = (node, name, ds) => {
@@ -18,8 +19,7 @@ module.exports = (node, name, ds) => {
   function visitor (node) {
     return pull(
       pull.values(node.links),
-      pull.map((link) => ds.getStream(link.hash)),
-      pull.flatten()
+      paramap((link, cb) => ds.get(link.hash, cb))
     )
   }